python 2.6 linecache.getline() and stdin. How does it work?
I have a script which runs through lines of input to find the occurrence of an ID string while keeping track of the linenumber.
Then it runs backwards up the input to trace parentID/childID relationships. The script accepts either a logfile using a '-f' flag as an argument or the contents of stdin from a pipe.
The logfile as input portion works just fine, but reading from stdin seems not to work.
For the sake of reasonable clarity I've included the portion of the script that this concerns, but don't expect to be able to run it. It's just to show you sorta whats going on (anyone who works in financial services around FIX protocol would recognize a few things):
import os
import sys
import linecache
from types import *
from ____ import FixMessage # custom message class that is used throughout
# Feel free to ignore all the getArgs and validation crap
def getArgs():
import argparse
parser = argparse.ArgumentParser(
description='Get amendment history.')
parser.add_argument('-f', '--file',
help="input logfile.'")
args = parser.parse_args()
return validateArgs(args)
def validateArgs(args):
try:
if sys.stdin.isatty():
if args.file:
assert os.path.isfile(args.file.strip('n')),
'File "{0}" does not exist'.format(args.file)
args.file = open(args.file, 'r')
else:
args.file = sys.stdin
assert args.file,
"Please either include a file with '-f' or pipe some text in"
except AssertionError as err:
print err
exit(1)
return args
defGetMessageTrail(logfile, orderId):
# some input validation
if isinstance(logfile, StringType):
try: logfile = open(logfile, 'r')
except IOError as err: exit(1)
elif not isinstance(logfile, FileType):
raise TypeError(
'Expected FileType and got {0}'.format(type(logfile)))
linenum = 0
# This retrieves the message containing the orderID as well as the linenum
for line in logfile:
linenum += 1
if orderId in line:
# FixMessage is a custom class that is treated here like
# a dictionary with some metadata
# Missing dict keys return 'None'
# .isvalid is bool results of some text validation
# .direction is either incoming or outgoing
# thats all you really need to know
msg = FixMessage(line)
if msg.isvalid and msg.direction == 'Incoming':
yield msg
break
# If there is a message parentID, it would be in msg['41']
if msg['41']:
messages = findParentMessages(logfile, startline=linenum, msg['41'])
for msg in messages: yield msg
def findParentMessages(logfile, startline, targetId):
# Some more input validation
assert isinstance(logfile, FileType)
assert isinstance(startline, IntType)
assert isinstance(targetId, StringType)
# should just make a integer decrementing generator,
# but this is fine for the example
for linenum in range(startline)[::-1]:
# *** This is where the question lies... ***
# print(logfile.name) # returns "<stdin>"
line = linecache.getline(logfile.name, linenum)
if 'Incoming' in line and '11=' + targetId in line:
msg = FixMessage(line)
yield msg
if msg['41']: findParentMessages(logfile, linenum, msg['41'])
else: break
def main():
log = getArgs().file
trail = getMessageTrail(log, 'ORDER123')
if __name__ == '__main__': main()
The question is, how does linecache.getline work when it comes to reading stdin as a file? is it different than how it would work if given a regular filename?
python file-io python-2.6
add a comment |
I have a script which runs through lines of input to find the occurrence of an ID string while keeping track of the linenumber.
Then it runs backwards up the input to trace parentID/childID relationships. The script accepts either a logfile using a '-f' flag as an argument or the contents of stdin from a pipe.
The logfile as input portion works just fine, but reading from stdin seems not to work.
For the sake of reasonable clarity I've included the portion of the script that this concerns, but don't expect to be able to run it. It's just to show you sorta whats going on (anyone who works in financial services around FIX protocol would recognize a few things):
import os
import sys
import linecache
from types import *
from ____ import FixMessage # custom message class that is used throughout
# Feel free to ignore all the getArgs and validation crap
def getArgs():
import argparse
parser = argparse.ArgumentParser(
description='Get amendment history.')
parser.add_argument('-f', '--file',
help="input logfile.'")
args = parser.parse_args()
return validateArgs(args)
def validateArgs(args):
try:
if sys.stdin.isatty():
if args.file:
assert os.path.isfile(args.file.strip('n')),
'File "{0}" does not exist'.format(args.file)
args.file = open(args.file, 'r')
else:
args.file = sys.stdin
assert args.file,
"Please either include a file with '-f' or pipe some text in"
except AssertionError as err:
print err
exit(1)
return args
defGetMessageTrail(logfile, orderId):
# some input validation
if isinstance(logfile, StringType):
try: logfile = open(logfile, 'r')
except IOError as err: exit(1)
elif not isinstance(logfile, FileType):
raise TypeError(
'Expected FileType and got {0}'.format(type(logfile)))
linenum = 0
# This retrieves the message containing the orderID as well as the linenum
for line in logfile:
linenum += 1
if orderId in line:
# FixMessage is a custom class that is treated here like
# a dictionary with some metadata
# Missing dict keys return 'None'
# .isvalid is bool results of some text validation
# .direction is either incoming or outgoing
# thats all you really need to know
msg = FixMessage(line)
if msg.isvalid and msg.direction == 'Incoming':
yield msg
break
# If there is a message parentID, it would be in msg['41']
if msg['41']:
messages = findParentMessages(logfile, startline=linenum, msg['41'])
for msg in messages: yield msg
def findParentMessages(logfile, startline, targetId):
# Some more input validation
assert isinstance(logfile, FileType)
assert isinstance(startline, IntType)
assert isinstance(targetId, StringType)
# should just make a integer decrementing generator,
# but this is fine for the example
for linenum in range(startline)[::-1]:
# *** This is where the question lies... ***
# print(logfile.name) # returns "<stdin>"
line = linecache.getline(logfile.name, linenum)
if 'Incoming' in line and '11=' + targetId in line:
msg = FixMessage(line)
yield msg
if msg['41']: findParentMessages(logfile, linenum, msg['41'])
else: break
def main():
log = getArgs().file
trail = getMessageTrail(log, 'ORDER123')
if __name__ == '__main__': main()
The question is, how does linecache.getline work when it comes to reading stdin as a file? is it different than how it would work if given a regular filename?
python file-io python-2.6
add a comment |
I have a script which runs through lines of input to find the occurrence of an ID string while keeping track of the linenumber.
Then it runs backwards up the input to trace parentID/childID relationships. The script accepts either a logfile using a '-f' flag as an argument or the contents of stdin from a pipe.
The logfile as input portion works just fine, but reading from stdin seems not to work.
For the sake of reasonable clarity I've included the portion of the script that this concerns, but don't expect to be able to run it. It's just to show you sorta whats going on (anyone who works in financial services around FIX protocol would recognize a few things):
import os
import sys
import linecache
from types import *
from ____ import FixMessage # custom message class that is used throughout
# Feel free to ignore all the getArgs and validation crap
def getArgs():
import argparse
parser = argparse.ArgumentParser(
description='Get amendment history.')
parser.add_argument('-f', '--file',
help="input logfile.'")
args = parser.parse_args()
return validateArgs(args)
def validateArgs(args):
try:
if sys.stdin.isatty():
if args.file:
assert os.path.isfile(args.file.strip('n')),
'File "{0}" does not exist'.format(args.file)
args.file = open(args.file, 'r')
else:
args.file = sys.stdin
assert args.file,
"Please either include a file with '-f' or pipe some text in"
except AssertionError as err:
print err
exit(1)
return args
defGetMessageTrail(logfile, orderId):
# some input validation
if isinstance(logfile, StringType):
try: logfile = open(logfile, 'r')
except IOError as err: exit(1)
elif not isinstance(logfile, FileType):
raise TypeError(
'Expected FileType and got {0}'.format(type(logfile)))
linenum = 0
# This retrieves the message containing the orderID as well as the linenum
for line in logfile:
linenum += 1
if orderId in line:
# FixMessage is a custom class that is treated here like
# a dictionary with some metadata
# Missing dict keys return 'None'
# .isvalid is bool results of some text validation
# .direction is either incoming or outgoing
# thats all you really need to know
msg = FixMessage(line)
if msg.isvalid and msg.direction == 'Incoming':
yield msg
break
# If there is a message parentID, it would be in msg['41']
if msg['41']:
messages = findParentMessages(logfile, startline=linenum, msg['41'])
for msg in messages: yield msg
def findParentMessages(logfile, startline, targetId):
# Some more input validation
assert isinstance(logfile, FileType)
assert isinstance(startline, IntType)
assert isinstance(targetId, StringType)
# should just make a integer decrementing generator,
# but this is fine for the example
for linenum in range(startline)[::-1]:
# *** This is where the question lies... ***
# print(logfile.name) # returns "<stdin>"
line = linecache.getline(logfile.name, linenum)
if 'Incoming' in line and '11=' + targetId in line:
msg = FixMessage(line)
yield msg
if msg['41']: findParentMessages(logfile, linenum, msg['41'])
else: break
def main():
log = getArgs().file
trail = getMessageTrail(log, 'ORDER123')
if __name__ == '__main__': main()
The question is, how does linecache.getline work when it comes to reading stdin as a file? is it different than how it would work if given a regular filename?
python file-io python-2.6
I have a script which runs through lines of input to find the occurrence of an ID string while keeping track of the linenumber.
Then it runs backwards up the input to trace parentID/childID relationships. The script accepts either a logfile using a '-f' flag as an argument or the contents of stdin from a pipe.
The logfile as input portion works just fine, but reading from stdin seems not to work.
For the sake of reasonable clarity I've included the portion of the script that this concerns, but don't expect to be able to run it. It's just to show you sorta whats going on (anyone who works in financial services around FIX protocol would recognize a few things):
import os
import sys
import linecache
from types import *
from ____ import FixMessage # custom message class that is used throughout
# Feel free to ignore all the getArgs and validation crap
def getArgs():
import argparse
parser = argparse.ArgumentParser(
description='Get amendment history.')
parser.add_argument('-f', '--file',
help="input logfile.'")
args = parser.parse_args()
return validateArgs(args)
def validateArgs(args):
try:
if sys.stdin.isatty():
if args.file:
assert os.path.isfile(args.file.strip('n')),
'File "{0}" does not exist'.format(args.file)
args.file = open(args.file, 'r')
else:
args.file = sys.stdin
assert args.file,
"Please either include a file with '-f' or pipe some text in"
except AssertionError as err:
print err
exit(1)
return args
defGetMessageTrail(logfile, orderId):
# some input validation
if isinstance(logfile, StringType):
try: logfile = open(logfile, 'r')
except IOError as err: exit(1)
elif not isinstance(logfile, FileType):
raise TypeError(
'Expected FileType and got {0}'.format(type(logfile)))
linenum = 0
# This retrieves the message containing the orderID as well as the linenum
for line in logfile:
linenum += 1
if orderId in line:
# FixMessage is a custom class that is treated here like
# a dictionary with some metadata
# Missing dict keys return 'None'
# .isvalid is bool results of some text validation
# .direction is either incoming or outgoing
# thats all you really need to know
msg = FixMessage(line)
if msg.isvalid and msg.direction == 'Incoming':
yield msg
break
# If there is a message parentID, it would be in msg['41']
if msg['41']:
messages = findParentMessages(logfile, startline=linenum, msg['41'])
for msg in messages: yield msg
def findParentMessages(logfile, startline, targetId):
# Some more input validation
assert isinstance(logfile, FileType)
assert isinstance(startline, IntType)
assert isinstance(targetId, StringType)
# should just make a integer decrementing generator,
# but this is fine for the example
for linenum in range(startline)[::-1]:
# *** This is where the question lies... ***
# print(logfile.name) # returns "<stdin>"
line = linecache.getline(logfile.name, linenum)
if 'Incoming' in line and '11=' + targetId in line:
msg = FixMessage(line)
yield msg
if msg['41']: findParentMessages(logfile, linenum, msg['41'])
else: break
def main():
log = getArgs().file
trail = getMessageTrail(log, 'ORDER123')
if __name__ == '__main__': main()
The question is, how does linecache.getline work when it comes to reading stdin as a file? is it different than how it would work if given a regular filename?
python file-io python-2.6
python file-io python-2.6
edited Dec 17 at 16:56
DumbCoder
5,11032238
5,11032238
asked Nov 22 at 22:06
edaniley
84
84
add a comment |
add a comment |
1 Answer
1
active
oldest
votes
linecache.getline() accepts a file name, not a file object. It is not designed to work that way as filename is passed to calls like open() and os.stat().
For reference: https://github.com/python/cpython/blob/2.6/Lib/linecache.py
so is it impossible then to use stdin to pipe to the script? I could make a tmp file but I dont want to!
– edaniley
Dec 3 at 17:21
@edaniley I don't think it's possible with linecache. But it's much simpler to read the input with readlines() into a list of lines and use the element index as line number. linecache does the same under the hood.
– Benjámin Budai
Dec 4 at 14:03
the only problem I have is that these files potentially can be enormous
– edaniley
Dec 5 at 18:06
Then you are out of luck because linecache too reads the entire file into memory. For files on disk you could build a index and use that to seek back and forth, but stdin is a sequential read only file.
– Benjámin Budai
Dec 6 at 19:23
super helpful. Thank you very much!
– edaniley
Dec 10 at 20:18
add a comment |
Your Answer
StackExchange.ifUsing("editor", function () {
StackExchange.using("externalEditor", function () {
StackExchange.using("snippets", function () {
StackExchange.snippets.init();
});
});
}, "code-snippets");
StackExchange.ready(function() {
var channelOptions = {
tags: "".split(" "),
id: "1"
};
initTagRenderer("".split(" "), "".split(" "), channelOptions);
StackExchange.using("externalEditor", function() {
// Have to fire editor after snippets, if snippets enabled
if (StackExchange.settings.snippets.snippetsEnabled) {
StackExchange.using("snippets", function() {
createEditor();
});
}
else {
createEditor();
}
});
function createEditor() {
StackExchange.prepareEditor({
heartbeatType: 'answer',
autoActivateHeartbeat: false,
convertImagesToLinks: true,
noModals: true,
showLowRepImageUploadWarning: true,
reputationToPostImages: 10,
bindNavPrevention: true,
postfix: "",
imageUploader: {
brandingHtml: "Powered by u003ca class="icon-imgur-white" href="https://imgur.com/"u003eu003c/au003e",
contentPolicyHtml: "User contributions licensed under u003ca href="https://creativecommons.org/licenses/by-sa/3.0/"u003ecc by-sa 3.0 with attribution requiredu003c/au003e u003ca href="https://stackoverflow.com/legal/content-policy"u003e(content policy)u003c/au003e",
allowUrls: true
},
onDemand: true,
discardSelector: ".discard-answer"
,immediatelyShowMarkdownHelp:true
});
}
});
Sign up or log in
StackExchange.ready(function () {
StackExchange.helpers.onClickDraftSave('#login-link');
});
Sign up using Google
Sign up using Facebook
Sign up using Email and Password
Post as a guest
Required, but never shown
StackExchange.ready(
function () {
StackExchange.openid.initPostLogin('.new-post-login', 'https%3a%2f%2fstackoverflow.com%2fquestions%2f53438474%2fpython-2-6-linecache-getline-and-stdin-how-does-it-work%23new-answer', 'question_page');
}
);
Post as a guest
Required, but never shown
1 Answer
1
active
oldest
votes
1 Answer
1
active
oldest
votes
active
oldest
votes
active
oldest
votes
linecache.getline() accepts a file name, not a file object. It is not designed to work that way as filename is passed to calls like open() and os.stat().
For reference: https://github.com/python/cpython/blob/2.6/Lib/linecache.py
so is it impossible then to use stdin to pipe to the script? I could make a tmp file but I dont want to!
– edaniley
Dec 3 at 17:21
@edaniley I don't think it's possible with linecache. But it's much simpler to read the input with readlines() into a list of lines and use the element index as line number. linecache does the same under the hood.
– Benjámin Budai
Dec 4 at 14:03
the only problem I have is that these files potentially can be enormous
– edaniley
Dec 5 at 18:06
Then you are out of luck because linecache too reads the entire file into memory. For files on disk you could build a index and use that to seek back and forth, but stdin is a sequential read only file.
– Benjámin Budai
Dec 6 at 19:23
super helpful. Thank you very much!
– edaniley
Dec 10 at 20:18
add a comment |
linecache.getline() accepts a file name, not a file object. It is not designed to work that way as filename is passed to calls like open() and os.stat().
For reference: https://github.com/python/cpython/blob/2.6/Lib/linecache.py
so is it impossible then to use stdin to pipe to the script? I could make a tmp file but I dont want to!
– edaniley
Dec 3 at 17:21
@edaniley I don't think it's possible with linecache. But it's much simpler to read the input with readlines() into a list of lines and use the element index as line number. linecache does the same under the hood.
– Benjámin Budai
Dec 4 at 14:03
the only problem I have is that these files potentially can be enormous
– edaniley
Dec 5 at 18:06
Then you are out of luck because linecache too reads the entire file into memory. For files on disk you could build a index and use that to seek back and forth, but stdin is a sequential read only file.
– Benjámin Budai
Dec 6 at 19:23
super helpful. Thank you very much!
– edaniley
Dec 10 at 20:18
add a comment |
linecache.getline() accepts a file name, not a file object. It is not designed to work that way as filename is passed to calls like open() and os.stat().
For reference: https://github.com/python/cpython/blob/2.6/Lib/linecache.py
linecache.getline() accepts a file name, not a file object. It is not designed to work that way as filename is passed to calls like open() and os.stat().
For reference: https://github.com/python/cpython/blob/2.6/Lib/linecache.py
answered Nov 22 at 23:29
Benjámin Budai
587
587
so is it impossible then to use stdin to pipe to the script? I could make a tmp file but I dont want to!
– edaniley
Dec 3 at 17:21
@edaniley I don't think it's possible with linecache. But it's much simpler to read the input with readlines() into a list of lines and use the element index as line number. linecache does the same under the hood.
– Benjámin Budai
Dec 4 at 14:03
the only problem I have is that these files potentially can be enormous
– edaniley
Dec 5 at 18:06
Then you are out of luck because linecache too reads the entire file into memory. For files on disk you could build a index and use that to seek back and forth, but stdin is a sequential read only file.
– Benjámin Budai
Dec 6 at 19:23
super helpful. Thank you very much!
– edaniley
Dec 10 at 20:18
add a comment |
so is it impossible then to use stdin to pipe to the script? I could make a tmp file but I dont want to!
– edaniley
Dec 3 at 17:21
@edaniley I don't think it's possible with linecache. But it's much simpler to read the input with readlines() into a list of lines and use the element index as line number. linecache does the same under the hood.
– Benjámin Budai
Dec 4 at 14:03
the only problem I have is that these files potentially can be enormous
– edaniley
Dec 5 at 18:06
Then you are out of luck because linecache too reads the entire file into memory. For files on disk you could build a index and use that to seek back and forth, but stdin is a sequential read only file.
– Benjámin Budai
Dec 6 at 19:23
super helpful. Thank you very much!
– edaniley
Dec 10 at 20:18
so is it impossible then to use stdin to pipe to the script? I could make a tmp file but I dont want to!
– edaniley
Dec 3 at 17:21
so is it impossible then to use stdin to pipe to the script? I could make a tmp file but I dont want to!
– edaniley
Dec 3 at 17:21
@edaniley I don't think it's possible with linecache. But it's much simpler to read the input with readlines() into a list of lines and use the element index as line number. linecache does the same under the hood.
– Benjámin Budai
Dec 4 at 14:03
@edaniley I don't think it's possible with linecache. But it's much simpler to read the input with readlines() into a list of lines and use the element index as line number. linecache does the same under the hood.
– Benjámin Budai
Dec 4 at 14:03
the only problem I have is that these files potentially can be enormous
– edaniley
Dec 5 at 18:06
the only problem I have is that these files potentially can be enormous
– edaniley
Dec 5 at 18:06
Then you are out of luck because linecache too reads the entire file into memory. For files on disk you could build a index and use that to seek back and forth, but stdin is a sequential read only file.
– Benjámin Budai
Dec 6 at 19:23
Then you are out of luck because linecache too reads the entire file into memory. For files on disk you could build a index and use that to seek back and forth, but stdin is a sequential read only file.
– Benjámin Budai
Dec 6 at 19:23
super helpful. Thank you very much!
– edaniley
Dec 10 at 20:18
super helpful. Thank you very much!
– edaniley
Dec 10 at 20:18
add a comment |
Thanks for contributing an answer to Stack Overflow!
- Please be sure to answer the question. Provide details and share your research!
But avoid …
- Asking for help, clarification, or responding to other answers.
- Making statements based on opinion; back them up with references or personal experience.
To learn more, see our tips on writing great answers.
Some of your past answers have not been well-received, and you're in danger of being blocked from answering.
Please pay close attention to the following guidance:
- Please be sure to answer the question. Provide details and share your research!
But avoid …
- Asking for help, clarification, or responding to other answers.
- Making statements based on opinion; back them up with references or personal experience.
To learn more, see our tips on writing great answers.
Sign up or log in
StackExchange.ready(function () {
StackExchange.helpers.onClickDraftSave('#login-link');
});
Sign up using Google
Sign up using Facebook
Sign up using Email and Password
Post as a guest
Required, but never shown
StackExchange.ready(
function () {
StackExchange.openid.initPostLogin('.new-post-login', 'https%3a%2f%2fstackoverflow.com%2fquestions%2f53438474%2fpython-2-6-linecache-getline-and-stdin-how-does-it-work%23new-answer', 'question_page');
}
);
Post as a guest
Required, but never shown
Sign up or log in
StackExchange.ready(function () {
StackExchange.helpers.onClickDraftSave('#login-link');
});
Sign up using Google
Sign up using Facebook
Sign up using Email and Password
Post as a guest
Required, but never shown
Sign up or log in
StackExchange.ready(function () {
StackExchange.helpers.onClickDraftSave('#login-link');
});
Sign up using Google
Sign up using Facebook
Sign up using Email and Password
Post as a guest
Required, but never shown
Sign up or log in
StackExchange.ready(function () {
StackExchange.helpers.onClickDraftSave('#login-link');
});
Sign up using Google
Sign up using Facebook
Sign up using Email and Password
Sign up using Google
Sign up using Facebook
Sign up using Email and Password
Post as a guest
Required, but never shown
Required, but never shown
Required, but never shown
Required, but never shown
Required, but never shown
Required, but never shown
Required, but never shown
Required, but never shown
Required, but never shown