| 52 | # making public and/or figuring out if there is something we can |
| 53 | # take off their plate. |
| 54 | class _timelex: |
| 55 | # Fractional seconds are sometimes split by a comma |
| 56 | _split_decimal = re.compile("([.,])") |
| 57 | |
| 58 | def __init__(self, instream): |
| 59 | if isinstance(instream, (bytes, bytearray)): |
| 60 | instream = instream.decode() |
| 61 | |
| 62 | if isinstance(instream, str): |
| 63 | instream = StringIO(instream) |
| 64 | elif getattr(instream, 'read', None) is None: |
| 65 | raise TypeError('Parser must be a string or character stream, not ' |
| 66 | '{itype}'.format(itype=instream.__class__.__name__)) |
| 67 | |
| 68 | self.instream = instream |
| 69 | self.charstack = [] |
| 70 | self.tokenstack = [] |
| 71 | self.eof = False |
| 72 | |
| 73 | def get_token(self): |
| 74 | """ |
| 75 | This function breaks the time string into lexical units (tokens), which |
| 76 | can be parsed by the parser. Lexical units are demarcated by changes in |
| 77 | the character set, so any continuous string of letters is considered |
| 78 | one unit, any continuous string of numbers is considered one unit. |
| 79 | |
| 80 | The main complication arises from the fact that dots ('.') can be used |
| 81 | both as separators (e.g. "Sep.20.2009") or decimal points (e.g. |
| 82 | "4:30:21.447"). As such, it is necessary to read the full context of |
| 83 | any dot-separated strings before breaking it into tokens; as such, this |
| 84 | function maintains a "token stack", for when the ambiguous context |
| 85 | demands that multiple tokens be parsed at once. |
| 86 | """ |
| 87 | if self.tokenstack: |
| 88 | return self.tokenstack.pop(0) |
| 89 | |
| 90 | seenletters = False |
| 91 | token = None |
| 92 | state = None |
| 93 | |
| 94 | while not self.eof: |
| 95 | # We only realize that we've reached the end of a token when we |
| 96 | # find a character that's not part of the current token - since |
| 97 | # that character may be part of the next token, it's stored in the |
| 98 | # charstack. |
| 99 | if self.charstack: |
| 100 | nextchar = self.charstack.pop(0) |
| 101 | else: |
| 102 | nextchar = self.instream.read(1) |
| 103 | while nextchar == '\x00': |
| 104 | nextchar = self.instream.read(1) |
| 105 | |
| 106 | if not nextchar: |
| 107 | self.eof = True |
| 108 | break |
| 109 | elif not state: |
| 110 | # First character of the token - determines if we're starting |
| 111 | # to parse a word, a number or something else. |
no outgoing calls