This function breaks the time string into lexical units (tokens), which can be parsed by the parser. Lexical units are demarcated by changes in the character set, so any continuous string of letters is considered one unit, any continuous string of numbers is consider
(self)
| 75 | self.eof = False |
| 76 | |
| 77 | def get_token(self): |
| 78 | """ |
| 79 | This function breaks the time string into lexical units (tokens), which |
| 80 | can be parsed by the parser. Lexical units are demarcated by changes in |
| 81 | the character set, so any continuous string of letters is considered |
| 82 | one unit, any continuous string of numbers is considered one unit. |
| 83 | |
| 84 | The main complication arises from the fact that dots ('.') can be used |
| 85 | both as separators (e.g. "Sep.20.2009") or decimal points (e.g. |
| 86 | "4:30:21.447"). As such, it is necessary to read the full context of |
| 87 | any dot-separated strings before breaking it into tokens; as such, this |
| 88 | function maintains a "token stack", for when the ambiguous context |
| 89 | demands that multiple tokens be parsed at once. |
| 90 | """ |
| 91 | if self.tokenstack: |
| 92 | return self.tokenstack.pop(0) |
| 93 | |
| 94 | seenletters = False |
| 95 | token = None |
| 96 | state = None |
| 97 | |
| 98 | while not self.eof: |
| 99 | # We only realize that we've reached the end of a token when we |
| 100 | # find a character that's not part of the current token - since |
| 101 | # that character may be part of the next token, it's stored in the |
| 102 | # charstack. |
| 103 | if self.charstack: |
| 104 | nextchar = self.charstack.pop(0) |
| 105 | else: |
| 106 | nextchar = self.instream.read(1) |
| 107 | while nextchar == '\x00': |
| 108 | nextchar = self.instream.read(1) |
| 109 | |
| 110 | if not nextchar: |
| 111 | self.eof = True |
| 112 | break |
| 113 | elif not state: |
| 114 | # First character of the token - determines if we're starting |
| 115 | # to parse a word, a number or something else. |
| 116 | token = nextchar |
| 117 | if self.isword(nextchar): |
| 118 | state = 'a' |
| 119 | elif self.isnum(nextchar): |
| 120 | state = '0' |
| 121 | elif self.isspace(nextchar): |
| 122 | token = ' ' |
| 123 | break # emit token |
| 124 | else: |
| 125 | break # emit token |
| 126 | elif state == 'a': |
| 127 | # If we've already started reading a word, we keep reading |
| 128 | # letters until we find something that's not part of a word. |
| 129 | seenletters = True |
| 130 | if self.isword(nextchar): |
| 131 | token += nextchar |
| 132 | elif nextchar == '.': |
| 133 | token += nextchar |
| 134 | state = 'a.' |