MCPcopy Create free account
hub / github.com/dateutil/dateutil / _timelex

Class _timelex

src/dateutil/parser/_parser.py:54–210  ·  view source on GitHub ↗

Source from the content-addressed store, hash-verified

52# making public and/or figuring out if there is something we can
53# take off their plate.
54class _timelex:
55 # Fractional seconds are sometimes split by a comma
56 _split_decimal = re.compile("([.,])")
57
58 def __init__(self, instream):
59 if isinstance(instream, (bytes, bytearray)):
60 instream = instream.decode()
61
62 if isinstance(instream, str):
63 instream = StringIO(instream)
64 elif getattr(instream, 'read', None) is None:
65 raise TypeError('Parser must be a string or character stream, not '
66 '{itype}'.format(itype=instream.__class__.__name__))
67
68 self.instream = instream
69 self.charstack = []
70 self.tokenstack = []
71 self.eof = False
72
73 def get_token(self):
74 """
75 This function breaks the time string into lexical units (tokens), which
76 can be parsed by the parser. Lexical units are demarcated by changes in
77 the character set, so any continuous string of letters is considered
78 one unit, any continuous string of numbers is considered one unit.
79
80 The main complication arises from the fact that dots ('.') can be used
81 both as separators (e.g. "Sep.20.2009") or decimal points (e.g.
82 "4:30:21.447"). As such, it is necessary to read the full context of
83 any dot-separated strings before breaking it into tokens; as such, this
84 function maintains a "token stack", for when the ambiguous context
85 demands that multiple tokens be parsed at once.
86 """
87 if self.tokenstack:
88 return self.tokenstack.pop(0)
89
90 seenletters = False
91 token = None
92 state = None
93
94 while not self.eof:
95 # We only realize that we've reached the end of a token when we
96 # find a character that's not part of the current token - since
97 # that character may be part of the next token, it's stored in the
98 # charstack.
99 if self.charstack:
100 nextchar = self.charstack.pop(0)
101 else:
102 nextchar = self.instream.read(1)
103 while nextchar == '\x00':
104 nextchar = self.instream.read(1)
105
106 if not nextchar:
107 self.eof = True
108 break
109 elif not state:
110 # First character of the token - determines if we're starting
111 # to parse a word, a number or something else.

Callers 2

Calls

no outgoing calls

Tested by 2