Translate a type comment into a list of tokens.
(s)
| 171 | |
| 172 | |
| 173 | def tokenize(s): |
| 174 | # type: (str) -> List[Token] |
| 175 | """Translate a type comment into a list of tokens.""" |
| 176 | original = s |
| 177 | tokens = [] # type: List[Token] |
| 178 | while True: |
| 179 | if not s: |
| 180 | tokens.append(End()) |
| 181 | return tokens |
| 182 | elif s[0] == ' ': |
| 183 | s = s[1:] |
| 184 | elif s[0] in '()[],*': |
| 185 | tokens.append(Separator(s[0])) |
| 186 | s = s[1:] |
| 187 | elif s[:2] == '->': |
| 188 | tokens.append(Separator('->')) |
| 189 | s = s[2:] |
| 190 | else: |
| 191 | m = re.match(r'[-\w]+(\s*(\.|:)\s*[-/\w]*)*', s) |
| 192 | if not m: |
| 193 | raise ParseError(original) |
| 194 | fullname = m.group(0) |
| 195 | fullname = fullname.replace(' ', '') |
| 196 | if fullname in TYPE_FIXUPS: |
| 197 | fullname = TYPE_FIXUPS[fullname] |
| 198 | # pytz creates classes with the name of the timezone being used: |
| 199 | # https://github.com/stub42/pytz/blob/f55399cddbef67c56db1b83e0939ecc1e276cf42/src/pytz/tzfile.py#L120-L123 |
| 200 | # This causes pyannotates to crash as it's invalid to have a class |
| 201 | # name with a `/` in it (e.g. "pytz.tzfile.America/Los_Angeles") |
| 202 | if fullname.startswith('pytz.tzfile.'): |
| 203 | fullname = 'datetime.tzinfo' |
| 204 | if '-' in fullname or '/' in fullname: |
| 205 | # Not a valid Python name; there are many places that |
| 206 | # generate these, so we just substitute Any rather |
| 207 | # than crashing. |
| 208 | fullname = 'Any' |
| 209 | tokens.append(DottedName(fullname)) |
| 210 | s = s[len(m.group(0)):] |
| 211 | |
| 212 | |
| 213 | def parse_type_comment(comment): |