(lines, pos)
| 308 | |
| 309 | |
| 310 | def TokenizeLines(lines, pos): |
| 311 | while True: |
| 312 | found = FindFirst(lines, TOKEN_TABLE, pos) |
| 313 | if not found: |
| 314 | yield MakeToken(lines, pos, Eof(), 'code') |
| 315 | return |
| 316 | |
| 317 | if found.start == pos: |
| 318 | prev_token = None |
| 319 | prev_token_rstripped = None |
| 320 | else: |
| 321 | prev_token = MakeToken(lines, pos, found.start, 'code') |
| 322 | prev_token_rstripped = RStripNewLineFromToken(prev_token) |
| 323 | |
| 324 | if found.token_type == '$var': |
| 325 | if prev_token_rstripped: |
| 326 | yield prev_token_rstripped |
| 327 | yield found |
| 328 | id_token = ParseToken(lines, found.end, ID_REGEX, 'id') |
| 329 | yield id_token |
| 330 | pos = Skip(lines, id_token.end, OPTIONAL_WHITE_SPACES_REGEX) |
| 331 | |
| 332 | eq_token = ParseToken(lines, pos, EQ_REGEX, '=') |
| 333 | yield eq_token |
| 334 | pos = Skip(lines, eq_token.end, r'\s*') |
| 335 | |
| 336 | if SubString(lines, pos, pos + 2) != '[[': |
| 337 | exp_token = ParseToken(lines, pos, REST_OF_LINE_REGEX, 'exp') |
| 338 | yield exp_token |
| 339 | pos = Cursor(exp_token.end.line + 1, 0) |
| 340 | elif found.token_type == '$for': |
| 341 | if prev_token_rstripped: |
| 342 | yield prev_token_rstripped |
| 343 | yield found |
| 344 | id_token = ParseToken(lines, found.end, ID_REGEX, 'id') |
| 345 | yield id_token |
| 346 | pos = Skip(lines, id_token.end, WHITE_SPACE_REGEX) |
| 347 | elif found.token_type == '$range': |
| 348 | if prev_token_rstripped: |
| 349 | yield prev_token_rstripped |
| 350 | yield found |
| 351 | id_token = ParseToken(lines, found.end, ID_REGEX, 'id') |
| 352 | yield id_token |
| 353 | pos = Skip(lines, id_token.end, OPTIONAL_WHITE_SPACES_REGEX) |
| 354 | |
| 355 | dots_pos = SkipUntil(lines, pos, DOT_DOT_REGEX, '..') |
| 356 | yield MakeToken(lines, pos, dots_pos, 'exp') |
| 357 | yield MakeToken(lines, dots_pos, dots_pos + 2, '..') |
| 358 | pos = dots_pos + 2 |
| 359 | new_pos = Cursor(pos.line + 1, 0) |
| 360 | yield MakeToken(lines, pos, new_pos, 'exp') |
| 361 | pos = new_pos |
| 362 | elif found.token_type == '$': |
| 363 | if prev_token: |
| 364 | yield prev_token |
| 365 | yield found |
| 366 | exp_token = ParseExpTokenInParens(lines, found.end) |
| 367 | yield exp_token |
no test coverage detected