(readline, encoding)
| 431 | |
| 432 | |
| 433 | def _tokenize(readline, encoding): |
| 434 | lnum = parenlev = continued = 0 |
| 435 | numchars = '0123456789' |
| 436 | contstr, needcont = '', 0 |
| 437 | contline = None |
| 438 | indents = [0] |
| 439 | |
| 440 | if encoding is not None: |
| 441 | if encoding == "utf-8-sig": |
| 442 | # BOM will already have been stripped. |
| 443 | encoding = "utf-8" |
| 444 | yield TokenInfo(ENCODING, encoding, (0, 0), (0, 0), '') |
| 445 | last_line = b'' |
| 446 | line = b'' |
| 447 | while True: # loop over lines in stream |
| 448 | try: |
| 449 | # We capture the value of the line variable here because |
| 450 | # readline uses the empty string '' to signal end of input, |
| 451 | # hence `line` itself will always be overwritten at the end |
| 452 | # of this loop. |
| 453 | last_line = line |
| 454 | line = readline() |
| 455 | except StopIteration: |
| 456 | line = b'' |
| 457 | |
| 458 | if encoding is not None: |
| 459 | line = line.decode(encoding) |
| 460 | lnum += 1 |
| 461 | pos, max = 0, len(line) |
| 462 | |
| 463 | if contstr: # continued string |
| 464 | if not line: |
| 465 | raise TokenError("EOF in multi-line string", strstart) |
| 466 | endmatch = endprog.match(line) |
| 467 | if endmatch: |
| 468 | pos = end = endmatch.end(0) |
| 469 | yield TokenInfo(STRING, contstr + line[:end], |
| 470 | strstart, (lnum, end), contline + line) |
| 471 | contstr, needcont = '', 0 |
| 472 | contline = None |
| 473 | elif needcont and line[-2:] != '\\\n' and line[-3:] != '\\\r\n': |
| 474 | yield TokenInfo(ERRORTOKEN, contstr + line, |
| 475 | strstart, (lnum, len(line)), contline) |
| 476 | contstr = '' |
| 477 | contline = None |
| 478 | continue |
| 479 | else: |
| 480 | contstr = contstr + line |
| 481 | contline = contline + line |
| 482 | continue |
| 483 | |
| 484 | elif parenlev == 0 and not continued: # new statement |
| 485 | if not line: break |
| 486 | column = 0 |
| 487 | while pos < max: # measure leading whitespace |
| 488 | if line[pos] == ' ': |
| 489 | column += 1 |
| 490 | elif line[pos] == '\t': |
no test coverage detected