()
| 622 | return _tokenize(readline, None) |
| 623 | |
| 624 | def main(): |
| 625 | import argparse |
| 626 | |
| 627 | # Helper error handling routines |
| 628 | def perror(message): |
| 629 | sys.stderr.write(message) |
| 630 | sys.stderr.write('\n') |
| 631 | |
| 632 | def error(message, filename=None, location=None): |
| 633 | if location: |
| 634 | args = (filename,) + location + (message,) |
| 635 | perror("%s:%d:%d: error: %s" % args) |
| 636 | elif filename: |
| 637 | perror("%s: error: %s" % (filename, message)) |
| 638 | else: |
| 639 | perror("error: %s" % message) |
| 640 | sys.exit(1) |
| 641 | |
| 642 | # Parse the arguments and options |
| 643 | parser = argparse.ArgumentParser(prog='python -m tokenize') |
| 644 | parser.add_argument(dest='filename', nargs='?', |
| 645 | metavar='filename.py', |
| 646 | help='the file to tokenize; defaults to stdin') |
| 647 | parser.add_argument('-e', '--exact', dest='exact', action='store_true', |
| 648 | help='display token names using the exact type') |
| 649 | args = parser.parse_args() |
| 650 | |
| 651 | try: |
| 652 | # Tokenize the input |
| 653 | if args.filename: |
| 654 | filename = args.filename |
| 655 | with _builtin_open(filename, 'rb') as f: |
| 656 | tokens = list(tokenize(f.readline)) |
| 657 | else: |
| 658 | filename = "<stdin>" |
| 659 | tokens = _tokenize(sys.stdin.readline, None) |
| 660 | |
| 661 | # Output the tokenization |
| 662 | for token in tokens: |
| 663 | token_type = token.type |
| 664 | if args.exact: |
| 665 | token_type = token.exact_type |
| 666 | token_range = "%d,%d-%d,%d:" % (token.start + token.end) |
| 667 | print("%-20s%-15s%-15r" % |
| 668 | (token_range, tok_name[token_type], token.string)) |
| 669 | except IndentationError as err: |
| 670 | line, column = err.args[1][1:3] |
| 671 | error(err.args[0], filename, (line, column)) |
| 672 | except TokenError as err: |
| 673 | line, column = err.args[1] |
| 674 | error(err.args[0], filename, (line, column)) |
| 675 | except SyntaxError as err: |
| 676 | error(err, filename) |
| 677 | except OSError as err: |
| 678 | error(err) |
| 679 | except KeyboardInterrupt: |
| 680 | print("interrupted\n") |
| 681 | except Exception as err: |
no test coverage detected