Test roundtrip for `untokenize`. `f` is an open file or a string. The source code in f is tokenized to both 5- and 2-tuples. Both sequences are converted back to source code via tokenize.untokenize(), and the latter tokenized again to 2-tuples. The test fails
(self, f)
| 1996 | class TestRoundtrip(TestCase): |
| 1997 | |
| 1998 | def check_roundtrip(self, f): |
| 1999 | """ |
| 2000 | Test roundtrip for `untokenize`. `f` is an open file or a string. |
| 2001 | The source code in f is tokenized to both 5- and 2-tuples. |
| 2002 | Both sequences are converted back to source code via |
| 2003 | tokenize.untokenize(), and the latter tokenized again to 2-tuples. |
| 2004 | The test fails if the 3 pair tokenizations do not match. |
| 2005 | |
| 2006 | If the source code can be untokenized unambiguously, the |
| 2007 | untokenized code must match the original code exactly. |
| 2008 | |
| 2009 | When untokenize bugs are fixed, untokenize with 5-tuples should |
| 2010 | reproduce code that does not contain a backslash continuation |
| 2011 | following spaces. A proper test should test this. |
| 2012 | """ |
| 2013 | # Get source code and original tokenizations |
| 2014 | if isinstance(f, str): |
| 2015 | code = f.encode('utf-8') |
| 2016 | else: |
| 2017 | code = f.read() |
| 2018 | readline = iter(code.splitlines(keepends=True)).__next__ |
| 2019 | tokens5 = list(tokenize.tokenize(readline)) |
| 2020 | tokens2 = [tok[:2] for tok in tokens5] |
| 2021 | # Reproduce tokens2 from pairs |
| 2022 | bytes_from2 = tokenize.untokenize(tokens2) |
| 2023 | readline2 = iter(bytes_from2.splitlines(keepends=True)).__next__ |
| 2024 | tokens2_from2 = [tok[:2] for tok in tokenize.tokenize(readline2)] |
| 2025 | self.assertEqual(tokens2_from2, tokens2) |
| 2026 | # Reproduce tokens2 from 5-tuples |
| 2027 | bytes_from5 = tokenize.untokenize(tokens5) |
| 2028 | readline5 = iter(bytes_from5.splitlines(keepends=True)).__next__ |
| 2029 | tokens2_from5 = [tok[:2] for tok in tokenize.tokenize(readline5)] |
| 2030 | self.assertEqual(tokens2_from5, tokens2) |
| 2031 | |
| 2032 | if not contains_ambiguous_backslash(code): |
| 2033 | # The BOM does not produce a token so there is no way to preserve it. |
| 2034 | code_without_bom = code.removeprefix(b'\xef\xbb\xbf') |
| 2035 | readline = iter(code_without_bom.splitlines(keepends=True)).__next__ |
| 2036 | untokenized_code = tokenize.untokenize(tokenize.tokenize(readline)) |
| 2037 | self.assertEqual(code_without_bom, untokenized_code) |
| 2038 | |
| 2039 | def check_line_extraction(self, f): |
| 2040 | if isinstance(f, str): |
no test coverage detected