Method check_line_extraction

Lib/test/test_tokenize.py:2039–2048

(self, f)

            self.assertEqual(code_without_bom, untokenized_code)

    def check_line_extraction(self, f):
        if isinstance(f, str):
            code = f.encode('utf-8')
        else:
            code = f.read()
        readline = iter(code.splitlines(keepends=True)).__next__
        for tok in tokenize.tokenize(readline):
            if tok.type in  {tokenize.ENCODING, tokenize.ENDMARKER}:
                continue
            self.assertEqual(tok.string, tok.line[tok.start[1]: tok.end[1]])

    def test_roundtrip(self):
        # There are some standard formatting practices that are easy to get right.