Convert latex source into sequence of single-token substrings.
(tex)
| 80 | return (Codec().encode,Codec().decode,StreamReader,StreamWriter) |
| 81 | |
| 82 | def _tokenize(tex): |
| 83 | """Convert latex source into sequence of single-token substrings.""" |
| 84 | start = 0 |
| 85 | try: |
| 86 | # skip quickly across boring stuff |
| 87 | pos = _stoppers.finditer(tex).next().span()[0] |
| 88 | except StopIteration: |
| 89 | yield tex |
| 90 | return |
| 91 | |
| 92 | while 1: |
| 93 | if pos > start: |
| 94 | yield tex[start:pos] |
| 95 | if tex[start] == '\\' and not (tex[pos-1].isdigit() and tex[start+1].isalpha()): |
| 96 | while pos < len(tex) and tex[pos].isspace(): # skip blanks after csname |
| 97 | pos += 1 |
| 98 | |
| 99 | while pos < len(tex) and tex[pos] in _ignore: |
| 100 | pos += 1 # flush control characters |
| 101 | if pos >= len(tex): |
| 102 | return |
| 103 | start = pos |
| 104 | if tex[pos:pos+2] in {'$$':None, '/~':None}: # protect ~ in urls |
| 105 | pos += 2 |
| 106 | elif tex[pos].isdigit(): |
| 107 | while pos < len(tex) and tex[pos].isdigit(): |
| 108 | pos += 1 |
| 109 | elif tex[pos] == '-': |
| 110 | while pos < len(tex) and tex[pos] == '-': |
| 111 | pos += 1 |
| 112 | elif tex[pos] != '\\' or pos == len(tex) - 1: |
| 113 | pos += 1 |
| 114 | elif not tex[pos+1].isalpha(): |
| 115 | pos += 2 |
| 116 | else: |
| 117 | pos += 1 |
| 118 | while pos < len(tex) and tex[pos].isalpha(): |
| 119 | pos += 1 |
| 120 | if tex[start:pos] == '\\char' or tex[start:pos] == '\\accent': |
| 121 | while pos < len(tex) and tex[pos].isdigit(): |
| 122 | pos += 1 |
| 123 | |
| 124 | class _unlatex: |
| 125 | """Convert tokenized tex into sequence of unicode strings. Helper for decode().""" |
no test coverage detected