()
| 30 | |
| 31 | # Token generator: |
| 32 | def token(): |
| 33 | global _token, _kind, _linenr, _column, _pos |
| 34 | |
| 35 | # C_tokenize returns 0 upon end-of-file. |
| 36 | while int(libtoken.C_tokenize(byref(_token), byref(_kind), byref(_linenr), |
| 37 | byref(_column), byref(_pos))): |
| 38 | # Turn ctypes into real Python values: |
| 39 | lin = _linenr.value |
| 40 | col = _column.value |
| 41 | pos = _pos.value # not used for now |
| 42 | clas = _kind.value.decode() |
| 43 | text = _token.value.decode() |
| 44 | yield (lin,col,clas,text) |
| 45 | |
| 46 | if len(sys.argv) == 1: |
| 47 | for tok in token(): |