| 213 | } |
| 214 | |
| 215 | static int is_id_follow(int cp, int utf8_len) |
| 216 | { |
| 217 | if (utf8_len == 1 && isalnum(cp) || cp == '_' || utf8_len > 1) |
| 218 | return 1; |
| 219 | return 0; |
| 220 | } |
| 221 | |
| 222 | /* Tokenization of Python programming language source text. |
| 223 | Returns 1 when a next valid token is recognized. |