Returns length of matched token (ws inclusive) or 0 when in error.
| 195 | |
| 196 | // Returns length of matched token (ws inclusive) or 0 when in error. |
| 197 | unsigned get_token(char const *text, unsigned start) |
| 198 | { |
| 199 | if (!*text) return 0; |
| 200 | regex_t *re = compiled_token_RE; |
| 201 | size_t nmatch = NMATCH; |
| 202 | regmatch_t pmatch[NMATCH]; |
| 203 | |
| 204 | if (regexec(re, text, nmatch, pmatch, REG_NOTEOL) == REG_NOMATCH) { |
| 205 | // Warn about the failed match: |
| 206 | fprintf(stderr, "(W) [%u:%u] not a valid token; skipped.\n", |
| 207 | linenrs[start],columns[start]); |
| 208 | // Cannot recover; no more input. |
| 209 | return 0; |
| 210 | } |
| 211 | // Here: a valid token recognized. |
| 212 | |
| 213 | // Total length matched (leading white-space inclusive): |
| 214 | assert(pmatch[0].rm_so == 0); |
| 215 | unsigned skiplen = pmatch[0].rm_eo; |
| 216 | |
| 217 | unsigned i; |
| 218 | for (i = 0; i < NMATCH; i++) { |
| 219 | const char *key = classes[i]; |
| 220 | if (!key) continue; |
| 221 | int offset = pmatch[i].rm_so; |
| 222 | unsigned len = pmatch[i].rm_eo - offset; |
| 223 | char const *p = text + offset; |
| 224 | if (!len) continue; |
| 225 | |
| 226 | // Only CSV-style output for now; no escaping of " yet. |
| 227 | // FIXME: Must check identifier for reserved words. |
| 228 | fprintf(stdout, "%u,%u,%s,", linenrs[start+offset],columns[start+offset],key); |
| 229 | //fprintf(stdout, "%2d: ", i); |
| 230 | fwrite(p, 1, len, stdout); |
| 231 | fputc('\n', stdout); |
| 232 | } |
| 233 | return skiplen; |
| 234 | } |
| 235 | |
| 236 | /* Deal with DOS (\r \n) and classic Mac OS (\r) line endings. |
| 237 | Keeps track of physical coordinates and absolute location for each character. |