Simplified tokenizer: comments and preproc directives removed, identifiers are a token, others are single char tokens. */
| 29 | /* Simplified tokenizer: comments and preproc directives removed, |
| 30 | identifiers are a token, others are single char tokens. */ |
| 31 | static struct token *tokenize(const void *ctx, const char *code) |
| 32 | { |
| 33 | unsigned int i, len, tok_start = -1; |
| 34 | bool start_of_line = true; |
| 35 | struct token *toks = tal_arr(ctx, struct token, 0); |
| 36 | |
| 37 | for (i = 0; code[i]; i += len) { |
| 38 | if (code[i] == '#' && start_of_line) { |
| 39 | /* Preprocessor line. */ |
| 40 | len = to_eol(code + i); |
| 41 | } else if (code[i] == '/' && code[i+1] == '/') { |
| 42 | /* One line comment. */ |
| 43 | len = to_eol(code + i); |
| 44 | if (tok_start != -1U) { |
| 45 | add_token(&toks, code+tok_start, i - tok_start); |
| 46 | tok_start = -1U; |
| 47 | } |
| 48 | } else if (code[i] == '/' && code[i+1] == '*') { |
| 49 | /* Multi-line comment. */ |
| 50 | const char *end = strstr(code+i+2, "*/"); |
| 51 | len = (end + 2) - (code + i); |
| 52 | if (!end) |
| 53 | len = strlen(code + i); |
| 54 | if (tok_start != -1U) { |
| 55 | add_token(&toks, code+tok_start, i - tok_start); |
| 56 | tok_start = -1U; |
| 57 | } |
| 58 | } else if (cisalnum(code[i]) || code[i] == '_') { |
| 59 | /* Identifier or part thereof */ |
| 60 | if (tok_start == -1U) |
| 61 | tok_start = i; |
| 62 | len = 1; |
| 63 | } else if (!cisspace(code[i])) { |
| 64 | /* Punctuation: treat as single char token. */ |
| 65 | if (tok_start != -1U) { |
| 66 | add_token(&toks, code+tok_start, i - tok_start); |
| 67 | tok_start = -1U; |
| 68 | } |
| 69 | add_token(&toks, code+i, 1); |
| 70 | len = 1; |
| 71 | } else { |
| 72 | /* Whitespace. */ |
| 73 | if (tok_start != -1U) { |
| 74 | add_token(&toks, code+tok_start, i - tok_start); |
| 75 | tok_start = -1U; |
| 76 | } |
| 77 | len = 1; |
| 78 | } |
| 79 | if (code[i] == '\n') |
| 80 | start_of_line = true; |
| 81 | else if (!cisspace(code[i])) |
| 82 | start_of_line = false; |
| 83 | } |
| 84 | |
| 85 | /* Add terminating NULL. */ |
| 86 | tal_resizez(&toks, tal_count(toks) + 1); |
| 87 | return toks; |
| 88 | } |
no test coverage detected