MCPcopy Create free account
hub / github.com/IBM/Project_CodeNet / get

Function get

tools/tokenizer/antlr4tojson.c:207–304  ·  view source on GitHub ↗

Returns length of matched token (ws inclusive) or 0 when in error.

Source from the content-addressed store, hash-verified

205
206// Returns length of matched token (ws inclusive) or 0 when in error.
207static unsigned get(char const *text)
208{
209 if (!*text) return 0;
210 regex_t *re = compiled_token_RE;
211 size_t nmatch = NMATCH;
212 regmatch_t pmatch[NMATCH];
213
214 if (regexec(re, text, nmatch, pmatch, REG_NOTEOL) == REG_NOMATCH) {
215 // Warn about the failed match:
216 fprintf(stderr, "(W) [%s:%u] not a valid token; skipped.\n",
217 filename, linenr);
218 // Cannot recover; no more input.
219 return 0;
220 }
221 // Here: a valid token recognized.
222
223 // Total length matched (leading white-space inclusive):
224 assert(pmatch[INPUT].rm_so == 0);
225 unsigned skiplen = pmatch[INPUT].rm_eo;
226
227 int need_comma = 0;
228 unsigned i;
229 if (mode == JSON || mode == JSONL)
230 fputc('{', stdout);
231 int channel_absent = 1;
232 for (i = 0; i < NMATCH; i++) {
233
234 if (i == LINE && channel_absent && mode == CSV) {
235 // no channel; insert default 0:
236 fputs(",0", stdout);
237 }
238
239 const char *key = fields[i];
240 if (!key) continue;
241 int offset = pmatch[i].rm_so;
242 unsigned len = pmatch[i].rm_eo - offset;
243 char const *p = text + offset;
244 if (!len) continue;
245 if (need_comma)
246 fputc(',', stdout);
247 // For CSV output don't show key.
248 if (mode == JSON || mode == JSONL)
249 fprintf(stdout, "\"%s\":", key);
250 // values can be: integer, identifier, single-quoted string
251 // use JSON values: integer, "identifier", "'...'"
252
253 // Special treatment for text and some class key values;
254 // must escape some chars to comply with JSON string!
255 switch (i) {
256 case CLASS_IDENT:
257 // CSV output does not need the quoting.
258 if (mode == JSON || mode == JSONL)
259 fputc('"', stdout);
260 // Undo the capitalization?
261 fputc(tolower(*p), stdout);
262 fwrite(p+1, 1, len-1, stdout);
263 if (mode == JSON || mode == JSONL)
264 fputc('"', stdout);

Callers 1

mainFunction · 0.70

Calls 2

JSON_escapeFunction · 0.70
CSV_escapeFunction · 0.70

Tested by

no test coverage detected