MCPcopy Create free account
hub / github.com/colbymchenry/codegraph / tokenizeSource

Function tokenizeSource

src/extraction/syntax-tokens.ts:422–448  ·  view source on GitHub ↗
(
  source: string,
  language: Language
)

Source from the content-addressed store, hash-verified

420 * having one.
421 */
422export async function tokenizeSource(
423 source: string,
424 language: Language
425): Promise<TokenizeResult | null> {
426 const regions = syntaxRegionsFor(source, language);
427 if (regions === null) {
428 const spans = await tokenizeRegion(source, language, 0);
429 return spans ? { spans, grammars: [language] } : null;
430 }
431 if (regions.length === 0) return null;
432
433 const spans: SyntaxSpan[] = [];
434 const grammars = new Set<string>();
435 for (const region of regions) {
436 const part = await tokenizeRegion(
437 source.slice(region.start, region.end),
438 region.language,
439 region.start
440 );
441 if (!part) continue;
442 grammars.add(region.language);
443 spans.push(...part);
444 }
445 if (spans.length === 0) return null;
446 spans.sort((a, b) => a.start - b.start);
447 return { spans, grammars: [...grammars] };
448}
449
450async function tokenizeRegion(
451 source: string,

Callers 2

tokenize-worker.tsFile · 0.90
tokenizeBoundedFunction · 0.90

Calls 4

syntaxRegionsForFunction · 0.85
tokenizeRegionFunction · 0.85
pushMethod · 0.80
addMethod · 0.45

Tested by

no test coverage detected