( source: string, language: Language )
| 420 | * having one. |
| 421 | */ |
| 422 | export async function tokenizeSource( |
| 423 | source: string, |
| 424 | language: Language |
| 425 | ): Promise<TokenizeResult | null> { |
| 426 | const regions = syntaxRegionsFor(source, language); |
| 427 | if (regions === null) { |
| 428 | const spans = await tokenizeRegion(source, language, 0); |
| 429 | return spans ? { spans, grammars: [language] } : null; |
| 430 | } |
| 431 | if (regions.length === 0) return null; |
| 432 | |
| 433 | const spans: SyntaxSpan[] = []; |
| 434 | const grammars = new Set<string>(); |
| 435 | for (const region of regions) { |
| 436 | const part = await tokenizeRegion( |
| 437 | source.slice(region.start, region.end), |
| 438 | region.language, |
| 439 | region.start |
| 440 | ); |
| 441 | if (!part) continue; |
| 442 | grammars.add(region.language); |
| 443 | spans.push(...part); |
| 444 | } |
| 445 | if (spans.length === 0) return null; |
| 446 | spans.sort((a, b) => a.start - b.start); |
| 447 | return { spans, grammars: [...grammars] }; |
| 448 | } |
| 449 | |
| 450 | async function tokenizeRegion( |
| 451 | source: string, |
no test coverage detected