MCPcopy Create free account
hub / github.com/colbymchenry/codegraph / tokenizeBounded

Function tokenizeBounded

src/ui-server/highlight/bounded-tokenize.ts:81–107  ·  view source on GitHub ↗
(
  text: string,
  language: Language,
  deadlineMs = TOKENIZE_DEADLINE_MS
)

Source from the content-addressed store, hash-verified

79
80/** Classify `text` in the highlight worker, giving up after {@link TOKENIZE_DEADLINE_MS}. */
81export async function tokenizeBounded(
82 text: string,
83 language: Language,
84 deadlineMs = TOKENIZE_DEADLINE_MS
85): Promise<BoundedTokenize> {
86 const file = workerFile();
87 // No compiled worker (a source checkout that was never built): classify
88 // in-process, as before.
89 if (!file) return { result: await tokenizeSource(text, language) };
90 let w: Worker;
91 try {
92 w = ensureWorker(file);
93 } catch {
94 return { result: await tokenizeSource(text, language) };
95 }
96 const id = nextId++;
97 return new Promise<BoundedTokenize>((resolve) => {
98 const timer = setTimeout(() => {
99 // Everything queued behind the slice that hung waits on the same
100 // worker: end it, serve them all plain, and start over next time.
101 if (worker === w) retire({ timedOut: true });
102 }, deadlineMs);
103 timer.unref();
104 pending.set(id, { resolve, timer });
105 w.postMessage({ id, text, language });
106 });
107}
108
109/** End the highlight worker — for tests, and for a server that is shutting down. */
110export function stopHighlightWorker(): void {

Callers 2

highlightUncachedFunction · 0.90

Calls 6

tokenizeSourceFunction · 0.90
workerFileFunction · 0.85
ensureWorkerFunction · 0.85
retireFunction · 0.70
postMessageMethod · 0.65
setMethod · 0.45

Tested by

no test coverage detected