Function codeTokens

src/codegraph/helper-extract.ts:62–64

(normalized: string)

/** Code-aware tokens: identifiers/keywords AND operators/punctuation (structure matters). PURE. */
export function codeTokens(normalized: string): string[] {
  return normalized.match(/[A-Za-z_$][\w$]*|[+\-*/%<>=!&|^~?:;,.(){}\[\]]/g) ?? [];
}

function round(x: number): number { return Math.round(x * 100) / 100; }