MCPcopy Create free account
hub / github.com/DeusData/codebase-memory-mcp / tokenize

Function tokenize

graph-ui/src/browser-ai/browser-ai.worker.ts:135–146  ·  view source on GitHub ↗
(messages: readonly BrowserChatMessage[])

Source from the content-addressed store, hash-verified

133} as const;
134
135function tokenize(messages: readonly BrowserChatMessage[]): { input_ids: Tensor; attention_mask: Tensor } {
136 if (!model || !tokenizer) throw new Error('Load a browser model before sending a message.');
137 if (!messages.length || messages.some(message => !['system', 'user', 'assistant'].includes(message.role) || typeof message.content !== 'string')) {
138 throw new Error('The conversation contains an invalid message.');
139 }
140 // This exact tensor is counted and sent to generate: no character slicing or pipeline truncation.
141 const templateOptions = {
142 tokenize: true, return_dict: true, add_generation_prompt: true,
143 enable_thinking: false, truncation: false, padding: false,
144 } as const;
145 return tokenizer.apply_chat_template([...messages], templateOptions) as { input_ids: Tensor; attention_mask: Tensor };
146}
147
148async function generate(id: number, messages: readonly BrowserChatMessage[], options: BrowserChatOptions): Promise<{ output: string; stopReason: BrowserStopReason }> {
149 if (options.generationProfile !== undefined && options.generationProfile !== 'automatic-explanation')

Callers 2

generateFunction · 0.70

Calls

no outgoing calls

Tested by

no test coverage detected