MCPcopy Create free account
hub / github.com/Roy3838/Observer / generateWithLocalModel

Method generateWithLocalModel

app/src/utils/ModelManager.ts:597–637  ·  view source on GitHub ↗

* Generate a response using a local model. * Routes to the appropriate underlying manager based on the server sentinel.

(
    server: string,
    messages: LocalLlmMessage[],
    onToken?: (token: string) => void,
    onReasoningToken?: (token: string) => void,
  )

Source from the content-addressed store, hash-verified

595 * Routes to the appropriate underlying manager based on the server sentinel.
596 */
597 public async generateWithLocalModel(
598 server: string,
599 messages: LocalLlmMessage[],
600 onToken?: (token: string) => void,
601 onReasoningToken?: (token: string) => void,
602 ): Promise<string> {
603 if (server === ModelManager.SKIP_MODEL) {
604 return '';
605 }
606
607 if (server === ModelManager.BROWSER_LOCAL) {
608 const manager = GemmaModelManager.getInstance();
609 if (!manager.isReady()) {
610 throw new Error('Transformers.js model not loaded');
611 }
612 const enableThinking = manager.getState().loadSettings?.enableThinking ?? false;
613 return manager.generate(messages, onToken, enableThinking, onReasoningToken);
614 }
615
616 if (server === ModelManager.LLAMA_CPP_LOCAL && isTauri()) {
617 const manager = NativeLlmManager.getInstance();
618 if (!manager.isReady()) {
619 throw new Error('llama.cpp model not loaded');
620 }
621 const enableThinking = manager.getState().enableThinking;
622 if (!enableThinking || !onReasoningToken) {
623 return manager.generate(messages, onToken);
624 }
625 // Route thinking tokens to onReasoningToken, answer tokens to onToken.
626 // Accumulate answer tokens — manager.generate returns the raw backend string which
627 // still contains the thinking delimiters, so we can't use it directly.
628 let answer = '';
629 const collectAndForward = (t: string) => { answer += t; onToken?.(t); };
630 const router = makeLlamaCppThinkingRouter(collectAndForward, onReasoningToken);
631 await manager.generate(messages, router);
632 router.flush();
633 return answer;
634 }
635
636 throw new Error(`Cannot generate: unsupported server type "${server}"`);
637 }
638
639 // ===========================================================================
640 // Per-Model Inference Params (remote models only — local models use their own settings)

Callers 2

sendPromptMethod · 0.95
sendMessagesMethod · 0.95

Calls 7

isTauriFunction · 0.90
flushMethod · 0.80
isReadyMethod · 0.65
getStateMethod · 0.65
generateMethod · 0.65
getInstanceMethod · 0.45

Tested by

no test coverage detected