MCPcopy Index your code
hub / github.com/abetlen/llama-cpp-python / CompletionService

Class CompletionService

examples/server/server.py:15463–15734  ·  view source on GitHub ↗

Source from the content-addressed store, hash-verified

15461
15462
15463class CompletionService:
15464 def __init__(self, scheduler: CompletionScheduler) -> None:
15465 self.scheduler = scheduler
15466 self.formatter = scheduler.formatter
15467 self.condition = threading.Condition()
15468 self.commands: Deque[Callable[[], None]] = deque()
15469 self.closed = False
15470 self.worker = threading.Thread(
15471 target=self.run_loop,
15472 name="completion-service",
15473 daemon=True,
15474 )
15475 self.worker.start()
15476
15477 def close(self) -> None:
15478 with self.condition:
15479 self.closed = True
15480 self.condition.notify_all()
15481 self.worker.join()
15482 self.scheduler.close()
15483
15484 def enqueue(self, command: Callable[[], None]) -> None:
15485 with self.condition:
15486 if self.closed:
15487 raise RuntimeError("completion service closed")
15488 self.commands.append(command)
15489 self.condition.notify_all()
15490
15491 def call_on_scheduler(self, callback: Callable[[], Any]) -> Any:
15492 result_box: Dict[str, Any] = {}
15493 error_box: Dict[str, BaseException] = {}
15494 done = threading.Event()
15495
15496 def run_callback() -> None:
15497 try:
15498 result_box["result"] = callback()
15499 except BaseException as exc: # noqa: BLE001
15500 error_box["error"] = exc
15501 finally:
15502 done.set()
15503
15504 self.enqueue(run_callback)
15505 done.wait()
15506 if "error" in error_box:
15507 raise error_box["error"]
15508 return result_box.get("result")
15509
15510 def call_on_idle_scheduler(self, callback: Callable[[], Any]) -> Any:
15511 result_box: Dict[str, Any] = {}
15512 error_box: Dict[str, BaseException] = {}
15513 done = threading.Event()
15514
15515 def run_callback() -> None:
15516 if not self.scheduler.is_idle():
15517 with self.condition:
15518 self.commands.appendleft(run_callback)
15519 self.condition.notify_all()
15520 return

Callers 1

mainFunction · 0.85

Calls

no outgoing calls

Tested by

no test coverage detected

Used in the wild real call sites across dependent graphs

searching dependent graphs…