Function buildComputerUseTools

packages/runtime/src/computer-use-tools.ts:612–2875

(deps: {
  backend: CuDispatchBackend;
  overlay?: CuOverlayHook;
  /**
   * Whether the machine is locked right now.
   *
   * Consulted before every action and every observation, because a locked
   * screen is the user saying they have left: past that point the agent would
   * be driving and reading a machine its owner deliberately closed off, with
   * nobody watching and no way to intervene. Refusing produces the
   * `screen_locked` session state, which is what keeps the refusal from having
   * to be re-derived on every subsequent call — and which only
   * `sessionEvents.screenUnlocked` can leave.
   *
   * Takes the session so the host can record which sessions it will have to
   * release when the machine comes back. Without that, a session whose very
   * first call landed on a locked screen would latch `screen_locked` while
   * being unknown to the host, and would stay latched for the rest of its life.
   *
   * Absent means the host cannot answer the question, not that the machine is
   * unlocked; there is no lock guard at all in that case.
   */
  screenLocked?: (context: { sessionId: string }) => boolean | Promise<boolean>;
  presentationReadyTimeoutMs?: number;
  presentationFinishedTimeoutMs?: number;
  /** Diagnostics only. Never on by default, never able to change an outcome. */
  debug?: (record: CuDebugRecord) => void;
})

}

export function buildComputerUseTools(deps: {
  backend: CuDispatchBackend;
  overlay?: CuOverlayHook;
  /**
   * Whether the machine is locked right now.
   *
   * Consulted before every action and every observation, because a locked
   * screen is the user saying they have left: past that point the agent would
   * be driving and reading a machine its owner deliberately closed off, with
   * nobody watching and no way to intervene. Refusing produces the
   * `screen_locked` session state, which is what keeps the refusal from having
   * to be re-derived on every subsequent call — and which only
   * `sessionEvents.screenUnlocked` can leave.
   *
   * Takes the session so the host can record which sessions it will have to
   * release when the machine comes back. Without that, a session whose very
   * first call landed on a locked screen would latch `screen_locked` while
   * being unknown to the host, and would stay latched for the rest of its life.
   *
   * Absent means the host cannot answer the question, not that the machine is
   * unlocked; there is no lock guard at all in that case.
   */
  screenLocked?: (context: { sessionId: string }) => boolean | Promise<boolean>;
  presentationReadyTimeoutMs?: number;
  presentationFinishedTimeoutMs?: number;
  /** Diagnostics only. Never on by default, never able to change an outcome. */
  debug?: (record: CuDebugRecord) => void;
}): ComputerUseToolSet {
  const presentationReadyTimeoutMs = deps.presentationReadyTimeoutMs ?? 1_000;
  const presentationFinishedTimeoutMs =
    deps.presentationFinishedTimeoutMs ?? DEFAULT_PRESENTATION_FINISHED_TIMEOUT_MS;
  const invocationQueues = new Map<string, Promise<void>>();
  const presentationWaiters = new Map<string, Set<() => void>>();
  const presentationQueueWaiters = new Map<string, Set<() => void>>();
  const presentationGenerations = new Map<string, number>();
  const pendingInvocationTurns = new Map<string, Set<string>>();
  let presentationQueue = Promise.resolve();
  let accessibilityPermissionRequested = false;
  interface SessionObservationRecord {
    turnId: string;
    state: CuaFrameState;
    backendObservationId?: string;
    appId?: string;
    /** The non-canonical name that resolved this observation, if any. */
    appAlias?: string;
    windowId?: number;
    elements?: Map<string, CuObservedElement>;
    /** From the last observation: the windows stacked above the target. */
    obscuringRects?: Array<{ x: number; y: number; width: number; height: number }>;
  }
  const observations = new Map<string, SessionObservationRecord>();
  interface SessionStateRecord {
    turnId?: string;
    state: CuaSessionState;
  }
  const sessionStates = new Map<string, SessionStateRecord>();

  function sessionState(sessionId: string, turnId?: string): CuaSessionState {