Interface ToolProbeDefinition

tools/develop/llm-parameter-sweep/sweep.ts:187–216

* Tool probes verify real function-calling support by invoking a tool and inspecting the response. * Distinct from sweeps (which vary AixAPI_Model parameters): a probe sets request-level `tools` + `toolsPolicy` * and a user prompt, then classifies pass/fail from the parser's function-call events.

 * entries (e.g. `"fn": ["auto"]`). Absence of the key means the probe did not pass for that model.
 */
interface ToolProbeDefinition {
  /** Probe name, shown in logs. Also used as the output JSON key (e.g. 'fn'). */
  name: string;
  /** Human-readable description for --help */
  description: string;
  applicability:
    | { type: 'all' }
    | { type: 'dialects'; dialects: AixAPI_Access['dialect'][] };
  userPrompt: string;
  tools: AixTools_ToolDefinition[];
  toolsPolicy: AixTools_ToolsPolicy;
  /** Function name that must be called for the probe to pass. */
  expectedFunctionName: string;
  /** Value listed under the output key when the probe passes (e.g. 'auto'). */
  passValue: string;
  /** Optional additional validation on the parsed args object (must return true). */
  validateArgs?: (args: any) => boolean;
  /**
   * If set, after the model emits a valid tool call we inject a canned tool_response and re-dispatch.
   * Pass then additionally requires: turn 2 emits text (no loop, no empty), and that text contains
   * at least one of `signalTokens` (case-insensitive substring) - proving the model actually consumed
   * the tool result rather than ignoring it.
   */
  roundtrip?: {
    /** Serialized tool result (typically a JSON-stringified object). Must be a JSON object string per validation upstream. */
    cannedResult: string;
    /** At least one of these substrings must appear (case-insensitive) in turn 2's text. */
    signalTokens: string[];
  };
}

const _getWeatherTool: AixTools_ToolDefinition = {
  type: 'function_call',