Function validateFixtures

src/fixture-loader.ts:430–909

(fixtures: Fixture[])

}

export function validateFixtures(fixtures: Fixture[]): ValidationResult[] {
  const results: ValidationResult[] = [];

  const seenUserMessages = new Map<string, number>();

  for (let i = 0; i < fixtures.length; i++) {
    const f = fixtures[i];
    const response = f.response;

    // Skip response-shape validation for function responses — they are
    // evaluated at runtime so we cannot statically inspect them.
    if (typeof response === "function") {
      // Still validate match fields and numeric options below.
    } else {
      // --- Error checks ---

      // Response type recognition
      // Note: isContentWithToolCallsResponse must be checked before isTextResponse
      // and isToolCallResponse since it is a structural superset of both.
      if (
        !isContentWithToolCallsResponse(response) &&
        !isTextResponse(response) &&
        !isToolCallResponse(response) &&
        !isErrorResponse(response) &&
        !isEmbeddingResponse(response) &&
        !isImageResponse(response) &&
        !isAudioResponse(response) &&
        !isTranscriptionResponse(response) &&
        !isVideoResponse(response) &&
        !isJSONResponse(response)
      ) {
        results.push({
          severity: "error",
          fixtureIndex: i,
          message:
            "response is not a recognized type (must have content, toolCalls, error, embedding, image, audio, transcription, video, or json)",
        });
      }

      // When a non-empty ordered `blocks` array is present, the builders stream
      // `blocks` and IGNORE the legacy `content` mirror (see validateBlocks's
      // divergence note + isContentWithToolCallsResponse's BLOCKS-ONLY clause).
      // So an empty-string `content` is harmless in that case and must NOT raise
      // the "content is empty string" hard error. Fixtures WITHOUT blocks keep
      // the error (an empty content with no blocks produces no output).
      const hasNonEmptyBlocks =
        Array.isArray((response as { blocks?: unknown }).blocks) &&
        (response as { blocks: unknown[] }).blocks.length > 0;

      // Text response checks
      if (isTextResponse(response)) {
        if (response.content === "" && !hasNonEmptyBlocks) {
          results.push({
            severity: "error",
            fixtureIndex: i,
            message: "content is empty string",
          });
        }