Function outputsAreEqual

app/src/server/tasks/evaluateTestSetEntries.task.ts:517–542

(
  firstOutput: DatasetEntryOutput["output"],
  secondOutput: DatasetEntryOutput["output"],
)

};

const outputsAreEqual = (
  firstOutput: DatasetEntryOutput["output"],
  secondOutput: DatasetEntryOutput["output"],
) => {
  try {
    const typedFirstOutput = chatCompletionMessage.parse(firstOutput);
    const typedSecondOutput = chatCompletionMessage.parse(secondOutput);

    typedFirstOutput.tool_calls = typedFirstOutput.tool_calls?.map((call) => ({
      ...call,
      // Remove ids, which are not relevant to the comparison
      id: "",
      // Ignore differences in the serialization of arguments
      function: { ...call.function, arguments: JSON.parse(call.function.arguments) },
    }));
    typedSecondOutput.tool_calls = typedSecondOutput.tool_calls?.map((call) => ({
      ...call,
      id: "",
      function: { ...call.function, arguments: JSON.parse(call.function.arguments) },
    }));

    return isEqual(typedFirstOutput, typedSecondOutput);
  } catch {
    return false;
  }
};

const formatDatasetEntryInputInstructions = (datasetEntryInput: DatasetEntryInput) => {
  const { messages, tool_choice, tools } = typedDatasetEntryInput(datasetEntryInput);