Function replayHaystack

packages/plugin/scripts/longmemeval/runner.ts:525–607

(args: {
    client: OpencodeClient;
    config: RunnerConfig;
    question: LongMemEvalQuestion;
    questionState: QuestionRunState;
    state: RunnerStateFile;
})

}

async function replayHaystack(args: {
    client: OpencodeClient;
    config: RunnerConfig;
    question: LongMemEvalQuestion;
    questionState: QuestionRunState;
    state: RunnerStateFile;
}): Promise<void> {
    const { client, config, question, questionState, state } = args;

    for (let haystackIndex = questionState.currentHaystackSessionIndex; haystackIndex < question.haystack_sessions.length; haystackIndex += 1) {
        const datasetSession = question.haystack_sessions[haystackIndex] ?? [];
        const sessionProgress = questionState.haystackSessions[haystackIndex]!;
        if (sessionProgress.completed) {
            questionState.currentHaystackSessionIndex = haystackIndex + 1;
            questionState.currentTurnIndex = 0;
            continue;
        }

        questionState.status = "replaying";
        questionState.currentHaystackSessionIndex = haystackIndex;
        questionState.currentTurnIndex = sessionProgress.nextTurnIndex;
        questionState.updatedAt = nowIso();
        state.inProgress[question.question_id] = questionState;
        await persistState(config, state);

        const openCodeSessionId = await ensureReplaySession(client, config, question, questionState, sessionProgress, state);
        await sendBannerIfNeeded({
            client,
            config,
            sessionId: openCodeSessionId,
            questionState,
            state,
            questionId: question.question_id,
            bannerText: buildSessionBanner(sessionProgress.date, sessionProgress.datasetSessionId),
            markBannerSent: () => {
                sessionProgress.bannerSent = true;
            },
            alreadySent: sessionProgress.bannerSent,
            operationLabel: `banner:${question.question_id}:${haystackIndex}`,
        });

        for (let turnIndex = sessionProgress.nextTurnIndex; turnIndex < datasetSession.length; turnIndex += 1) {
            const turn = datasetSession[turnIndex]!;
            if (turn.role !== "user") {
                continue;
            }

            const result = await sendPromptAndCapture(client, config, openCodeSessionId, turn.content, {
                operationLabel: `replay:${question.question_id}:${haystackIndex}:${turnIndex}`,
                tools: {},
            });
            addOpenCodeUsage(questionState, result.usage, result.actualCostUsd, config);

            sessionProgress.nextTurnIndex = turnIndex + 1;
            questionState.currentTurnIndex = sessionProgress.nextTurnIndex;
            questionState.updatedAt = nowIso();
            state.inProgress[question.question_id] = questionState;
            await persistState(config, state);