Function processQuestion

packages/plugin/scripts/longmemeval/runner.ts:790–841

(args: {
    client: OpencodeClient;
    config: RunnerConfig;
    question: LongMemEvalQuestion;
    datasetIndex: number;
    state: RunnerStateFile;
    resultsMap: Map<string, QuestionResultRecord>;
})

}

async function processQuestion(args: {
    client: OpencodeClient;
    config: RunnerConfig;
    question: LongMemEvalQuestion;
    datasetIndex: number;
    state: RunnerStateFile;
    resultsMap: Map<string, QuestionResultRecord>;
}): Promise<void> {
    const { client, config, question, datasetIndex, state, resultsMap } = args;
    const resumedState = state.inProgress[question.question_id];
    if (resultsMap.has(question.question_id)) {
        if (resumedState) {
            await cleanupSessions({
                client,
                config,
                questionState: resumedState,
                state,
                questionId: question.question_id,
            });
            await markQuestionCompleted({ config, state, questionState: resumedState });
        }
        return;
    }
    if (state.completedQuestionIds.includes(question.question_id)) {
        return;
    }

    const questionState = resumedState ?? createQuestionState(question, datasetIndex);
    questionState.attemptCount += 1;
    questionState.updatedAt = nowIso();
    state.inProgress[question.question_id] = questionState;
    await persistState(config, state);

    try {
        await appendLog(config.paths.logFile, `Starting question ${question.question_id} (${question.question_type})`);
        await replayHaystack({ client, config, question, questionState, state });
        await askFinalQuestion({ client, config, question, questionState, state });
        await judgeQuestion({ config, question, questionState, state, resultsMap });
        await cleanupSessions({ client, config, questionState, state, questionId: question.question_id });
        await markQuestionCompleted({ config, state, questionState });
        await appendLog(config.paths.logFile, `Completed question ${question.question_id}`);
    } catch (error) {
        updateErrorState(questionState, "process-question", error);
        state.inProgress[question.question_id] = questionState;
        await persistState(config, state);
        await appendLog(
            config.paths.logFile,
            `Question ${question.question_id} failed: ${questionState.lastError?.message ?? "unknown"}`,
        );
        throw error;
    }
}

function buildSummary(results: QuestionResultRecord[], selectedCount: number, config: RunnerConfig): RunSummary {
    const rows: RunSummaryRow[] = LONGMEMEVAL_QUESTION_TYPES.map((category) => {