| 187 | } |
| 188 | |
| 189 | function selectQuestions(dataset: LongMemEvalQuestion[], config: RunnerConfig): Array<{ |
| 190 | question: LongMemEvalQuestion; |
| 191 | datasetIndex: number; |
| 192 | }> { |
| 193 | const typeSet = new Set(config.selection.types); |
| 194 | const questionIdSet = new Set(config.selection.questionIds); |
| 195 | const start = config.selection.start ?? 0; |
| 196 | const end = config.selection.end ?? dataset.length; |
| 197 | |
| 198 | return dataset |
| 199 | .map((question, datasetIndex) => ({ question, datasetIndex })) |
| 200 | .filter(({ datasetIndex }) => datasetIndex >= start && datasetIndex < end) |
| 201 | .filter(({ question }) => typeSet.has(question.question_type)) |
| 202 | .filter(({ question }) => questionIdSet.size === 0 || questionIdSet.has(question.question_id)); |
| 203 | } |
| 204 | |
| 205 | async function loadResultsMap(resultsFile: string): Promise<Map<string, QuestionResultRecord>> { |
| 206 | try { |