Load samples from a JSONL file, deduplicated by doc_id. lm-eval writes one line per (doc_id, filter) combination. The response text is the same across filters, so we keep only the first occurrence.
(path: Path)
| 244 | |
| 245 | |
| 246 | def load_samples(path: Path) -> list[dict]: |
| 247 | """Load samples from a JSONL file, deduplicated by doc_id. |
| 248 | |
| 249 | lm-eval writes one line per (doc_id, filter) combination. The response |
| 250 | text is the same across filters, so we keep only the first occurrence. |
| 251 | """ |
| 252 | seen = set() |
| 253 | samples = [] |
| 254 | with open(path) as f: |
| 255 | for line in f: |
| 256 | line = line.strip() |
| 257 | if not line: |
| 258 | continue |
| 259 | sample = json.loads(line) |
| 260 | doc_id = sample["doc_id"] |
| 261 | if doc_id not in seen: |
| 262 | seen.add(doc_id) |
| 263 | samples.append(sample) |
| 264 | return samples |
| 265 | |
| 266 | |
| 267 | def load_existing_results(path: Path) -> dict[int, dict]: |