(
input_shard: Path, output_shard: Path
)
| 485 | |
| 486 | |
| 487 | def validate_rank_shard( |
| 488 | input_shard: Path, output_shard: Path |
| 489 | ) -> tuple[list[JsonRow], list[JsonRow]]: |
| 490 | shard_inputs = load_jsonl(input_shard, "Rank input shard", validate_rank_input_row) |
| 491 | require_unique_paths(shard_inputs, f"Rank input shard {input_shard.name}") |
| 492 | shard_outputs = load_jsonl(output_shard, "Rank output shard", validate_rank_output_row) |
| 493 | require_unique_paths(shard_outputs, f"Rank output shard {output_shard.name}") |
| 494 | |
| 495 | expected_paths = {str(row["path"]) for row in shard_inputs} |
| 496 | actual_paths = {str(row["path"]) for row in shard_outputs} |
| 497 | if expected_paths != actual_paths: |
| 498 | missing = sorted(expected_paths - actual_paths) |
| 499 | unknown = sorted(actual_paths - expected_paths) |
| 500 | raise SystemExit( |
| 501 | f"{output_shard}: paths do not match its input shard; " |
| 502 | f"missing={missing}; unknown={unknown}" |
| 503 | ) |
| 504 | |
| 505 | area_by_path = {str(row["path"]): row["area"] for row in shard_inputs} |
| 506 | for row in shard_outputs: |
| 507 | row_path = str(row["path"]) |
| 508 | if row["area"] != area_by_path[row_path]: |
| 509 | raise SystemExit(f"{output_shard}: area does not match rank input for {row_path}") |
| 510 | return shard_inputs, shard_outputs |
| 511 | |
| 512 | |
| 513 | def validate_rank_shard_command(args: argparse.Namespace) -> None: |
no test coverage detected