Validate a single task directory. Returns a list of error strings.
(task_dir: Path, schema: dict)
| 111 | |
| 112 | |
| 113 | def validate_task_dir(task_dir: Path, schema: dict) -> list[str]: |
| 114 | """Validate a single task directory. Returns a list of error strings.""" |
| 115 | errors: list[str] = [] |
| 116 | |
| 117 | # Check task.toml exists and is valid |
| 118 | toml_path = task_dir / "task.toml" |
| 119 | if not toml_path.exists(): |
| 120 | errors.append("task.toml not found") |
| 121 | return errors |
| 122 | |
| 123 | try: |
| 124 | with open(toml_path, "rb") as fh: |
| 125 | data = normalize_task_data(tomllib.load(fh), task_dir) |
| 126 | except Exception as exc: |
| 127 | errors.append(f"task.toml parse error: {exc}") |
| 128 | return errors |
| 129 | |
| 130 | errors.extend(validate_toml_against_schema(data, schema)) |
| 131 | |
| 132 | # Check required files |
| 133 | required_files = [ |
| 134 | "instruction.md", |
| 135 | "verifier/test_output.py", |
| 136 | ] |
| 137 | for rel_path in required_files: |
| 138 | if not (task_dir / rel_path).exists(): |
| 139 | errors.append(f"Missing required file: {rel_path}") |
| 140 | |
| 141 | return errors |
| 142 | |
| 143 | |
| 144 | def run_oracle(task_dir: Path) -> tuple[bool, str]: |
no test coverage detected