extractMissingDataFromRun extracts missing data reports from a workflow run's artifacts. experimentName and variant are the pre-resolved experiment assignment for this run; pass empty strings when no experiment context is available.
(runDir string, run WorkflowRun, verbose bool, experimentName, variant string)
| 538 | // experimentName and variant are the pre-resolved experiment assignment for this run; pass empty |
| 539 | // strings when no experiment context is available. |
| 540 | func extractMissingDataFromRun(runDir string, run WorkflowRun, verbose bool, experimentName, variant string) ([]MissingDataReport, error) { |
| 541 | logsMetricsLog.Printf("Extracting missing data from run: %d", run.DatabaseID) |
| 542 | var missingData []MissingDataReport |
| 543 | |
| 544 | // Look for the safe output artifact file that contains structured JSON with items array |
| 545 | // This file is created by the collect_ndjson_output.cjs script during workflow execution |
| 546 | // After artifact refactoring, the file is flattened to agent_output.json at root |
| 547 | agentOutputJSONPath := filepath.Join(runDir, constants.AgentOutputFilename) |
| 548 | |
| 549 | // Support both new flattened form (agent_output.json) and old forms for backward compatibility: |
| 550 | // 1. New: agent_output.json at root (after flattening) |
| 551 | // 2. Old: agent-output directory with nested agent-output file |
| 552 | // 3. Fallback: search recursively |
| 553 | var resolvedAgentOutputFile string |
| 554 | if stat, err := os.Stat(agentOutputJSONPath); err == nil && !stat.IsDir() { |
| 555 | // New flattened structure: agent_output.json at root |
| 556 | resolvedAgentOutputFile = agentOutputJSONPath |
| 557 | if verbose { |
| 558 | fmt.Fprintln(os.Stderr, console.FormatInfoMessage(fmt.Sprintf("Found %s at root: %s", constants.AgentOutputFilename, agentOutputJSONPath))) |
| 559 | } |
| 560 | } else { |
| 561 | // Try old structure: agent-output directory |
| 562 | agentOutputPath := filepath.Join(runDir, constants.AgentOutputArtifactName) |
| 563 | if stat, err := os.Stat(agentOutputPath); err == nil { |
| 564 | if stat.IsDir() { |
| 565 | // Directory form – look for nested file |
| 566 | nested := filepath.Join(agentOutputPath, constants.AgentOutputArtifactName) |
| 567 | if fileutil.FileExists(nested) { |
| 568 | resolvedAgentOutputFile = nested |
| 569 | if verbose { |
| 570 | fmt.Fprintln(os.Stderr, console.FormatInfoMessage("agent_output.json is a directory; using nested file "+nested)) |
| 571 | } |
| 572 | } else if verbose { |
| 573 | if _, nestedErr := os.Stat(nested); nestedErr != nil { |
| 574 | fmt.Fprintln(os.Stderr, console.FormatWarningMessage( |
| 575 | fmt.Sprintf("agent_output.json directory present but nested file missing: %v", nestedErr))) |
| 576 | } else { |
| 577 | fmt.Fprintln(os.Stderr, console.FormatWarningMessage( |
| 578 | "agent_output.json directory present but nested path is not a file")) |
| 579 | } |
| 580 | } |
| 581 | } else { |
| 582 | // Regular file |
| 583 | resolvedAgentOutputFile = agentOutputPath |
| 584 | } |
| 585 | } else { |
| 586 | // Not present at root – search recursively (depth-first) for a file named agent_output.json |
| 587 | if found, ok := findAgentOutputFile(runDir); ok { |
| 588 | resolvedAgentOutputFile = found |
| 589 | if verbose && found != agentOutputPath { |
| 590 | fmt.Fprintln(os.Stderr, console.FormatInfoMessage("Found agent_output.json at "+found)) |
| 591 | } |
| 592 | } |
| 593 | } |
| 594 | } |
| 595 | |
| 596 | if resolvedAgentOutputFile != "" { |
| 597 | // Sanitize the path to prevent path traversal attacks |