Walk through the output folder (including sub-dir) and analyze the overall.json file Rule: - valid overall file: **/{agent}/{task}/overall.json - if a same (agent, task) pair, select the latest one
(config: str, output: str, since_timestamp: float)
| 54 | |
| 55 | |
| 56 | def analyze_output(config: str, output: str, since_timestamp: float): |
| 57 | """ |
| 58 | Walk through the output folder (including sub-dir) and analyze the overall.json file |
| 59 | Rule: |
| 60 | - valid overall file: **/{agent}/{task}/overall.json |
| 61 | - if a same (agent, task) pair, select the latest one |
| 62 | """ |
| 63 | loader = ConfigLoader() |
| 64 | config: dict = loader.load_from(config) |
| 65 | assert "definition" in config, "definition not found in config" |
| 66 | assert "agent" in config["definition"], "agent not found in config.definition" |
| 67 | assert "task" in config["definition"], "task not found in config.definition" |
| 68 | agents = set(config["definition"]["agent"].keys()).intersection( |
| 69 | set(MODEL_MAP.keys()) |
| 70 | ) |
| 71 | tasks = list(config["definition"]["task"].keys()) |
| 72 | |
| 73 | print( |
| 74 | ColorMessage.cyan( |
| 75 | f"Available Agents ({len(agents)}):\n " |
| 76 | + "\n ".join(agents) |
| 77 | + "\n\n" |
| 78 | + f"Available Tasks ({len(tasks)}):\n " |
| 79 | + "\n ".join(tasks) |
| 80 | + "\n" |
| 81 | ) |
| 82 | ) |
| 83 | |
| 84 | overall_dict = OrderedDict() # agent -> task -> {file: str, time: float} |
| 85 | for root, dirs, files in os.walk(output): |
| 86 | if "overall.json" in files: |
| 87 | # get full path of root |
| 88 | root = os.path.abspath(root) |
| 89 | # get agent and task name |
| 90 | pattern = root.split("/") |
| 91 | if len(pattern) < 2: |
| 92 | continue |
| 93 | agent = pattern[-2] |
| 94 | task = pattern[-1] |
| 95 | ct = os.path.getmtime(os.path.join(root, "overall.json")) |
| 96 | if agent not in agents: |
| 97 | continue |
| 98 | elif task not in tasks: |
| 99 | continue |
| 100 | elif ct < since_timestamp: |
| 101 | continue |
| 102 | agent = MODEL_MAP[agent] |
| 103 | if agent in overall_dict and task in overall_dict[agent]: |
| 104 | # get time |
| 105 | if ct < overall_dict[agent][task]["time"]: |
| 106 | continue |
| 107 | overall_dict.setdefault(agent, OrderedDict()) |
| 108 | overall_dict[agent][task] = { |
| 109 | "file": os.path.join(root, "overall.json"), |
| 110 | "time": os.path.getmtime(os.path.join(root, "overall.json")), |
| 111 | } |
| 112 | |
| 113 | # agent -> task -> {file: str, time: str(YYYY-MM-DD HH:MM:SS), overall: dict} |
no test coverage detected