| 79 | |
| 80 | @app.command() |
| 81 | def plot(files: List[Path], output: Optional[Path] = None, test_regex: Optional[str] = None, server_regex: Optional[str] = None): |
| 82 | |
| 83 | lines: List[Dict] = [] |
| 84 | for file in files: |
| 85 | if not file.exists(): |
| 86 | logger.error(f"File not found: {file}") |
| 87 | continue |
| 88 | |
| 89 | try: |
| 90 | with file.open() as f: |
| 91 | raw_data = f.read() |
| 92 | logger.info(f"Reading {file} ({len(raw_data)} bytes)") |
| 93 | |
| 94 | for line_num, line in enumerate(raw_data.split('\n'), 1): |
| 95 | line = line.strip() |
| 96 | if not line: |
| 97 | continue |
| 98 | try: |
| 99 | record = json.loads(line) |
| 100 | lines.append(record) |
| 101 | except json.JSONDecodeError as e: |
| 102 | logger.warning(f"Invalid JSON at {file}:{line_num} - {e}") |
| 103 | except Exception as e: |
| 104 | logger.error(f"Error processing {file}: {e}") |
| 105 | |
| 106 | if not lines: |
| 107 | raise Exception("No valid data was loaded") |
| 108 | |
| 109 | data_dict: Dict[Tuple, float] = {} |
| 110 | models: List[str] = [] |
| 111 | temps = set() |
| 112 | tests = set() |
| 113 | server_names = set() |
| 114 | total_counts = set() |
| 115 | for rec in lines: |
| 116 | try: |
| 117 | model = rec["model"] |
| 118 | temp = rec["temp"] |
| 119 | server_name = rec["server_name"] |
| 120 | test = rec["test"] |
| 121 | success = rec["success_ratio"] |
| 122 | success_count = rec["success_count"] |
| 123 | failure_count = rec["failure_count"] |
| 124 | total_count = success_count + failure_count |
| 125 | total_counts.add(total_count) |
| 126 | |
| 127 | if test_regex and not re.search(test_regex, test): |
| 128 | continue |
| 129 | |
| 130 | if server_regex and not re.search(server_regex, server_name): |
| 131 | continue |
| 132 | |
| 133 | data_dict[(model, temp, server_name, test)] = success |
| 134 | |
| 135 | if model not in models: |
| 136 | models.append(model) |
| 137 | temps.add(temp) |
| 138 | tests.add(test) |