(
future,
output_handle,
error_handle,
stats,
)
| 231 | |
| 232 | |
| 233 | def write_finished_result( |
| 234 | future, |
| 235 | output_handle, |
| 236 | error_handle, |
| 237 | stats, |
| 238 | ): |
| 239 | sample = future.result() |
| 240 | if sample["status"] == "error": |
| 241 | error_handle.write(json.dumps(sample, ensure_ascii=False) + "\n") |
| 242 | stats["errors"] += 1 |
| 243 | return |
| 244 | |
| 245 | context_length = compute_context_length(sample.get("conversations", [])) |
| 246 | stats["context_sum"] += context_length |
| 247 | stats["context_min"] = ( |
| 248 | context_length |
| 249 | if stats["context_min"] is None |
| 250 | else min(stats["context_min"], context_length) |
| 251 | ) |
| 252 | stats["context_max"] = max(stats["context_max"], context_length) |
| 253 | stats["success"] += 1 |
| 254 | output_handle.write(json.dumps(sample, ensure_ascii=False) + "\n") |
| 255 | |
| 256 | |
| 257 | def print_config(args): |
no test coverage detected