| 346 | |
| 347 | @dataclasses.dataclass |
| 348 | class ValidationGenerationsLogger: |
| 349 | project_name: str = None |
| 350 | experiment_name: str = None |
| 351 | |
| 352 | def log(self, loggers, samples, step): |
| 353 | if "wandb" in loggers: |
| 354 | self.log_generations_to_wandb(samples, step) |
| 355 | if "swanlab" in loggers: |
| 356 | self.log_generations_to_swanlab(samples, step) |
| 357 | if "mlflow" in loggers: |
| 358 | self.log_generations_to_mlflow(samples, step) |
| 359 | |
| 360 | if "clearml" in loggers: |
| 361 | self.log_generations_to_clearml(samples, step) |
| 362 | if "tensorboard" in loggers: |
| 363 | self.log_generations_to_tensorboard(samples, step) |
| 364 | |
| 365 | if "vemlp_wandb" in loggers: |
| 366 | self.log_generations_to_vemlp_wandb(samples, step) |
| 367 | |
| 368 | def log_generations_to_vemlp_wandb(self, samples, step): |
| 369 | from volcengine_ml_platform import wandb as vemlp_wandb |
| 370 | |
| 371 | self._log_generations_to_wandb(samples, step, vemlp_wandb) |
| 372 | |
| 373 | def log_generations_to_wandb(self, samples, step): |
| 374 | import wandb |
| 375 | |
| 376 | self._log_generations_to_wandb(samples, step, wandb) |
| 377 | |
| 378 | def _log_generations_to_wandb(self, samples, step, wandb): |
| 379 | """Log samples to wandb as a table""" |
| 380 | |
| 381 | # Create column names for all samples |
| 382 | columns = ["step"] + sum( |
| 383 | [[f"input_{i + 1}", f"output_{i + 1}", f"score_{i + 1}"] for i in range(len(samples))], [] |
| 384 | ) |
| 385 | |
| 386 | if not hasattr(self, "validation_table"): |
| 387 | # Initialize the table on first call |
| 388 | self.validation_table = wandb.Table(columns=columns) |
| 389 | |
| 390 | # Create a new table with same columns and existing data |
| 391 | # Workaround for https://github.com/wandb/wandb/issues/2981#issuecomment-1997445737 |
| 392 | new_table = wandb.Table(columns=columns, data=self.validation_table.data) |
| 393 | |
| 394 | # Add new row with all data |
| 395 | row_data = [] |
| 396 | row_data.append(step) |
| 397 | for sample in samples: |
| 398 | row_data.extend(sample) |
| 399 | |
| 400 | new_table.add_data(*row_data) |
| 401 | |
| 402 | # Update reference and log |
| 403 | if wandb.run is not None: |
| 404 | wandb.log({"val/generations": new_table}, step=step) |
| 405 | self.validation_table = new_table |