| 706 | |
| 707 | |
| 708 | def evaluate(model, data, epoch, args, tb_writer=None, step=None, num_feed_images=None): |
| 709 | metrics = {} |
| 710 | models = [model] |
| 711 | names = [''] |
| 712 | assert len(names) == len(models) |
| 713 | for name, model_i in zip(names, models): |
| 714 | model_i.eval() |
| 715 | zero_shot_metrics = zero_shot_eval(model_i, data, epoch, args) |
| 716 | zero_shot_metrics = dict((name + k, v) |
| 717 | for k, v in zero_shot_metrics.items()) |
| 718 | metrics.update(zero_shot_metrics) |
| 719 | |
| 720 | if not metrics: |
| 721 | return metrics |
| 722 | |
| 723 | if not is_master(args): |
| 724 | return metrics |
| 725 | |
| 726 | logging.info( |
| 727 | f"Eval Epoch: {epoch} " |
| 728 | + "\t".join([f"{k}: {round(v, 4):.4f}" for k, v in metrics.items()]) |
| 729 | ) |
| 730 | |
| 731 | if args.save_logs: |
| 732 | for name, val in metrics.items(): |
| 733 | if tb_writer is not None: |
| 734 | tb_writer.add_scalar(f"val/{name}", val, epoch) |
| 735 | |
| 736 | with open(os.path.join(args.checkpoint_path, "results.jsonl"), "a+") as f: |
| 737 | f.write(json.dumps(metrics)) |
| 738 | f.write("\n") |
| 739 | |
| 740 | if args.wandb: |
| 741 | assert wandb is not None, 'Please install wandb.' |
| 742 | for name, val in metrics.items(): |
| 743 | log = {f"val/{name}": val, 'epoch': epoch} |
| 744 | extra_kwargs = dict() |
| 745 | if step is not None: |
| 746 | log['step'] = step |
| 747 | extra_kwargs['step'] = step |
| 748 | if num_feed_images is not None: |
| 749 | log['num_feed_images'] = num_feed_images |
| 750 | wandb.log(log, **extra_kwargs) |
| 751 | return metrics |
| 752 | |
| 753 | |
| 754 | def get_metrics(image_features, text_features, logit_scale): |