Create evaluator(s) for a given dataset. This uses the special metadata "evaluator_type" associated with each builtin dataset. For your own dataset, you can simply create an evaluator manually in your script and do not have to worry about the hacky if-else logic here.
(cfg, dataset_name, output_folder=None)
| 474 | |
| 475 | |
| 476 | def build_evaluator(cfg, dataset_name, output_folder=None): |
| 477 | """ |
| 478 | Create evaluator(s) for a given dataset. |
| 479 | This uses the special metadata "evaluator_type" associated with each |
| 480 | builtin dataset. For your own dataset, you can simply create an |
| 481 | evaluator manually in your script and do not have to worry about the |
| 482 | hacky if-else logic here. |
| 483 | """ |
| 484 | |
| 485 | cfg_model_decoder_test = cfg["MODEL"]["DECODER"]["TEST"] |
| 486 | |
| 487 | if output_folder is None: |
| 488 | output_folder = os.path.join(cfg["OUTPUT_DIR"], "inference") |
| 489 | evaluator_list = [] |
| 490 | evaluator_type = MetadataCatalog.get(dataset_name).evaluator_type |
| 491 | # for pascal part |
| 492 | if evaluator_type == "pascal_part": |
| 493 | return PASCALPARTEvaluator(dataset_name, output_dir=output_folder) |
| 494 | # FIXME interactive |
| 495 | if evaluator_type in ['sam_interactive', 'pascal_part_interactive', 'coco_panoptic_seg_interactive']: |
| 496 | evaluator_list.append(InteractiveEvaluator(dataset_name, output_dir=output_folder)) |
| 497 | # for box interactive evaluation |
| 498 | if evaluator_type in ['coco_panoptic_seg_interactive_jointboxpoint']: |
| 499 | box_interactive = cfg_model_decoder_test.get('BOX_INTERACTIVE', False) |
| 500 | evaluator_list.append(JointBoxPointInteractiveEvaluator(dataset_name, output_dir=output_folder, box_interactive=box_interactive)) |
| 501 | # evaluator_list.append(COCOPanopticEvaluator(dataset_name, output_folder)) |
| 502 | # evaluator_list.append(COCOEvaluator(dataset_name, output_dir=output_folder)) |
| 503 | # evaluator_list.append(SemSegEvaluator(dataset_name, distributed=True, output_dir=output_folder)) |
| 504 | if evaluator_type == 'sam': |
| 505 | evaluator_list.append(COCOEvaluator("coco_2017_val", output_dir=output_folder)) |
| 506 | |
| 507 | # semantic segmentation |
| 508 | if evaluator_type in ["sem_seg", "ade20k_panoptic_seg"]: |
| 509 | evaluator_list.append( |
| 510 | SemSegEvaluator( |
| 511 | dataset_name, |
| 512 | distributed=True, |
| 513 | output_dir=output_folder, |
| 514 | ) |
| 515 | ) |
| 516 | # instance segmentation |
| 517 | if evaluator_type == "coco": |
| 518 | evaluator_list.append(COCOEvaluator(dataset_name, output_dir=output_folder)) |
| 519 | |
| 520 | # panoptic segmentation |
| 521 | if evaluator_type in [ |
| 522 | "coco_panoptic_seg", |
| 523 | # "coco_panoptic_seg_interactive", |
| 524 | "ade20k_panoptic_seg", |
| 525 | "cityscapes_panoptic_seg", |
| 526 | "mapillary_vistas_panoptic_seg", |
| 527 | "scannet_panoptic_seg", |
| 528 | "bdd_panoptic_pano" |
| 529 | ]: |
| 530 | if cfg_model_decoder_test["PANOPTIC_ON"]: |
| 531 | evaluator_list.append(COCOPanopticEvaluator(dataset_name, output_folder)) |
| 532 | # COCO |
| 533 | if (evaluator_type == "coco_panoptic_seg" and cfg_model_decoder_test["INSTANCE_ON"]) or evaluator_type == "object365_od": |
no test coverage detected