Create evaluator(s) for a given dataset. This uses the special metadata "evaluator_type" associated with each builtin dataset. For your own dataset, you can simply create an evaluator manually in your script and do not have to worry about the hacky if-else logic here
(cls, cfg, dataset_name, output_folder=None)
| 50 | |
| 51 | @classmethod |
| 52 | def build_evaluator(cls, cfg, dataset_name, output_folder=None): |
| 53 | """ |
| 54 | Create evaluator(s) for a given dataset. |
| 55 | This uses the special metadata "evaluator_type" associated with each builtin dataset. |
| 56 | For your own dataset, you can simply create an evaluator manually in your |
| 57 | script and do not have to worry about the hacky if-else logic here. |
| 58 | """ |
| 59 | if output_folder is None: |
| 60 | output_folder = os.path.join(cfg.OUTPUT_DIR, "inference") |
| 61 | evaluator_list = [] |
| 62 | evaluator_type = MetadataCatalog.get(dataset_name).evaluator_type |
| 63 | if evaluator_type in ["sem_seg", "coco_panoptic_seg"]: |
| 64 | evaluator_list.append( |
| 65 | SemSegEvaluator( |
| 66 | dataset_name, |
| 67 | distributed=True, |
| 68 | num_classes=cfg.MODEL.SEM_SEG_HEAD.NUM_CLASSES, |
| 69 | ignore_label=cfg.MODEL.SEM_SEG_HEAD.IGNORE_VALUE, |
| 70 | output_dir=output_folder, |
| 71 | ) |
| 72 | ) |
| 73 | if evaluator_type in ["coco", "coco_panoptic_seg"]: |
| 74 | evaluator_list.append(COCOEvaluator(dataset_name, cfg, True, output_folder)) |
| 75 | if evaluator_type == "coco_panoptic_seg": |
| 76 | evaluator_list.append(COCOPanopticEvaluator(dataset_name, output_folder)) |
| 77 | if evaluator_type == "cityscapes_instance": |
| 78 | assert ( |
| 79 | torch.cuda.device_count() >= comm.get_rank() |
| 80 | ), "CityscapesEvaluator currently do not work with multiple machines." |
| 81 | return CityscapesInstanceEvaluator(dataset_name) |
| 82 | if evaluator_type == "cityscapes_sem_seg": |
| 83 | assert ( |
| 84 | torch.cuda.device_count() >= comm.get_rank() |
| 85 | ), "CityscapesEvaluator currently do not work with multiple machines." |
| 86 | return CityscapesSemSegEvaluator(dataset_name) |
| 87 | elif evaluator_type == "pascal_voc": |
| 88 | return PascalVOCDetectionEvaluator(dataset_name) |
| 89 | elif evaluator_type == "lvis": |
| 90 | return LVISEvaluator(dataset_name, cfg, True, output_folder) |
| 91 | if len(evaluator_list) == 0: |
| 92 | raise NotImplementedError( |
| 93 | "no Evaluator for the dataset {} with the type {}".format( |
| 94 | dataset_name, evaluator_type |
| 95 | ) |
| 96 | ) |
| 97 | elif len(evaluator_list) == 1: |
| 98 | return evaluator_list[0] |
| 99 | return DatasetEvaluators(evaluator_list) |
| 100 | |
| 101 | @classmethod |
| 102 | def test_with_TTA(cls, cfg, model): |