Create evaluator(s) for a given dataset. This uses the special metadata "evaluator_type" associated with each builtin dataset. For your own dataset, you can simply create an evaluator manually in your script and do not have to worry about the hacky if-else logic here.
(cfg, dataset_name, output_folder=None)
| 59 | |
| 60 | |
| 61 | def get_evaluator(cfg, dataset_name, output_folder=None): |
| 62 | """ |
| 63 | Create evaluator(s) for a given dataset. |
| 64 | This uses the special metadata "evaluator_type" associated with each builtin dataset. |
| 65 | For your own dataset, you can simply create an evaluator manually in your |
| 66 | script and do not have to worry about the hacky if-else logic here. |
| 67 | """ |
| 68 | if output_folder is None: |
| 69 | output_folder = os.path.join(cfg.OUTPUT_DIR, "inference") |
| 70 | evaluator_list = [] |
| 71 | evaluator_type = MetadataCatalog.get(dataset_name).evaluator_type |
| 72 | if evaluator_type in ["sem_seg", "coco_panoptic_seg"]: |
| 73 | evaluator_list.append( |
| 74 | SemSegEvaluator( |
| 75 | dataset_name, |
| 76 | distributed=True, |
| 77 | num_classes=cfg.MODEL.SEM_SEG_HEAD.NUM_CLASSES, |
| 78 | ignore_label=cfg.MODEL.SEM_SEG_HEAD.IGNORE_VALUE, |
| 79 | output_dir=output_folder, |
| 80 | ) |
| 81 | ) |
| 82 | if evaluator_type in ["coco", "coco_panoptic_seg"]: |
| 83 | evaluator_list.append(COCOEvaluator(dataset_name, cfg, True, output_folder)) |
| 84 | if evaluator_type == "coco_panoptic_seg": |
| 85 | evaluator_list.append(COCOPanopticEvaluator(dataset_name, output_folder)) |
| 86 | if evaluator_type == "cityscapes_instance": |
| 87 | assert ( |
| 88 | torch.cuda.device_count() >= comm.get_rank() |
| 89 | ), "CityscapesEvaluator currently do not work with multiple machines." |
| 90 | return CityscapesInstanceEvaluator(dataset_name) |
| 91 | if evaluator_type == "cityscapes_sem_seg": |
| 92 | assert ( |
| 93 | torch.cuda.device_count() >= comm.get_rank() |
| 94 | ), "CityscapesEvaluator currently do not work with multiple machines." |
| 95 | return CityscapesSemSegEvaluator(dataset_name) |
| 96 | if evaluator_type == "pascal_voc": |
| 97 | return PascalVOCDetectionEvaluator(dataset_name) |
| 98 | if evaluator_type == "lvis": |
| 99 | return LVISEvaluator(dataset_name, cfg, True, output_folder) |
| 100 | if len(evaluator_list) == 0: |
| 101 | raise NotImplementedError( |
| 102 | "no Evaluator for the dataset {} with the type {}".format(dataset_name, evaluator_type) |
| 103 | ) |
| 104 | if len(evaluator_list) == 1: |
| 105 | return evaluator_list[0] |
| 106 | return DatasetEvaluators(evaluator_list) |
| 107 | |
| 108 | |
| 109 | def do_test(cfg, model): |
no test coverage detected