Method test_fast_eval

det/tests/data/test_coco_evaluation.py:21–124

(self)

class TestCOCOeval(unittest.TestCase):
    def test_fast_eval(self):
        # A small set of images/categories from COCO val
        # fmt: off
        detections = [{"image_id": 139, "category_id": 1, "bbox": [417.3332824707031, 159.27003479003906, 47.66064453125, 143.00193786621094], "score": 0.9949821829795837, "segmentation": {"size": [426, 640], "counts": "Tc`52W=3N0N4aNN^E7]:4XE1g:8kDMT;U100000001O1gE[Nk8h1dFiNY9Z1aFkN]9g2J3NdN`FlN`9S1cFRN07]9g1bFoM6;X9c1cFoM=8R9g1bFQN>3U9Y30O01OO1O001N2O1N1O4L4L5UNoE3V:CVF6Q:@YF9l9@ZF<k9[O`F=];HYnX2"}}, {"image_id": 139, "category_id": 1, "bbox": [383.5909118652344, 172.0777587890625, 17.959075927734375, 36.94813537597656], "score": 0.7685421705245972, "segmentation": {"size": [426, 640], "counts": "lZP5m0Z<300O100O100000001O00]OlC0T<OnCOT<OnCNX<JnC2bQT3"}}, {"image_id": 139, "category_id": 1, "bbox": [457.8359069824219, 158.88027954101562, 9.89764404296875, 8.771820068359375], "score": 0.07092753797769547, "segmentation": {"size": [426, 640], "counts": "bSo54T=2N2O1001O006ImiW2"}}] # noqa
        gt_annotations = {"categories": [{"supercategory": "person", "id": 1, "name": "person"}, {"supercategory": "furniture", "id": 65, "name": "bed"}], "images": [{"license": 4, "file_name": "000000000285.jpg", "coco_url": "http://images.cocodataset.org/val2017/000000000285.jpg", "height": 640, "width": 586, "date_captured": "2013-11-18 13:09:47", "flickr_url": "http://farm8.staticflickr.com/7434/9138147604_c6225224b8_z.jpg", "id": 285}, {"license": 2, "file_name": "000000000139.jpg", "coco_url": "http://images.cocodataset.org/val2017/000000000139.jpg", "height": 426, "width": 640, "date_captured": "2013-11-21 01:34:01", "flickr_url": "http://farm9.staticflickr.com/8035/8024364858_9c41dc1666_z.jpg", "id": 139}], "annotations": [{"segmentation": [[428.19, 219.47, 430.94, 209.57, 430.39, 210.12, 421.32, 216.17, 412.8, 217.27, 413.9, 214.24, 422.42, 211.22, 429.29, 201.6, 430.67, 181.8, 430.12, 175.2, 427.09, 168.06, 426.27, 164.21, 430.94, 159.26, 440.29, 157.61, 446.06, 163.93, 448.53, 168.06, 448.53, 173.01, 449.08, 174.93, 454.03, 185.1, 455.41, 188.4, 458.43, 195.0, 460.08, 210.94, 462.28, 226.61, 460.91, 233.76, 454.31, 234.04, 460.08, 256.85, 462.56, 268.13, 465.58, 290.67, 465.85, 293.14, 463.38, 295.62, 452.66, 295.34, 448.26, 294.52, 443.59, 282.7, 446.06, 235.14, 446.34, 230.19, 438.09, 232.39, 438.09, 221.67, 434.24, 221.12, 427.09, 219.74]], "area": 2913.1103999999987, "iscrowd": 0, "image_id": 139, "bbox": [412.8, 157.61, 53.05, 138.01], "category_id": 1, "id": 230831}, {"segmentation": [[384.98, 206.58, 384.43, 199.98, 385.25, 193.66, 385.25, 190.08, 387.18, 185.13, 387.18, 182.93, 386.08, 181.01, 385.25, 178.81, 385.25, 175.79, 388.0, 172.76, 394.88, 172.21, 398.72, 173.31, 399.27, 176.06, 399.55, 183.48, 397.9, 185.68, 395.15, 188.98, 396.8, 193.38, 398.45, 194.48, 399.0, 205.75, 395.43, 207.95, 388.83, 206.03]], "area": 435.1449499999997, "iscrowd": 0, "image_id": 139, "bbox": [384.43, 172.21, 15.12, 35.74], "category_id": 1, "id": 233201}]} # noqa
        # fmt: on

        # Test a small dataset for typical COCO format
        experiments = {"full": (detections, gt_annotations, {})}

        # Test what happens if the list of detections or ground truth annotations is empty
        experiments["empty_dt"] = ([], gt_annotations, {})
        gt = copy.deepcopy(gt_annotations)
        gt["annotations"] = []
        experiments["empty_gt"] = (detections, gt, {})

        # Test changing parameter settings
        experiments["no_categories"] = (detections, gt_annotations, {"useCats": 0})
        experiments["no_ious"] = (detections, gt_annotations, {"iouThrs": []})
        experiments["no_rec_thrs"] = (detections, gt_annotations, {"recThrs": []})
        experiments["no_max_dets"] = (detections, gt_annotations, {"maxDets": []})
        experiments["one_max_det"] = (detections, gt_annotations, {"maxDets": [1]})
        experiments["no_area"] = (detections, gt_annotations, {"areaRng": [], "areaRngLbl": []})

        # Test what happens if one omits different fields from the annotation structure
        annotation_fields = [
            "id",
            "image_id",
            "category_id",
            "score",
            "area",
            "iscrowd",
            "ignore",
            "bbox",
            "segmentation",
        ]
        for a in annotation_fields:
            gt = copy.deepcopy(gt_annotations)
            for g in gt["annotations"]:
                if a in g:
                    del g[a]
            dt = copy.deepcopy(detections)
            for d in dt:
                if a in d:
                    del d[a]
            experiments["omit_gt_" + a] = (detections, gt, {})
            experiments["omit_dt_" + a] = (dt, gt_annotations, {})

        # Compare precision/recall for original COCO PythonAPI to custom optimized one
        for name, (dt, gt, params) in experiments.items():
            # Dump to json.
            try:
                with tempfile.TemporaryDirectory() as tmpdir:
                    json_file_name = os.path.join(tmpdir, "gt_" + name + ".json")
                    with open(json_file_name, "w") as f:
                        json.dump(gt, f)
                    with contextlib.redirect_stdout(io.StringIO()):
                        coco_api = COCO(json_file_name)