gt_roots: a list of paths to datasets, i.e., [path_to_DatasetA, path_to_DatasetB, ...] mask_roots: same as above, but the .png are masks predicted by the model strict: when True, all videos in the dataset must have corresponding predictions. Setting it to False is useful in
(
gt_roots,
mask_roots,
strict=True,
num_processes=None,
*,
verbose=True,
skip_first_and_last=True,
)
| 314 | |
| 315 | |
| 316 | def benchmark( |
| 317 | gt_roots, |
| 318 | mask_roots, |
| 319 | strict=True, |
| 320 | num_processes=None, |
| 321 | *, |
| 322 | verbose=True, |
| 323 | skip_first_and_last=True, |
| 324 | ): |
| 325 | """ |
| 326 | gt_roots: a list of paths to datasets, i.e., [path_to_DatasetA, path_to_DatasetB, ...] |
| 327 | mask_roots: same as above, but the .png are masks predicted by the model |
| 328 | strict: when True, all videos in the dataset must have corresponding predictions. |
| 329 | Setting it to False is useful in cases where the ground-truth contains both train/val |
| 330 | sets, but the model only predicts the val subset. |
| 331 | Either way, if a video is predicted (i.e., the corresponding folder exists), |
| 332 | then it must at least contain all the masks in the ground truth annotations. |
| 333 | Masks that are in the prediction but not in the ground-truth |
| 334 | (i.e., sparse annotations) are ignored. |
| 335 | skip_first_and_last: whether we should skip the first and the last frame in evaluation. |
| 336 | This is used by DAVIS 2017 in their semi-supervised evaluation. |
| 337 | It should be disabled for unsupervised evaluation. |
| 338 | """ |
| 339 | |
| 340 | assert len(gt_roots) == len(mask_roots) |
| 341 | single_dataset = len(gt_roots) == 1 |
| 342 | |
| 343 | if verbose: |
| 344 | if skip_first_and_last: |
| 345 | print( |
| 346 | "We are *SKIPPING* the evaluation of the first and the last frame (standard for semi-supervised video object segmentation)." |
| 347 | ) |
| 348 | else: |
| 349 | print( |
| 350 | "We are *NOT SKIPPING* the evaluation of the first and the last frame (*NOT STANDARD* for semi-supervised video object segmentation)." |
| 351 | ) |
| 352 | |
| 353 | pool = Pool(num_processes) |
| 354 | start = time.time() |
| 355 | to_wait = [] |
| 356 | for gt_root, mask_root in zip(gt_roots, mask_roots): |
| 357 | # Validate folders |
| 358 | validated = True |
| 359 | gt_videos = os.listdir(gt_root) |
| 360 | mask_videos = os.listdir(mask_root) |
| 361 | |
| 362 | # if the user passed the root directory instead of Annotations |
| 363 | if len(gt_videos) != len(mask_videos): |
| 364 | if "Annotations" in gt_videos: |
| 365 | if ".png" not in os.listdir(path.join(gt_root, "Annotations"))[0]: |
| 366 | gt_root = path.join(gt_root, "Annotations") |
| 367 | gt_videos = os.listdir(gt_root) |
| 368 | |
| 369 | # remove non-folder items |
| 370 | gt_videos = list(filter(lambda x: path.isdir(path.join(gt_root, x)), gt_videos)) |
| 371 | mask_videos = list( |
| 372 | filter(lambda x: path.isdir(path.join(mask_root, x)), mask_videos) |
| 373 | ) |
no test coverage detected