Computes the classification score.
(annotation_dict, prediction_dict)
| 348 | |
| 349 | |
| 350 | def score_classification(annotation_dict, prediction_dict): |
| 351 | """Computes the classification score.""" |
| 352 | match_list = [] |
| 353 | category_prediction_list = [] |
| 354 | for ex_id in list(annotation_dict.keys()): |
| 355 | annotations = annotation_dict[ex_id] |
| 356 | predict_label = (prediction_dict[ex_id].category == 'IMPOSSIBLE') |
| 357 | category_prediction_list.append(predict_label) |
| 358 | annotation_labels = [ |
| 359 | (label.category == 'IMPOSSIBLE') for label in annotations |
| 360 | ] |
| 361 | match_list.extend([(predict_label == annotation_label) |
| 362 | for annotation_label in annotation_labels]) |
| 363 | binary_agreement_score = get_avg(match_list) |
| 364 | impossible_ratio = get_avg(category_prediction_list) |
| 365 | logging.info('Impossible Category Ratio: %.2f', impossible_ratio) |
| 366 | logging.info('Annotation Match Score: %.2f', binary_agreement_score) |
| 367 | return dict( |
| 368 | impossible_ratio=impossible_ratio, |
| 369 | binary_agreement_score=binary_agreement_score) |