Process annotation dictionary to generate references. Not all annotations will be a reference set to which prediction will be evaluated against. The median annotation will reserved to approximate human performance. In case there's only one reference annotation, the `allow_single_annotations
(annotation_dict, allow_single_annotations=False)
| 198 | |
| 199 | |
| 200 | def process_annotation(annotation_dict, allow_single_annotations=False): |
| 201 | """Process annotation dictionary to generate references. |
| 202 | |
| 203 | Not all annotations will be a reference set to which prediction will be |
| 204 | evaluated against. The median annotation will reserved to approximate human |
| 205 | performance. In case there's only one reference annotation, the |
| 206 | `allow_single_annotations` argument determines whether it is reserved. |
| 207 | |
| 208 | The output dictionaries will only contain examples where at least half the |
| 209 | annotators found decontextualization is feasible. |
| 210 | |
| 211 | Args: |
| 212 | annotation_dict: dictionary with a list of decontext labels. |
| 213 | allow_single_annotations: bool, when there's only a single annotation, |
| 214 | whether to pass that single annotation to the reference_sents_dict. |
| 215 | |
| 216 | Returns: |
| 217 | For all dictionaries, key is the example id. |
| 218 | |
| 219 | original_sent_dict: value is the original sentence. |
| 220 | reference_sents_dict: value is a list of reference decontextualized |
| 221 | sentences. |
| 222 | median_annotation_dict: key is an example id, |
| 223 | value is the median length human annotation, |
| 224 | considered to approximate human references. |
| 225 | |
| 226 | """ |
| 227 | reference_sents_dict = {} |
| 228 | median_annotation_dict = {} |
| 229 | original_sent_dict = {} |
| 230 | |
| 231 | for ex_id in list(annotation_dict.keys()): |
| 232 | annotations = annotation_dict[ex_id] |
| 233 | if not annotations: |
| 234 | raise ValueError('Empty annotations encountered while processing ' |
| 235 | 'reference annotations.') |
| 236 | not_impossible = [ |
| 237 | get_sent(label) |
| 238 | for label in annotations |
| 239 | if label.category != 'IMPOSSIBLE' |
| 240 | ] |
| 241 | not_impossible = sorted(not_impossible, key=len) |
| 242 | if 2 * len(not_impossible) < len(annotations): |
| 243 | # Skip the example if the majority of annotators marked it as IMPOSSIBLE. |
| 244 | continue |
| 245 | median_val = (len(not_impossible) - 1) // 2 |
| 246 | if len(not_impossible) == 1 and allow_single_annotations: |
| 247 | median_annotation_dict[ex_id] = not_impossible[median_val] |
| 248 | else: |
| 249 | median_annotation_dict[ex_id] = not_impossible.pop(median_val) |
| 250 | reference_sents_dict[ex_id] = not_impossible |
| 251 | original_sent_dict[ex_id] = annotations[0].original_sentence |
| 252 | logging.info('%d examples needs to be decontextualized out of %d', |
| 253 | len(median_annotation_dict), len(annotation_dict)) |
| 254 | return original_sent_dict, reference_sents_dict, median_annotation_dict |
| 255 | |
| 256 | |
| 257 | def compute_sentence_generation_scores(original_sent_dict, |