Fill in index and other extra data for the provided datapoints.
(self,
data: types.JsonDict,
dataset_name: Optional[str] = None,
**unused_kw)
| 315 | return ret |
| 316 | |
| 317 | def _annotate_new_data(self, |
| 318 | data: types.JsonDict, |
| 319 | dataset_name: Optional[str] = None, |
| 320 | **unused_kw) -> list[IndexedInput]: |
| 321 | """Fill in index and other extra data for the provided datapoints.""" |
| 322 | # TODO(lit-dev): unify this with hash fn on dataset objects. |
| 323 | if dataset_name is None: |
| 324 | raise ValueError('Must provide a "dataset_name" to annotate.') |
| 325 | |
| 326 | # Generate annotated versions of new datapoints. |
| 327 | dataset = self._datasets[dataset_name] |
| 328 | input_examples = [example['data'] for example in data['inputs']] |
| 329 | dataset_to_annotate = lit_dataset.Dataset( |
| 330 | base=dataset, examples=input_examples) |
| 331 | annotated_dataset = self._run_annotators(dataset_to_annotate) |
| 332 | |
| 333 | # Add annotations and IDs to new datapoints. |
| 334 | for i, example in enumerate(data['inputs']): |
| 335 | new_id = caching.input_hash(example['data']) |
| 336 | example['data'] = dict(annotated_dataset.examples[i], _id=new_id) |
| 337 | example['id'] = new_id |
| 338 | |
| 339 | return data['inputs'] # pytype: disable=bad-return-type # always-use-return-annotations |
| 340 | |
| 341 | def _post_new_data( |
| 342 | self, |
no test coverage detected