| 97 | |
| 98 | |
| 99 | def _drop_duplicates_in_input(untokenized_dataset): |
| 100 | # from scrolls/evaluator/dataset_evaluator.py |
| 101 | |
| 102 | indices_to_keep = [] |
| 103 | id_to_idx = {} |
| 104 | outputs = [] |
| 105 | for i, (id_, output) in enumerate(zip(untokenized_dataset["id"], untokenized_dataset["output"])): |
| 106 | if id_ in id_to_idx: |
| 107 | outputs[id_to_idx[id_]].append(output) |
| 108 | continue |
| 109 | indices_to_keep.append(i) |
| 110 | id_to_idx[id_] = len(outputs) |
| 111 | outputs.append([output]) |
| 112 | untokenized_dataset = untokenized_dataset.select(indices_to_keep).flatten_indices() |
| 113 | untokenized_dataset = untokenized_dataset.remove_columns("output") |
| 114 | untokenized_dataset = untokenized_dataset.add_column("outputs", outputs) |
| 115 | return untokenized_dataset |
| 116 | |
| 117 | |
| 118 | def _num_cpu_cores(): |