(index)
| 70 | return lengths |
| 71 | |
| 72 | def get_data(index): |
| 73 | item = global_torch_dataset[index] |
| 74 | length = len(global_torch_dataset[index]["input_ids"]) |
| 75 | return item, index, length |
| 76 | |
| 77 | def torch_dataset_to_hf_dataset(torch_dataset, num_worker=16): |
| 78 | global global_torch_dataset |
nothing calls this directly
no outgoing calls
no test coverage detected