(self, data_path, dataset, index_file=".index.json")
| 29 | |
| 30 | class TokenExtender: |
| 31 | def __init__(self, data_path, dataset, index_file=".index.json"): |
| 32 | self.data_path = data_path |
| 33 | self.dataset = dataset |
| 34 | self.index_file = index_file |
| 35 | self.indices = None |
| 36 | self.new_tokens = None |
| 37 | |
| 38 | def _load_data(self): |
| 39 | with open(os.path.join(self.data_path, self.dataset + self.index_file), 'r') as f: |
nothing calls this directly
no outgoing calls
no test coverage detected