| 162 | |
| 163 | class FalconVal(Dataset): |
| 164 | def __init__(self, data_meta_path, data_root, tokenizer_path, max_words=None): |
| 165 | |
| 166 | with open(data_meta_path, 'r') as f: |
| 167 | filenames = json.load(f) |
| 168 | filenames = [f"{data_root}/{_}" for _ in filenames] |
| 169 | |
| 170 | filenames = [_.replace('.parquet', '.pkl') for _ in filenames] |
| 171 | |
| 172 | filename = filenames[-1] |
| 173 | print(f"Falcon val filename: {filename}") |
| 174 | with open(filename, 'rb') as f: |
| 175 | ann = pickle.load(f) |
| 176 | self.contents = ann |
| 177 | |
| 178 | self.tokenizer = Tokenizer(model_path=tokenizer_path) |
| 179 | |
| 180 | def __len__(self): |
| 181 | return len(self.contents) |