| 165 | |
| 166 | class FalconVal(Dataset): |
| 167 | def __init__(self, data_meta_path, data_root, tokenizer_path, max_words=1024): |
| 168 | |
| 169 | with open(data_meta_path, 'r') as f: |
| 170 | filenames = json.load(f) |
| 171 | filenames = [f"{data_root}/{_}" for _ in filenames] |
| 172 | |
| 173 | |
| 174 | |
| 175 | filename = filenames[-1] |
| 176 | print(f"Falcon val filename: {filename}") |
| 177 | |
| 178 | ann = pds.read_parquet(filename)['content'] |
| 179 | self.contents = ann.tolist() |
| 180 | |
| 181 | self.max_words = max_words |
| 182 | self.tokenizer = Tokenizer(model_path=tokenizer_path) |
| 183 | |
| 184 | def __len__(self): |
| 185 | return len(self.contents) |