Return all the file name Not expanded to the other two dataset yet
(self)
| 40 | return self.candidates |
| 41 | |
| 42 | def get_data(self): |
| 43 | """ |
| 44 | Return all the file name |
| 45 | Not expanded to the other two dataset yet |
| 46 | """ |
| 47 | sample = {} |
| 48 | for filename in os.listdir(self.processed_dir): |
| 49 | if not filename.endswith(".jsonl"): |
| 50 | continue |
| 51 | sample_file = self.processed_dir + "/" + filename |
| 52 | fin = open(sample_file, "r") |
| 53 | file = [] |
| 54 | for line in fin: |
| 55 | paper = json.loads(line.strip()) |
| 56 | file.append(paper) |
| 57 | sample[filename] = file |
| 58 | return sample |
| 59 | |
| 60 | @property |
| 61 | def raw_file_names(self): |
no outgoing calls