(self)
| 118 | return nodes2dict(recall_results) |
| 119 | |
| 120 | def load_nodes(self): |
| 121 | files = os.listdir(self.node_dir) |
| 122 | parsed_files = [] |
| 123 | max_workers = 10 |
| 124 | if max_workers == 1: |
| 125 | for file in tqdm(files): |
| 126 | input_file = os.path.join(self.node_dir, file) |
| 127 | suffix = input_file.split('.')[-1] |
| 128 | if suffix != 'node': |
| 129 | continue |
| 130 | nodes = nodefile2node(input_file) |
| 131 | parsed_files.extend(nodes) |
| 132 | else: |
| 133 | def parse_file(file,node_dir): |
| 134 | input_file = os.path.join(node_dir, file) |
| 135 | suffix = input_file.split('.')[-1] |
| 136 | if suffix != 'node': |
| 137 | return [] |
| 138 | return nodefile2node(input_file) |
| 139 | with ThreadPoolExecutor(max_workers=max_workers) as executor: |
| 140 | # results = list(tqdm(executor.map(parse_file, files, self.node_dir), total=len(files))) |
| 141 | results = list(tqdm(executor.map(parse_file, files, [self.node_dir]*len(files)), total=len(files))) |
| 142 | for result in results: |
| 143 | parsed_files.extend(result) |
| 144 | return parsed_files |
| 145 | |
| 146 | def load_query_engine(self): |
| 147 | print('Loading nodes...') |
no test coverage detected