| 44 | |
| 45 | |
| 46 | def load_index(self, input_dir): |
| 47 | # meta data |
| 48 | with open(os.path.join(input_dir, f"{self.embed_model_name_file}_meta_data.json"), "r") as f: |
| 49 | self.meta_data = json.load(f) |
| 50 | assert self.meta_data["model_name"] == self.embed_model_name |
| 51 | print(f"loading model {self.embed_model_name}; loading index from {input_dir}; loading index length {self.meta_data['num_vectors']}...") |
| 52 | |
| 53 | self.index = faiss.read_index(os.path.join(input_dir, f"{self.embed_model_name_file}_faiss_index.bin")) |
| 54 | with open(os.path.join(input_dir, f"{self.embed_model_name_file}_file_data_list.jsonl"), "r") as f: |
| 55 | self.file_data_list = [json.loads(line) for line in f] |
| 56 | assert self.meta_data["num_vectors"] == self.index.ntotal \ |
| 57 | and len(self.file_data_list) == self.index.ntotal, \ |
| 58 | f"meta_data_num_vectors: {self.meta_data['num_vectors']} and index_ntotal: {self.index.ntotal} mismatch \n or file_data_list_length: {len(self.file_data_list)} and index_ntotal: {self.index.ntotal} mismatch" |
| 59 | |
| 60 | def load_multi_index_corpus_together(self, input_dir_list): |
| 61 | for input_dir in input_dir_list: |