(self, dim=None)
| 75 | return self.n |
| 76 | |
| 77 | def save(self, dim=None): |
| 78 | print("converting to numpy") |
| 79 | for key in tqdm(list(self.index_doc_id.keys())): |
| 80 | self.index_doc_id[key] = np.array(self.index_doc_id[key], dtype=np.int32) |
| 81 | self.index_doc_value[key] = np.array(self.index_doc_value[key], dtype=np.float32) |
| 82 | print("save to disk") |
| 83 | with h5py.File(self.filename, "w") as f: |
| 84 | if dim: |
| 85 | f.create_dataset("dim", data=int(dim)) |
| 86 | else: |
| 87 | f.create_dataset("dim", data=len(self.index_doc_id.keys())) |
| 88 | for key in tqdm(self.index_doc_id.keys()): |
| 89 | f.create_dataset("index_doc_id_{}".format(key), data=self.index_doc_id[key]) |
| 90 | f.create_dataset("index_doc_value_{}".format(key), data=self.index_doc_value[key]) |
| 91 | f.close() |
| 92 | print("saving index distribution...") # => size of each posting list in a dict |
| 93 | index_dist = {} |
| 94 | for k, v in self.index_doc_id.items(): |
| 95 | index_dist[int(k)] = len(v) |
| 96 | json.dump(index_dist, open(os.path.join(self.index_path, "index_dist.json"), "w")) |
| 97 | |
| 98 | |
| 99 |
no test coverage detected