(self, usage_counter: ModelUsageCounter = None)
| 56 | self.tmp_add_progress.extend(idxs) |
| 57 | |
| 58 | def load(self, usage_counter: ModelUsageCounter = None) -> Iterable[Any]: |
| 59 | # save the original total from __init__ before loading |
| 60 | original_total = self.usage["total"] |
| 61 | |
| 62 | # load progress |
| 63 | try: |
| 64 | usage: dict = load_json(self.usage_path) |
| 65 | for k, v in self.usage.items(): |
| 66 | self.usage[k] = usage.get(k, v) |
| 67 | self.detail_progress = np.array(usage.get("detail_progress", []), dtype=bool) |
| 68 | self.detail_completed = sum(self.detail_progress) |
| 69 | if self.detail_completed != self.usage["completed"]: |
| 70 | raise Exception(f"Error occurred when load buffer: completed={self.usage['completed']} is not equal to completed in detail_progress={self.detail_completed}") |
| 71 | self.tmp_add_progress = [] |
| 72 | |
| 73 | if usage_counter: |
| 74 | usage_counter.load_from_dict(usage) |
| 75 | except: |
| 76 | pass |
| 77 | |
| 78 | # resize detail_progress to match the current total if different from loaded total |
| 79 | if original_total != self.usage["total"]: |
| 80 | self.resize_total(original_total) |
| 81 | # load results |
| 82 | results: Iterable[Any] = [] |
| 83 | try: |
| 84 | results: Iterable[Any] = load_json(self.result_path) |
| 85 | except: |
| 86 | pass |
| 87 | |
| 88 | return results |
| 89 | |
| 90 | def save(self, results: Iterable[Any], usage_counter: ModelUsageCounter = None): |
| 91 | with self._lock: |
no test coverage detected