| 8 | |
| 9 | |
| 10 | class TaskBuffer: |
| 11 | def __init__(self, |
| 12 | total: int, |
| 13 | save_dir: str, |
| 14 | save_step: int = 4 |
| 15 | ) -> None: |
| 16 | """ |
| 17 | Initialize the buffer. |
| 18 | Args: |
| 19 | total: The total number of samples to be processed (completed + remain). |
| 20 | save_dir: The directory for saving temporary results and the progress. |
| 21 | save_step: The number of processing steps between each saving. |
| 22 | """ |
| 23 | self.save_step = save_step |
| 24 | self.save_dir = save_dir |
| 25 | self.usage_path = os.path.join(self.save_dir, "usage.json") |
| 26 | self.result_path = os.path.join(self.save_dir, "result.json") |
| 27 | |
| 28 | self.usage = { |
| 29 | "total": total, |
| 30 | "completed": 0, |
| 31 | "token": 0, |
| 32 | "time": 0.0, |
| 33 | } |
| 34 | self.detail_progress: np.ndarray[bool] = np.zeros(total, dtype=bool) |
| 35 | self.detail_completed: int = 0 |
| 36 | self.tmp_add_progress: List[int] = [] |
| 37 | |
| 38 | self._lock = threading.RLock() |
| 39 | |
| 40 | def resize_total(self, total: int): |
| 41 | if total > self.usage["total"]: |
| 42 | self.detail_progress: np.ndarray[bool] = np.concatenate([self.detail_progress, np.zeros(total - self.usage["total"], dtype=bool)]) |
| 43 | elif total < self.usage["total"]: |
| 44 | self.detail_progress: np.ndarray[bool] = self.detail_progress[ : total] |
| 45 | self.usage["total"] = total |
| 46 | |
| 47 | def add_progress(self, idxs: List[int]): |
| 48 | """ |
| 49 | Add current progress to buffer |
| 50 | Args: |
| 51 | idxs: The index of processed sample added after the last save. |
| 52 | """ |
| 53 | with self._lock: |
| 54 | if max(idxs) >= self.usage["total"]: |
| 55 | raise Exception(f"Error occurred when add progress to buffer: {idxs} to be added >= total={self.usage['total']}") |
| 56 | self.tmp_add_progress.extend(idxs) |
| 57 | |
| 58 | def load(self, usage_counter: ModelUsageCounter = None) -> Iterable[Any]: |
| 59 | # save the original total from __init__ before loading |
| 60 | original_total = self.usage["total"] |
| 61 | |
| 62 | # load progress |
| 63 | try: |
| 64 | usage: dict = load_json(self.usage_path) |
| 65 | for k, v in self.usage.items(): |
| 66 | self.usage[k] = usage.get(k, v) |
| 67 | self.detail_progress = np.array(usage.get("detail_progress", []), dtype=bool) |