| 173 | mox.file.make_dirs(self.bucket) |
| 174 | |
| 175 | def step_end(self, run_context): |
| 176 | cb_params = run_context.original_args() |
| 177 | cur_step = cb_params.cur_step_num + self.has_trained_step |
| 178 | if cur_step % self.syn_times == 0: |
| 179 | print("Copying checkpoint to the buckets start", flush=True) |
| 180 | self.syn_files() |
| 181 | print("Copying checkpoint to the buckets ends", flush=True) |
| 182 | |
| 183 | def syn_files(self): |
| 184 | process = Process(target=mox.file.copy_parallel, args=(self.cache_dir, self.bucket), name="checkpoint_sync") |