Args: always_resume (bool): If False, user-provided arguments `session_init` and `starting_epoch` will take priority. Otherwise, resume will take priority. kwargs: same as in :class:`TrainConfig`. Note: The main go
(self, always_resume=True, **kwargs)
| 186 | available to every worker, or the directories are different for different workers. |
| 187 | """ |
| 188 | def __init__(self, always_resume=True, **kwargs): |
| 189 | """ |
| 190 | Args: |
| 191 | always_resume (bool): If False, user-provided arguments |
| 192 | `session_init` and `starting_epoch` will take priority. |
| 193 | Otherwise, resume will take priority. |
| 194 | kwargs: same as in :class:`TrainConfig`. |
| 195 | |
| 196 | Note: |
| 197 | The main goal of this class is to let a training job resume |
| 198 | without changing any line of code or command line arguments. |
| 199 | So it's useful to let resume take priority over user-provided arguments sometimes. |
| 200 | |
| 201 | For example: if your training starts from a pre-trained model, |
| 202 | you would want it to use user-provided model loader at the |
| 203 | beginning, but a "resume" model loader when the job was |
| 204 | interrupted and restarted. |
| 205 | """ |
| 206 | found_sessinit = False |
| 207 | if always_resume or 'session_init' not in kwargs: |
| 208 | sessinit = self.get_sessinit_resume() |
| 209 | if sessinit is not None: |
| 210 | found_sessinit = True |
| 211 | path = sessinit.path |
| 212 | if 'session_init' in kwargs: |
| 213 | logger.info("Found checkpoint at {}. " |
| 214 | "session_init arguments will be overwritten.".format(path)) |
| 215 | else: |
| 216 | logger.info("Will load checkpoint at {}.".format(path)) |
| 217 | kwargs['session_init'] = sessinit |
| 218 | |
| 219 | found_last_epoch = False |
| 220 | if always_resume or 'starting_epoch' not in kwargs: |
| 221 | last_epoch = JSONWriter.load_existing_epoch_number() |
| 222 | if last_epoch is not None: |
| 223 | found_last_epoch = True |
| 224 | now_epoch = last_epoch + 1 |
| 225 | logger.info("Found history statistics from JSON. " |
| 226 | "Setting starting_epoch to {}.".format(now_epoch)) |
| 227 | kwargs['starting_epoch'] = now_epoch |
| 228 | assert found_sessinit == found_last_epoch, \ |
| 229 | "Found SessionInit={}, Found Last Epoch={}".format(found_sessinit, found_last_epoch) |
| 230 | |
| 231 | super(AutoResumeTrainConfig, self).__init__(**kwargs) |
| 232 | |
| 233 | @staticmethod |
| 234 | def get_sessinit_resume(dir=None): |
nothing calls this directly
no test coverage detected