Fit the estimator using the training data. Train the estimator for `self._train_steps` steps, after waiting for `delay_secs` seconds. If `self._train_steps` is `None`, train forever. Args: delay_secs: Start training after this many seconds. Returns: The trained estimat
(self, delay_secs=None)
| 328 | return old_export_strategies |
| 329 | |
| 330 | def train(self, delay_secs=None): |
| 331 | """Fit the estimator using the training data. |
| 332 | |
| 333 | Train the estimator for `self._train_steps` steps, after waiting for |
| 334 | `delay_secs` seconds. If `self._train_steps` is `None`, train forever. |
| 335 | |
| 336 | Args: |
| 337 | delay_secs: Start training after this many seconds. |
| 338 | |
| 339 | Returns: |
| 340 | The trained estimator. |
| 341 | """ |
| 342 | start = time.time() |
| 343 | |
| 344 | # Start the server, if needed. It's important to start the server before |
| 345 | # we (optionally) sleep for the case where no device_filters are set. |
| 346 | # Otherwise, the servers will wait to connect to each other before starting |
| 347 | # to train. We might as well start as soon as we can. |
| 348 | config = self._estimator.config |
| 349 | if isinstance(config, run_config.RunConfig): |
| 350 | if (config.cluster_spec and config.master and |
| 351 | config.environment == run_config.Environment.LOCAL): |
| 352 | logging.warn("ClusterSpec and master are provided, but environment is " |
| 353 | "set to 'local'. Set environment to 'cloud' if you intend " |
| 354 | "to use the distributed runtime.") |
| 355 | if (config.environment != run_config.Environment.LOCAL and |
| 356 | config.environment != run_config.Environment.GOOGLE and |
| 357 | config.cluster_spec and config.master): |
| 358 | self._start_server() |
| 359 | elif config.cluster_spec and config.master: |
| 360 | raise ValueError( |
| 361 | "For distributed runtime, Experiment class only works with " |
| 362 | "tf.contrib.learn.RunConfig for now, but provided {}".format( |
| 363 | type(config))) |
| 364 | |
| 365 | extra_hooks = [] |
| 366 | if delay_secs is None: |
| 367 | task_id = self._estimator.config.task_id or 0 |
| 368 | if self._delay_workers_by_global_step: |
| 369 | # Wait 5500 global steps for the second worker. Each worker waits more |
| 370 | # then previous one but with a diminishing number of steps. |
| 371 | extra_hooks.append( |
| 372 | basic_session_run_hooks.GlobalStepWaiterHook( |
| 373 | int(8000.0 * math.log(task_id + 1)))) |
| 374 | delay_secs = 0 |
| 375 | else: |
| 376 | # Wait 5 secs more for each new worker up to 60 secs. |
| 377 | delay_secs = min(60, task_id * 5) |
| 378 | |
| 379 | if delay_secs > 0: |
| 380 | elapsed_secs = time.time() - start |
| 381 | remaining = delay_secs - elapsed_secs |
| 382 | logging.info("Waiting %d secs before starting training.", remaining) |
| 383 | time.sleep(delay_secs) |
| 384 | |
| 385 | return self._call_train( |
| 386 | input_fn=self._train_input_fn, |
| 387 | max_steps=self._train_steps, |