Get the local works that needs to be restored.
(self)
| 492 | return self._restore_works_dir |
| 493 | |
| 494 | def _get_local_works(self): |
| 495 | """Get the local works that needs to be restored.""" |
| 496 | self._restore_works = [] |
| 497 | restore_work_file_dir = os.path.join( |
| 498 | self._restore_works_dir, |
| 499 | '{}_{}'.format(self._job_name, self._task_index)) |
| 500 | if gfile.IsDirectory(restore_work_file_dir): |
| 501 | restore_work_files = gfile.ListDirectory(restore_work_file_dir) |
| 502 | for work_file in restore_work_files: |
| 503 | work_file_path = os.path.join(restore_work_file_dir, work_file) |
| 504 | with gfile.GFile(work_file_path, 'r') as rfile: |
| 505 | for line in rfile.readlines(): |
| 506 | line = line.strip() |
| 507 | if not re.match(r'(.)*(?:\?start=\d+&end=\d+)$', line): |
| 508 | logging.error('Invalid format: {}'.format(line)) |
| 509 | continue |
| 510 | self._restore_works.append(line) |
| 511 | self._restore_works.sort( |
| 512 | key=lambda elm: int(re.findall(r'start=(.*)&', elm)[0])) |
| 513 | logging.info('Restore local worker works:{}'.format(self._restore_works)) |
| 514 | |
| 515 | def get_local_workqueue(self): |
| 516 | """Constructs a local work queue.""" |