| 46 | pass |
| 47 | |
| 48 | def run(self, ctx: Context, sync_point_interval=100): |
| 49 | with Session(ctx.engine) as session: |
| 50 | subtask_run = self._start_subtask(session, ctx.connection.id) |
| 51 | if ctx.incremental: |
| 52 | state = self._get_last_state(session, ctx.connection.id) |
| 53 | else: |
| 54 | self.delete(session, ctx) |
| 55 | state = dict() |
| 56 | |
| 57 | try: |
| 58 | records = self.fetch(state, session, ctx) |
| 59 | progress = last_progress = 0 |
| 60 | for data, state in records: |
| 61 | progress += 1 |
| 62 | self.process(data, session, ctx) |
| 63 | if progress % sync_point_interval == 0: |
| 64 | # Save current state |
| 65 | subtask_run.state = json.dumps(state) |
| 66 | session.merge(subtask_run) |
| 67 | session.commit() |
| 68 | # Send progress |
| 69 | yield RemoteProgress( |
| 70 | increment=sync_point_interval, |
| 71 | current=progress |
| 72 | ) |
| 73 | last_progress = progress |
| 74 | # Send final progress |
| 75 | if progress != last_progress: |
| 76 | yield RemoteProgress( |
| 77 | increment=progress - last_progress, |
| 78 | current=progress |
| 79 | ) |
| 80 | except Exception as e: |
| 81 | logger.error(f'{type(e).__name__}: {e}') |
| 82 | raise e |
| 83 | |
| 84 | subtask_run.state = json.dumps(state) |
| 85 | subtask_run.completed = datetime.now() |
| 86 | session.merge(subtask_run) |
| 87 | session.commit() |
| 88 | |
| 89 | def _start_subtask(self, session, connection_id): |
| 90 | subtask_run = SubtaskRun( |