Background loop: periodically checks for buffers that should be flushed based on time since last flush.
(self)
| 1061 | # ---------- Internal helpers ---------- |
| 1062 | |
| 1063 | def _run(self) -> None: |
| 1064 | """ |
| 1065 | Background loop: periodically checks for buffers that should be flushed |
| 1066 | based on time since last flush. |
| 1067 | """ |
| 1068 | interval = max(1, int(getattr(self._cfg, "batch_interval_seconds", 30))) |
| 1069 | logger.debug( |
| 1070 | "OfflineWriteBatcher background loop started with check interval=%s", |
| 1071 | interval, |
| 1072 | ) |
| 1073 | |
| 1074 | while not self._stop_event.wait(timeout=interval): |
| 1075 | now = time.time() |
| 1076 | try: |
| 1077 | with self._lock: |
| 1078 | keys_to_flush: List[_OfflineBatchKey] = [] |
| 1079 | for key, dfs in list(self._buffers.items()): |
| 1080 | if not dfs: |
| 1081 | continue |
| 1082 | last = self._last_flush[ |
| 1083 | key |
| 1084 | ] # this will also init the default timestamp |
| 1085 | age = now - last |
| 1086 | if age >= self._cfg.batch_interval_seconds: |
| 1087 | logger.debug( |
| 1088 | "OfflineWriteBatcher time threshold reached for %s: age=%s", |
| 1089 | key, |
| 1090 | age, |
| 1091 | ) |
| 1092 | keys_to_flush.append(key) |
| 1093 | for key in keys_to_flush: |
| 1094 | self._flush(key) |
| 1095 | except Exception: |
| 1096 | logger.exception("Error in OfflineWriteBatcher background loop") |
| 1097 | |
| 1098 | logger.debug("OfflineWriteBatcher background loop exiting") |
| 1099 | |
| 1100 | def _drain_locked(self, key: _OfflineBatchKey) -> Optional[List[pd.DataFrame]]: |
| 1101 | """ |