Enqueue a dataframe for offline write, grouped by push source + flags. Cheap and non-blocking; heavy I/O happens in background thread.
(
self,
push_source_name: str,
df: pd.DataFrame,
allow_registry_cache: bool,
transform_on_write: bool,
)
| 1002 | # ---------- Public API ---------- |
| 1003 | |
| 1004 | def enqueue( |
| 1005 | self, |
| 1006 | push_source_name: str, |
| 1007 | df: pd.DataFrame, |
| 1008 | allow_registry_cache: bool, |
| 1009 | transform_on_write: bool, |
| 1010 | ) -> None: |
| 1011 | """ |
| 1012 | Enqueue a dataframe for offline write, grouped by push source + flags. |
| 1013 | Cheap and non-blocking; heavy I/O happens in background thread. |
| 1014 | """ |
| 1015 | key = _OfflineBatchKey( |
| 1016 | push_source_name=push_source_name, |
| 1017 | allow_registry_cache=allow_registry_cache, |
| 1018 | transform_on_write=transform_on_write, |
| 1019 | ) |
| 1020 | |
| 1021 | with self._lock: |
| 1022 | self._buffers[key].append(df) |
| 1023 | total_rows = sum(len(d) for d in self._buffers[key]) |
| 1024 | should_flush = total_rows >= self._cfg.batch_size |
| 1025 | |
| 1026 | if should_flush: |
| 1027 | # Size-based flush |
| 1028 | logger.debug( |
| 1029 | "OfflineWriteBatcher size threshold reached for %s: %s rows", |
| 1030 | key, |
| 1031 | total_rows, |
| 1032 | ) |
| 1033 | self._flush(key) |
| 1034 | |
| 1035 | def flush_all(self) -> None: |
| 1036 | """ |
no test coverage detected