Re-queue a batch of events that failed to process. Events are pushed back to the **front** of the channel's queue so they are processed first on the next flush cycle. This prevents event loss when session creation or `session/prompt` fails transiently. Original `received_at` timestamps are preserved so the channel retains its fairness position. The retry delay comes from exponential backoff, not
(&mut self, batch: FlushBatch)
| 383 | /// Note: does NOT remove from `in_flight_channels` — caller must call |
| 384 | /// `mark_complete` separately. |
| 385 | pub fn requeue(&mut self, batch: FlushBatch) -> Option<FlushBatch> { |
| 386 | let channel_id = batch.channel_id; |
| 387 | let attempt = { |
| 388 | let count = self.retry_counts.entry(channel_id).or_insert(0); |
| 389 | *count += 1; |
| 390 | *count |
| 391 | }; |
| 392 | |
| 393 | if attempt > MAX_RETRIES { |
| 394 | tracing::error!( |
| 395 | channel_id = %channel_id, |
| 396 | attempt, |
| 397 | events = batch.events.len(), |
| 398 | "dead-lettering batch after {} retries — discarding {} events", |
| 399 | MAX_RETRIES, |
| 400 | batch.events.len(), |
| 401 | ); |
| 402 | self.retry_counts.remove(&channel_id); |
| 403 | // Also clear retry_after so fresh traffic on this channel isn't |
| 404 | // throttled by stale backoff from the discarded poison batch. |
| 405 | self.retry_after.remove(&channel_id); |
| 406 | return Some(batch); |
| 407 | } |
| 408 | |
| 409 | // Exponential backoff: BASE * 2^(attempt-1), capped at MAX, with ±20% jitter. |
| 410 | let base_secs = BASE_RETRY_DELAY_SECS.saturating_mul(1u64 << (attempt - 1).min(6)); |
| 411 | let capped_secs = base_secs.min(MAX_RETRY_DELAY_SECS); |
| 412 | // Jitter: multiply by 0.8..1.2 using subsecond nanos as entropy source. |
| 413 | let jitter = { |
| 414 | let nanos = std::time::SystemTime::now() |
| 415 | .duration_since(std::time::UNIX_EPOCH) |
| 416 | .unwrap_or_default() |
| 417 | .subsec_nanos(); |
| 418 | 0.8 + (nanos as f64 / u32::MAX as f64) * 0.4 |
| 419 | }; |
| 420 | let delay = Duration::from_secs_f64(capped_secs as f64 * jitter); |
| 421 | |
| 422 | tracing::warn!( |
| 423 | channel_id = %channel_id, |
| 424 | attempt, |
| 425 | max = MAX_RETRIES, |
| 426 | delay_secs = delay.as_secs_f64(), |
| 427 | events = batch.events.len(), |
| 428 | "requeueing failed batch with backoff" |
| 429 | ); |
| 430 | |
| 431 | let queue = self.queues.entry(channel_id).or_default(); |
| 432 | // Push to front in reverse order so original order is preserved. |
| 433 | for be in batch.events.into_iter().rev() { |
| 434 | queue.push_front(QueuedEvent { |
| 435 | channel_id, |
| 436 | event: be.event, |
| 437 | prompt_tag: be.prompt_tag, |
| 438 | received_at: be.received_at, // preserve original timestamp (#46) |
| 439 | }); |
| 440 | } |
| 441 | // Enforce per-channel cap: trim oldest (back) events if requeue pushed |
| 442 | // the queue over the limit. Without this, repeated requeue+push cycles |