Reset all generation state.
(&mut self)
| 496 | |
| 497 | /// Reset all generation state. |
| 498 | pub fn reset(&mut self) { |
| 499 | self.tokens.clear(); |
| 500 | self.ctx.cache.as_mut().expect("No cache specified").clear(); |
| 501 | self.index_pos = 0; |
| 502 | self.generated = 0; |
| 503 | self.prompt_len = 0; |
| 504 | |
| 505 | // Clear any stale CUDA error state left by tensor cleanup (CudaSlice drops). |
| 506 | // cudarc's error_state is an atomic that gets poisoned by internal operations |
| 507 | // (e.g. SyncOnDrop event recording, async memory frees) and causes the NEXT |
| 508 | // inference request to fail via check_err(). Clearing it here prevents the |
| 509 | // alternating success/failure pattern. |
| 510 | #[cfg(feature = "cuda")] |
| 511 | if let Device::Cuda(cuda_dev) = &self.ctx.device { |
| 512 | let _ = cuda_dev.cuda_stream().context().bind_to_thread(); |
| 513 | } |
| 514 | } |
| 515 | |
| 516 | /// Notify all remote blocks of session end (clears their KV caches). |
| 517 | pub async fn goodbye(&mut self) -> Result<()> { |