Tokenize a prompt string and set up token state for generation.
(&mut self, dialog: &str)
| 369 | |
| 370 | /// Tokenize a prompt string and set up token state for generation. |
| 371 | pub fn prepare_prompt(&mut self, dialog: &str) -> Result<()> { |
| 372 | // make sure we start clean |
| 373 | self.tokens.clear(); |
| 374 | self.ctx.cache.as_mut().expect("No cache specified").clear(); |
| 375 | self.index_pos = 0; |
| 376 | |
| 377 | log::debug!("dialog={}", dialog); |
| 378 | |
| 379 | // tokenize raw |
| 380 | self.tokens = self |
| 381 | .tokenizer |
| 382 | .encode(dialog, false) // do not add special tokens as we already added them |
| 383 | .map_err(anyhow::Error::msg)? |
| 384 | .get_ids() |
| 385 | .to_vec(); |
| 386 | |
| 387 | log::debug!("encoded={:?}", &self.tokens); |
| 388 | log::debug!("history tokens: {}", self.tokens.len()); |
| 389 | |
| 390 | // Track prompt length for repeat penalty scoping |
| 391 | self.prompt_len = self.tokens.len(); |
| 392 | |
| 393 | Ok(()) |
| 394 | } |
| 395 | |
| 396 | /// Generate the next token. Assumes `prepare_prompt()` has been called for the first token. |
| 397 | pub async fn next_token(&mut self, index: usize) -> Result<Token> { |
no test coverage detected