(self)
| 425 | self.tokenizer = AutoTokenizer.from_pretrained(self.config.model.model_path) |
| 426 | |
| 427 | async def test_model_len(self): |
| 428 | messages = [ |
| 429 | {"role": "system", "content": "You are a helpful assistant."}, |
| 430 | {"role": "user", "content": "What's the weather like today?"}, |
| 431 | ] |
| 432 | |
| 433 | def _check_experience(exp): |
| 434 | # check prompt content and length |
| 435 | encoded_prompt = self.tokenizer.encode(exp.prompt_text, add_special_tokens=False) |
| 436 | self.assertEqual(len(encoded_prompt), exp.prompt_length) |
| 437 | self.assertLessEqual(exp.prompt_length, self.config.model.max_prompt_tokens) |
| 438 | # check response content and length |
| 439 | if exp.truncate_status == "prompt_truncated": |
| 440 | self.assertEqual( |
| 441 | exp.response_text, "[This experience is masked out due to overlong prompt]" |
| 442 | ) |
| 443 | self.assertEqual(exp.prompt_text, self.tokenizer.decode(exp.tokens[:-1])) |
| 444 | self.assertEqual(len(exp.tokens), self.config.model.max_prompt_tokens + 1) |
| 445 | self.assertEqual(exp.prompt_length, self.config.model.max_prompt_tokens) |
| 446 | self.assertTrue(torch.equal(exp.logprobs, torch.zeros(1, dtype=torch.float32))) |
| 447 | else: |
| 448 | encoded_response = self.tokenizer.encode( |
| 449 | exp.response_text, add_special_tokens=False |
| 450 | ) |
| 451 | self.assertEqual(len(encoded_response), len(exp.tokens) - exp.prompt_length) |
| 452 | self.assertLessEqual( |
| 453 | len(exp.tokens) - exp.prompt_length, self.config.model.max_response_tokens |
| 454 | ) |
| 455 | # check full sequence |
| 456 | self.assertLessEqual(len(exp.tokens), self.config.model.max_model_len) |
| 457 | |
| 458 | # For vllm engine, max_prompt_tokens and max_response_tokens work |
| 459 | response = self.model_wrapper.chat(messages) |
| 460 | self.assertEqual(len(response), 1) |
| 461 | if self.max_prompt_tokens == 5: |
| 462 | self.assertEqual(response[0].truncate_status, "prompt_truncated") |
| 463 | _check_experience(response[0]) |
| 464 | |
| 465 | exps = self.model_wrapper.extract_experience_from_history() |
| 466 | self.assertEqual(len(exps), 1) |
| 467 | _check_experience(exps[0]) |
| 468 | |
| 469 | # For openai api, max_prompt_tokens and max_response_tokens do not work |
| 470 | openai_client = self.model_wrapper.get_openai_client() |
| 471 | model_id = openai_client.models.list().data[0].id |
| 472 | with self.assertRaises(BadRequestError): |
| 473 | # the prompt is longer than max_model_len |
| 474 | openai_client.chat.completions.create(model=model_id, messages=messages, n=1) |
| 475 | exps = self.model_wrapper.extract_experience_from_history() |
| 476 | self.assertEqual(len(exps), 0) |
| 477 | |
| 478 | response = openai_client.chat.completions.create(model=model_id, messages=messages[1:], n=1) |
| 479 | self.assertEqual(len(response.choices), 1) |
| 480 | exps = self.model_wrapper.extract_experience_from_history() |
| 481 | self.assertEqual(len(exps), 1) |
| 482 | # only generate max_response_tokens tokens |
| 483 | self.assertLessEqual( |
| 484 | len(exps[0].tokens) - response.usage.prompt_tokens, |
nothing calls this directly
no test coverage detected