Gets GPT2 tokenizer for fast approximate token counting.
(self, text_sample)
| 5795 | return self.api_key |
| 5796 | |
| 5797 | def token_counter(self, text_sample): |
| 5798 | |
| 5799 | """ Gets GPT2 tokenizer for fast approximate token counting. """ |
| 5800 | |
| 5801 | tokenizer = Utilities().get_default_tokenizer() |
| 5802 | toks = tokenizer.encode(text_sample).ids |
| 5803 | |
| 5804 | return len(toks) |
| 5805 | |
| 5806 | def prompt_engineer_chat(self, query, context, inference_dict=None): |
| 5807 |
nothing calls this directly
no test coverage detected