Gets GPT2 tokenizer for fast approximate token counting.
(self, text_sample)
| 6882 | return self.api_key |
| 6883 | |
| 6884 | def token_counter(self, text_sample): |
| 6885 | |
| 6886 | """ Gets GPT2 tokenizer for fast approximate token counting. """ |
| 6887 | |
| 6888 | tokenizer = Utilities().get_default_tokenizer() |
| 6889 | toks = tokenizer.encode(text_sample).ids |
| 6890 | return len(toks) |
| 6891 | |
| 6892 | def prompt_engineer(self, query, context, inference_dict=None): |
| 6893 |
nothing calls this directly
no test coverage detected