(model_name, inp)
| 229 | |
| 230 | |
| 231 | def process_input(model_name, inp): |
| 232 | if isinstance(inp, str): |
| 233 | inp = [inp] |
| 234 | elif isinstance(inp, list): |
| 235 | if isinstance(inp[0], int): |
| 236 | try: |
| 237 | decoding = tiktoken.model.encoding_for_model(model_name) |
| 238 | except KeyError: |
| 239 | logger.warning("Warning: model not found. Using cl100k_base encoding.") |
| 240 | model = "cl100k_base" |
| 241 | decoding = tiktoken.get_encoding(model) |
| 242 | inp = [decoding.decode(inp)] |
| 243 | elif isinstance(inp[0], list): |
| 244 | try: |
| 245 | decoding = tiktoken.model.encoding_for_model(model_name) |
| 246 | except KeyError: |
| 247 | logger.warning("Warning: model not found. Using cl100k_base encoding.") |
| 248 | model = "cl100k_base" |
| 249 | decoding = tiktoken.get_encoding(model) |
| 250 | inp = [decoding.decode(text) for text in inp] |
| 251 | |
| 252 | return inp |
| 253 | |
| 254 | |
| 255 | def create_openai_logprobs(logprob_dict): |
no outgoing calls
no test coverage detected
searching dependent graphs…