(X, sep)
| 56 | def tokenizer_image_token(prompt, tokenizer, image_token_index=IMAGE_TOKEN_INDEX, return_tensors=None): |
| 57 | prompt_chunks = [tokenizer(chunk).input_ids for chunk in prompt.split('<image>')] |
| 58 | def insert_separator(X, sep): |
| 59 | return [ele for sublist in zip(X, [sep]*len(X)) for ele in sublist][:-1] |
| 60 | input_ids = [] |
| 61 | offset = 0 |
| 62 | if len(prompt_chunks) > 0 and len(prompt_chunks[0]) > 0 and prompt_chunks[0][0] == tokenizer.bos_token_id: |
no outgoing calls
no test coverage detected