(messages, n_at_start=None, preprocessor=None)
| 14 | |
| 15 | |
| 16 | def _test_tokenization(messages, n_at_start=None, preprocessor=None): |
| 17 | if preprocessor is None: |
| 18 | preprocessor = get_preprocessor() |
| 19 | if n_at_start is None: |
| 20 | n = " ".join(messages).count(IMAGE_PROMPT) |
| 21 | else: |
| 22 | n = n_at_start |
| 23 | batch = preprocessor([DUMMY_IMAGE]*n, messages) |
| 24 | with_tokens = preprocessor.tokenizer.decode(batch["target_tokens"], truncate_at_eos=False) |
| 25 | out = postprocess_prompt(with_tokens) |
| 26 | if n_at_start is None: |
| 27 | expected = "".join(messages).replace(IMAGE_PROMPT, "IMAGE[144]") |
| 28 | else: |
| 29 | expected = "".join(["IMAGE[144]"]*n + messages) |
| 30 | assert out == expected, f"Expected \"{expected}\", but got \"{out}\"" |
| 31 | |
| 32 | |
| 33 | def test_text_only(): |
no test coverage detected