()
| 55 | |
| 56 | |
| 57 | def test_multi_message(): |
| 58 | pre = get_preprocessor() |
| 59 | messages = [ |
| 60 | [" turn11", " turn12", " turn13", " turn14"], |
| 61 | [" turn21", " turn22"] |
| 62 | ] |
| 63 | batch = pre([DUMMY_IMAGE], messages) |
| 64 | out = pre.tokenizer.decode(batch["target_tokens"], truncate_at_eos=False) |
| 65 | out = postprocess_prompt(out) |
| 66 | assert out == "IMAGE[144] turn11 turn12 turn13 turn14 turn21 turn22" |
| 67 | |
| 68 | seg = batch["subsegment_ids"] |
| 69 | tokens = batch["input_tokens"] |
| 70 | |
| 71 | seg1 = pre.tokenizer.decode(tokens[seg != seg[-1]], truncate_at_eos=False) |
| 72 | assert postprocess_prompt(seg1) == "IMAGE[144] turn11 turn12 turn13 turn14" |
| 73 | |
| 74 | seg2 = pre.tokenizer.decode(tokens[seg >= seg[-1]], truncate_at_eos=False) |
| 75 | assert postprocess_prompt(seg2) == "IMAGE[144] turn21 turn22" |
nothing calls this directly
no test coverage detected