| 91 | // larger than the kept head (the spill file covers the dropped middle). |
| 92 | if ToolTruncTail <= ToolTruncHead { |
| 93 | t.Fatalf("truncation must keep more tail than head, got head=%d tail=%d", ToolTruncHead, ToolTruncTail) |
| 94 | } |
| 95 | } |
| 96 | |
| 97 | // TestTruncateSnapsToRuneBoundary: Truncate's byte-offset cut must not slice |
| 98 | // multi-byte runes mid-sequence: output stays valid UTF-8. |
| 99 | func TestTruncateSnapsToRuneBoundary(t *testing.T) { |
| 100 | in := strings.Repeat("ä", 20000) // 2 bytes each, 40000 bytes total = 10000 tokens |
| 101 | out := Truncate(in) |
| 102 | if !strings.Contains(out, "truncated") { |
| 103 | t.Fatalf("expected truncation marker, got %q", out[:80]) |
| 104 | } |
| 105 | if !utf8.ValidString(out) { |
| 106 | t.Fatal("Truncate produced invalid UTF-8 - slice landed mid-rune") |
| 107 | } |
| 108 | } |
| 109 | |
| 110 | func TestPackNewestFirstWhole(t *testing.T) { |
| 111 | big := strings.Repeat("a", 4*1000) // 1000 tokens |
| 112 | history := []Message{ |
| 113 | {Role: RoleUser, Content: big}, |
| 114 | {Role: RoleAssistant, Content: big}, |
| 115 | {Role: RoleUser, Content: big}, |