Function TestSimpleTokenCounter_Count

pkg/context/token_counter_test.go:8–66

(t *testing.T)

)

func TestSimpleTokenCounter_Count(t *testing.T) {
	tests := []struct {
		name    string
		config  ModelConfig
		text    string
		wantMin int
		wantMax int
	}{
		{
			name:    "empty text",
			config:  GPT4Config,
			text:    "",
			wantMin: 0,
			wantMax: 0,
		},
		{
			name:    "simple english text",
			config:  GPT4Config,
			text:    "Hello, world!",
			wantMin: 2,
			wantMax: 5,
		},
		{
			name:    "chinese text",
			config:  ClaudeSonnet45Config,
			text:    "你好，世界！",
			wantMin: 1,
			wantMax: 3,
		},
		{
			name:    "mixed text",
			config:  GPT4Config,
			text:    "Hello 世界! This is a test.",
			wantMin: 5,
			wantMax: 10,
		},
		{
			name:    "long text",
			config:  GPT4Config,
			text:    "This is a longer piece of text that should result in more tokens being counted.",
			wantMin: 15,
			wantMax: 25,
		},
	}

	for _, tt := range tests {
		t.Run(tt.name, func(t *testing.T) {
			counter := NewSimpleTokenCounter(tt.config)
			got, err := counter.Count(context.Background(), tt.text)
			if err != nil {
				t.Errorf("Count() error = %v", err)
				return
			}
			if got < tt.wantMin || got > tt.wantMax {
				t.Errorf("Count() = %v, want between %v and %v", got, tt.wantMin, tt.wantMax)
			}
		})
	}