Function Count

internal/provider/custom/token.go:11–19

(input string)

}

func Count(input string) (int, error) {
	encoder, err := tiktoken.GetEncoding("cl100k_base")
	if err != nil {
		return 0, err
	}

	token := encoder.Encode(input, nil, nil)
	return len(token), nil
}