(model string)
| 34 | } |
| 35 | |
| 36 | func getTokenEncoder(model string) tokenizer.Codec { |
| 37 | // First, try to get the encoder from cache with read lock |
| 38 | tokenEncoderMutex.RLock() |
| 39 | if encoder, exists := tokenEncoderMap[model]; exists { |
| 40 | tokenEncoderMutex.RUnlock() |
| 41 | return encoder |
| 42 | } |
| 43 | tokenEncoderMutex.RUnlock() |
| 44 | |
| 45 | // If not in cache, create new encoder with write lock |
| 46 | tokenEncoderMutex.Lock() |
| 47 | defer tokenEncoderMutex.Unlock() |
| 48 | |
| 49 | // Double-check if another goroutine already created the encoder |
| 50 | if encoder, exists := tokenEncoderMap[model]; exists { |
| 51 | return encoder |
| 52 | } |
| 53 | |
| 54 | // Create new encoder |
| 55 | modelCodec, err := tokenizer.ForModel(tokenizer.Model(model)) |
| 56 | if err != nil { |
| 57 | // Cache the default encoder for this model to avoid repeated failures |
| 58 | tokenEncoderMap[model] = defaultTokenEncoder |
| 59 | return defaultTokenEncoder |
| 60 | } |
| 61 | |
| 62 | // Cache the new encoder |
| 63 | tokenEncoderMap[model] = modelCodec |
| 64 | return modelCodec |
| 65 | } |
| 66 | |
| 67 | func getTokenNum(tokenEncoder tokenizer.Codec, text string) int { |
| 68 | if text == "" { |
no outgoing calls
no test coverage detected