()
| 28 | var tokenEncoderMutex sync.RWMutex |
| 29 | |
| 30 | func InitTokenEncoders() { |
| 31 | common.SysLog("initializing token encoders") |
| 32 | defaultTokenEncoder = codec.NewCl100kBase() |
| 33 | common.SysLog("token encoders initialized") |
| 34 | } |
| 35 | |
| 36 | func getTokenEncoder(model string) tokenizer.Codec { |
| 37 | // First, try to get the encoder from cache with read lock |