(
#[napi(ts_arg_type = "'gpt2' | 'r50k_base' | 'p50k_base' | 'p50k_edit' | 'cl100k_base'")]
encoding: String,
)
| 46 | |
| 47 | #[napi] |
| 48 | pub fn get_encoding( |
| 49 | #[napi(ts_arg_type = "'gpt2' | 'r50k_base' | 'p50k_base' | 'p50k_edit' | 'cl100k_base'")] |
| 50 | encoding: String, |
| 51 | ) -> Result<Encoding, Error> { |
| 52 | let encoding: Result<tiktoken_rs::CoreBPE, Error> = match encoding.as_str() { |
| 53 | "gpt2" => Ok(tiktoken_rs::r50k_base().unwrap()), |
| 54 | "r50k_base" => Ok(tiktoken_rs::r50k_base().unwrap()), |
| 55 | "p50k_base" => Ok(tiktoken_rs::p50k_base().unwrap()), |
| 56 | "p50k_edit" => Ok(tiktoken_rs::p50k_edit().unwrap()), |
| 57 | "cl100k_base" => Ok(tiktoken_rs::cl100k_base().unwrap()), |
| 58 | _ => Err(Error::from_reason("Invalid encoding")), |
| 59 | }; |
| 60 | |
| 61 | match encoding { |
| 62 | Ok(encoding) => Ok(Encoding { encoding }), |
| 63 | Err(err) => Err(err), |
| 64 | } |
| 65 | } |
| 66 | |
| 67 | #[napi] |
| 68 | pub fn encoding_for_model(model_name: String) -> Result<Encoding, Error> { |
nothing calls this directly
no outgoing calls
no test coverage detected
searching dependent graphs…