()
| 59 | |
| 60 | |
| 61 | def p50k_edit(): |
| 62 | mergeable_ranks = load_tiktoken_bpe( |
| 63 | "https://openaipublic.blob.core.windows.net/encodings/p50k_base.tiktoken", |
| 64 | expected_hash="94b5ca7dff4d00767bc256fdd1b27e5b17361d7b8a5f968547f9f23eb70d2069", |
| 65 | ) |
| 66 | special_tokens = {ENDOFTEXT: 50256, FIM_PREFIX: 50281, FIM_MIDDLE: 50282, FIM_SUFFIX: 50283} |
| 67 | return { |
| 68 | "name": "p50k_edit", |
| 69 | "pat_str": r50k_pat_str, |
| 70 | "mergeable_ranks": mergeable_ranks, |
| 71 | "special_tokens": special_tokens, |
| 72 | } |
| 73 | |
| 74 | |
| 75 | def cl100k_base(): |
nothing calls this directly
no test coverage detected
searching dependent graphs…