| 252 | } |
| 253 | |
| 254 | async function loadEncoder(encoding) { |
| 255 | if (encodingCache.has(encoding)) { |
| 256 | return encodingCache.get(encoding); |
| 257 | } |
| 258 | const controller = new AbortController(); |
| 259 | const timeoutId = window.setTimeout(() => controller.abort(), 8000); |
| 260 | let response; |
| 261 | try { |
| 262 | response = await fetch(`https://tiktoken.pages.dev/js/${encoding}.json`, { |
| 263 | signal: controller.signal |
| 264 | }); |
| 265 | } finally { |
| 266 | window.clearTimeout(timeoutId); |
| 267 | } |
| 268 | if (!response.ok) { |
| 269 | throw new Error(`Failed to load ${encoding} ranks from CDN`); |
| 270 | } |
| 271 | const ranks = await response.json(); |
| 272 | const encoder = new Tiktoken(ranks); |
| 273 | encodingCache.set(encoding, encoder); |
| 274 | return encoder; |
| 275 | } |
| 276 | |
| 277 | function quickTokenEstimate(text) { |
| 278 | const source = String(text || ''); |