| 89 | } |
| 90 | |
| 91 | encode(text) { |
| 92 | if (!this.byte_encoder) throw new Error("Tokenizer not loaded."); |
| 93 | let bpe_tokens = []; |
| 94 | const matches = Array.from(text.matchAll(this.pat)).map((x) => x[0]); |
| 95 | for (let token of matches) { |
| 96 | const encoded_bytes = this.textEncoder.encode(token); |
| 97 | let bytes = []; |
| 98 | for (let i = 0; i < encoded_bytes.length; i++) { |
| 99 | bytes.push(this.byte_encoder[encoded_bytes[i].toString()]); |
| 100 | } |
| 101 | token = bytes.join(""); |
| 102 | |
| 103 | const new_tokens = this.bpe(token) |
| 104 | .split(" ") |
| 105 | .map((x) => this.encoder[x]); |
| 106 | bpe_tokens = bpe_tokens.concat(new_tokens); |
| 107 | } |
| 108 | return bpe_tokens; |
| 109 | } |
| 110 | |
| 111 | decode(tokens) { |
| 112 | if (!this.byte_decoder) throw new Error("Tokenizer not loaded."); |