| 11 | |
| 12 | |
| 13 | class ChatGLM4Tokenizer(PreTrainedTokenizer): |
| 14 | vocab_files_names = {"vocab_file": "tokenizer.model"} |
| 15 | model_input_names = ["input_ids", "attention_mask", "position_ids"] |
| 16 | |
| 17 | def __init__( |
| 18 | self, |
| 19 | vocab_file, |
| 20 | padding_side="left", |
| 21 | clean_up_tokenization_spaces=False, |
| 22 | encode_special_tokens=False, |
| 23 | **kwargs |
| 24 | ): |
| 25 | self.name = "GLMTokenizer" |
| 26 | self.vocab_file = vocab_file |
| 27 | pat_str = "(?i:'s|'t|'re|'ve|'m|'ll|'d)|[^\\r\\n\\p{L}\\p{N}]?\\p{L}+|\\p{N}{1,3}| ?[^\\s\\p{L}\\p{N}]+[\\r\\n]*|\\s*[\\r\\n]+|\\s+(?!\\S)|\\s+" |
| 28 | self.pat_str = re.compile(pat_str) |
| 29 | self.encode_special_tokens = encode_special_tokens |
| 30 | |
| 31 | mergeable_ranks = {} |
| 32 | with open(vocab_file) as f: |
| 33 | for line in f: |
| 34 | token, rank = line.strip().split() |
| 35 | rank = int(rank) |
| 36 | token = base64.b64decode(token) |
| 37 | mergeable_ranks[token] = rank |
| 38 | |
| 39 | self.mergeable_ranks = mergeable_ranks |
| 40 | self.special_tokens = ["<|endoftext|>", "[MASK]", "[gMASK]", "[sMASK]", "<sop>", "<eop>", "<|system|>", |
| 41 | "<|user|>", "<|assistant|>", "<|observation|>", "<|begin_of_image|>", "<|end_of_image|>", |
| 42 | "<|begin_of_video|>", "<|end_of_video|>"] |
| 43 | |
| 44 | self.special_tokens = { |
| 45 | token: idx for idx, token in enumerate(self.special_tokens, start=len(mergeable_ranks)) |
| 46 | } |
| 47 | self.special_token_ids = {idx: token for token, idx in self.special_tokens.items()} |
| 48 | |
| 49 | self.tokenizer = tiktoken.Encoding( |
| 50 | name="my_tokenizer", |
| 51 | pat_str=pat_str, |
| 52 | mergeable_ranks=mergeable_ranks, |
| 53 | special_tokens=self.special_tokens |
| 54 | ) |
| 55 | self.decoder = {rank: token for token, rank in mergeable_ranks.items()} |
| 56 | self.n_words = len(self.decoder) + len(self.special_tokens) |
| 57 | |
| 58 | super().__init__( |
| 59 | padding_side=padding_side, |
| 60 | clean_up_tokenization_spaces=clean_up_tokenization_spaces, |
| 61 | **kwargs |
| 62 | ) |
| 63 | |
| 64 | def get_command(self, token): |
| 65 | assert token in self.special_tokens |
| 66 | return self.special_tokens[token] |
| 67 | |
| 68 | @property |
| 69 | def vocab_size(self): |
| 70 | return self.n_words |
nothing calls this directly
no outgoing calls
no test coverage detected