(self, code: str)
| 49 | self.eos_token_id = self.tokenizer.eos_token_id |
| 50 | |
| 51 | def encode_code(self, code: str): |
| 52 | if self.mode == 'codegeex-13b': |
| 53 | code = encode_whitespaces(code, self.start_extra_id, self.max_len) |
| 54 | input_ids = self.tokenizer(code, is_split_into_words=False, verbose=False).input_ids |
| 55 | |
| 56 | return input_ids |
| 57 | |
| 58 | def decode_code(self, input_ids): |
| 59 | if self.mode == 'codegeex-13b': |