(self, code: str)
| 147 | self.eos_token_id = self.tokenizer.eos_token_id |
| 148 | |
| 149 | def encode_code(self, code: str): |
| 150 | if self.mode == '6b': |
| 151 | code = encode_whitespaces(code, self.start_extra_id, self.max_len) |
| 152 | input_ids = self.tokenizer(code).input_ids |
| 153 | |
| 154 | elif self.mode == '13b': |
| 155 | code = encode_whitespaces(code, self.start_extra_id, self.max_len) |
| 156 | input_ids = self.code_dict.map_tokens(self.tokenizer.encode(code)) |
| 157 | input_ids = np.array(input_ids, dtype=np.int64).reshape(1, -1) |
| 158 | |
| 159 | return input_ids |
| 160 | |
| 161 | def decode_code(self, input_ids): |
| 162 | if self.mode == '6b': |
no test coverage detected