| 129 | |
| 130 | |
| 131 | class CodeTokenizer(object): |
| 132 | def __init__( |
| 133 | self, |
| 134 | tokenizer: GPT2TokenizerFast = None, |
| 135 | start_extra_id: int = 10, |
| 136 | max_len: int = 10, |
| 137 | mode='13b', |
| 138 | dict_file: str = None, |
| 139 | ): |
| 140 | self.tokenizer = tokenizer if tokenizer is not None else AutoTokenizer.from_pretrained("EleutherAI/gpt-j-6B") |
| 141 | if mode not in ['6b', '13b']: |
| 142 | raise ValueError(f"Invalid mode {mode}, choose from ['6b', '13b']") |
| 143 | self.start_extra_id = start_extra_id |
| 144 | self.max_len = max_len |
| 145 | self.mode = mode |
| 146 | self.code_dict = Code13BDictionary(dict_file, pad_to_vocab_size=51200) if self.mode == '13b' else None |
| 147 | self.eos_token_id = self.tokenizer.eos_token_id |
| 148 | |
| 149 | def encode_code(self, code: str): |
| 150 | if self.mode == '6b': |
| 151 | code = encode_whitespaces(code, self.start_extra_id, self.max_len) |
| 152 | input_ids = self.tokenizer(code).input_ids |
| 153 | |
| 154 | elif self.mode == '13b': |
| 155 | code = encode_whitespaces(code, self.start_extra_id, self.max_len) |
| 156 | input_ids = self.code_dict.map_tokens(self.tokenizer.encode(code)) |
| 157 | input_ids = np.array(input_ids, dtype=np.int64).reshape(1, -1) |
| 158 | |
| 159 | return input_ids |
| 160 | |
| 161 | def decode_code(self, input_ids): |
| 162 | if self.mode == '6b': |
| 163 | texts = self.tokenizer.batch_decode(input_ids) |
| 164 | output_code = [decode_whitespaces(text, self.start_extra_id, self.max_len) for text in texts] |
| 165 | |
| 166 | elif self.mode == '13b': |
| 167 | input_ids = [self.code_dict.decode_tokens(input_ids.tolist()[0])] |
| 168 | texts = self.tokenizer.batch_decode(input_ids) |
| 169 | output_code = [decode_whitespaces(text, self.start_extra_id, self.max_len) for text in texts] |
| 170 | |
| 171 | return output_code |
no outgoing calls
no test coverage detected