(
self,
tokenizer: GPT2TokenizerFast = None,
tokenizer_path: str = "EleutherAI/gpt-j-6B",
start_extra_id: int = 10,
max_len : int = 10,
mode='codegeex-13b',
dict_file: str = None,
)
| 32 | |
| 33 | class CodeGeeXTokenizer(object): |
| 34 | def __init__( |
| 35 | self, |
| 36 | tokenizer: GPT2TokenizerFast = None, |
| 37 | tokenizer_path: str = "EleutherAI/gpt-j-6B", |
| 38 | start_extra_id: int = 10, |
| 39 | max_len : int = 10, |
| 40 | mode='codegeex-13b', |
| 41 | dict_file: str = None, |
| 42 | ): |
| 43 | self.tokenizer = tokenizer if tokenizer is not None else AutoTokenizer.from_pretrained(tokenizer_path) |
| 44 | if mode not in ['codegeex-13b']: |
| 45 | raise ValueError(f"Invalid mode {mode}, choose from ['codegeex-13b']") |
| 46 | self.start_extra_id = start_extra_id |
| 47 | self.max_len = max_len |
| 48 | self.mode = mode |
| 49 | self.eos_token_id = self.tokenizer.eos_token_id |
| 50 | |
| 51 | def encode_code(self, code: str): |
| 52 | if self.mode == 'codegeex-13b': |
nothing calls this directly
no test coverage detected