GPT-2 BPE tokenizer. Peculiarities: - Byte-level BPE
| 92 | |
| 93 | |
| 94 | class GPT2Tokenizer(object): |
| 95 | """ |
| 96 | GPT-2 BPE tokenizer. Peculiarities: |
| 97 | - Byte-level BPE |
| 98 | """ |
| 99 | |
| 100 | @classmethod |
| 101 | def from_pretrained( |
| 102 | cls, pretrained_model_name_or_path, cache_dir=None, *inputs, **kwargs |
| 103 | ): |
| 104 | """ |
| 105 | Instantiate a PreTrainedBertModel from a pre-trained model file. |
| 106 | Download and cache the pre-trained model file if needed. |
| 107 | """ |
| 108 | if pretrained_model_name_or_path in PRETRAINED_VOCAB_ARCHIVE_MAP: |
| 109 | vocab_file = PRETRAINED_VOCAB_ARCHIVE_MAP[pretrained_model_name_or_path] |
| 110 | merges_file = PRETRAINED_MERGES_ARCHIVE_MAP[pretrained_model_name_or_path] |
| 111 | special_tokens_file = None |
| 112 | else: |
| 113 | vocab_file = os.path.join(pretrained_model_name_or_path, VOCAB_NAME) |
| 114 | merges_file = os.path.join(pretrained_model_name_or_path, MERGES_NAME) |
| 115 | special_tokens_file = os.path.join( |
| 116 | pretrained_model_name_or_path, SPECIAL_TOKENS_NAME |
| 117 | ) |
| 118 | if not os.path.exists(special_tokens_file): |
| 119 | special_tokens_file = None |
| 120 | else: |
| 121 | logger.info( |
| 122 | "loading special tokens file {}".format(special_tokens_file) |
| 123 | ) |
| 124 | # redirect to the cache, if necessary |
| 125 | try: |
| 126 | from .file_utils import cached_path |
| 127 | |
| 128 | resolved_vocab_file = cached_path(vocab_file, cache_dir=cache_dir) |
| 129 | resolved_merges_file = cached_path(merges_file, cache_dir=cache_dir) |
| 130 | except EnvironmentError: |
| 131 | logger.error( |
| 132 | "Model name '{}' was not found in model name list ({}). " |
| 133 | "We assumed '{}' was a path or url but couldn't find files {} and {} " |
| 134 | "at this path or url.".format( |
| 135 | pretrained_model_name_or_path, |
| 136 | ", ".join(PRETRAINED_VOCAB_ARCHIVE_MAP.keys()), |
| 137 | pretrained_model_name_or_path, |
| 138 | vocab_file, |
| 139 | merges_file, |
| 140 | ) |
| 141 | ) |
| 142 | return None |
| 143 | if resolved_vocab_file == vocab_file and resolved_merges_file == merges_file: |
| 144 | logger.info("loading vocabulary file {}".format(vocab_file)) |
| 145 | logger.info("loading merges file {}".format(merges_file)) |
| 146 | else: |
| 147 | logger.info( |
| 148 | "loading vocabulary file {} from cache at {}".format( |
| 149 | vocab_file, resolved_vocab_file |
| 150 | ) |
| 151 | ) |