(encoder_file, bpe_file)
| 128 | |
| 129 | |
| 130 | def get_encoder(encoder_file, bpe_file): |
| 131 | # 以下是为了同一个函数入兼容sentencepiece |
| 132 | filepath, filename = os.path.split(encoder_file) |
| 133 | shotname, extension = os.path.splitext(filename) |
| 134 | |
| 135 | if (".model" == extension) and (bpe_file == ""): |
| 136 | return Encoder_SP(encoder_file) |
| 137 | else: |
| 138 | with open(encoder_file, 'r', encoding="utf-8") as f: |
| 139 | encoder = json.load(f) |
| 140 | with open(bpe_file, 'r', encoding="utf-8") as f: |
| 141 | bpe_data = f.read() |
| 142 | bpe_merges = [tuple(merge_str.split()) for merge_str in bpe_data.split('\n')[1:-1]] |
| 143 | return Encoder( |
| 144 | encoder=encoder, |
| 145 | bpe_merges=bpe_merges, |
| 146 | ) |
| 147 | |
| 148 | |
| 149 | def from_pretrained(): |
no test coverage detected