| 185 | |
| 186 | |
| 187 | def main(args: argparse.Namespace) -> None: |
| 188 | |
| 189 | def print_separator(title=""): |
| 190 | if title: |
| 191 | print(f"\n{'='*50}") |
| 192 | print(f" {title}") |
| 193 | print(f"{'='*50}") |
| 194 | else: |
| 195 | print(f"\n{'='*50}") |
| 196 | |
| 197 | # 检查参数 |
| 198 | if not any([args.encode, args.decode, args.vocab_size, args.info, args.vocab_export]): |
| 199 | print("请至少指定一个参数：--encode, --decode, --vocab-size, --info, --vocab-export") |
| 200 | return |
| 201 | |
| 202 | # 初始化tokenizer |
| 203 | preprocessor = InputPreprocessor(model_config=ModelConfig({"model": args.model_name_or_path})) |
| 204 | tokenizer = preprocessor.create_processor().tokenizer |
| 205 | |
| 206 | # 执行操作 |
| 207 | operations_count = 0 |
| 208 | |
| 209 | if args.encode: |
| 210 | print_separator("ENCODING") |
| 211 | print(f"Input text: {args.encode}") |
| 212 | encoded_text = tokenizer.encode(args.encode) |
| 213 | print(f"Encoded tokens: {encoded_text}") |
| 214 | operations_count += 1 |
| 215 | |
| 216 | if args.decode: |
| 217 | print_separator("DECODING") |
| 218 | print(f"Input tokens: {args.decode}") |
| 219 | try: |
| 220 | if isinstance(args.decode, str): |
| 221 | if args.decode.startswith("[") and args.decode.endswith("]"): |
| 222 | tokens = eval(args.decode) |
| 223 | else: |
| 224 | tokens = [int(x.strip()) for x in args.decode.split(",")] |
| 225 | else: |
| 226 | tokens = args.decode |
| 227 | |
| 228 | decoded_text = tokenizer.decode(tokens) |
| 229 | print(f"Decoded text: {decoded_text}") |
| 230 | except Exception as e: |
| 231 | print(f"Error decoding tokens: {e}") |
| 232 | operations_count += 1 |
| 233 | |
| 234 | if args.vocab_size: |
| 235 | print_separator("VOCABULARY SIZE") |
| 236 | print(f"Vocabulary size: {get_vocab_size(tokenizer)}") |
| 237 | operations_count += 1 |
| 238 | |
| 239 | if args.info: |
| 240 | print_separator("TOKENIZER INFO") |
| 241 | print(json.dumps(get_tokenizer_info(tokenizer), indent=2)) |
| 242 | operations_count += 1 |
| 243 | |
| 244 | if args.vocab_export: |