Initialize tokenizer.
(args)
| 50 | |
| 51 | |
| 52 | def build_hgf_tokenizer(args): |
| 53 | """Initialize tokenizer.""" |
| 54 | tokenizer_path = args.tokenizer_path |
| 55 | if args.rank == 0: |
| 56 | print(f"> building huggingface tokenizer from {tokenizer_path} ...", flush=True) |
| 57 | assert tokenizer_path is not None, "Tokenizer path must be provided." |
| 58 | |
| 59 | tokenizer = AutoTokenizer.from_pretrained(tokenizer_path) |
| 60 | if args.rank == 0: |
| 61 | print(f" > eos_token = {tokenizer.eos_token}", flush=True) |
| 62 | |
| 63 | ws_start_id = args.ws_encoding_start_id if "ws_encoding_start_id" in args else None |
| 64 | ws_len = args.ws_encoding_length if "ws_encoding_length" in args else None |
| 65 | |
| 66 | return HgfTokenizerWrapper( |
| 67 | tokenizer, ws_start=ws_start_id, ws_len=ws_len |
| 68 | ) |
| 69 | |
| 70 | |
| 71 | def build_tokenizer(args): |
no test coverage detected