| 563 | |
| 564 | |
| 565 | class LlamaHfVocab(Vocab): |
| 566 | tokenizer_model = "llama" |
| 567 | name = "hfft" |
| 568 | |
| 569 | def __init__(self, base_path: Path): |
| 570 | fname_tokenizer = base_path / FAST_TOKENIZER_FILE |
| 571 | # if this fails, FileNotFoundError propagates to caller |
| 572 | with open(fname_tokenizer, encoding='utf-8') as f: |
| 573 | tokenizer_json = json.load(f) |
| 574 | |
| 575 | # pre-check so we know if we need transformers |
| 576 | tokenizer_model: dict[str, Any] = tokenizer_json['model'] |
| 577 | is_llama3 = ( |
| 578 | tokenizer_model['type'] == 'BPE' and tokenizer_model.get('ignore_merges', False) |
| 579 | and not tokenizer_model.get('byte_fallback', True) |
| 580 | ) |
| 581 | if is_llama3: |
| 582 | raise TypeError('Llama 3 must be converted with BpeVocab') |
| 583 | |
| 584 | if not is_llama3 and ( |
| 585 | tokenizer_model['type'] != 'BPE' or not tokenizer_model.get('byte_fallback', False) |
| 586 | or tokenizer_json['decoder']['type'] != 'Sequence' |
| 587 | ): |
| 588 | raise FileNotFoundError('Cannot find Llama BPE tokenizer') |
| 589 | |
| 590 | try: |
| 591 | from transformers import AutoTokenizer |
| 592 | except ImportError as e: |
| 593 | raise ImportError( |
| 594 | "To use LlamaHfVocab, please install the `transformers` package. " |
| 595 | "You can install it with `pip install transformers`." |
| 596 | ) from e |
| 597 | |
| 598 | # Allow the tokenizer to default to slow or fast versions. |
| 599 | # Explicitly set tokenizer to use local paths. |
| 600 | self.tokenizer = AutoTokenizer.from_pretrained( |
| 601 | base_path, |
| 602 | cache_dir=base_path, |
| 603 | local_files_only=True, |
| 604 | ) |
| 605 | assert self.tokenizer.is_fast # assume tokenizer.json is used |
| 606 | |
| 607 | # Initialize lists and dictionaries for added tokens |
| 608 | self.added_tokens_list = [] |
| 609 | self.added_tokens_dict = dict() |
| 610 | self.added_tokens_ids = set() |
| 611 | |
| 612 | # Process added tokens |
| 613 | for tok, tokidx in sorted( |
| 614 | self.tokenizer.get_added_vocab().items(), key=lambda x: x[1] |
| 615 | ): |
| 616 | # Only consider added tokens that are not in the base vocabulary |
| 617 | if tokidx >= self.tokenizer.vocab_size: |
| 618 | self.added_tokens_list.append(tok) |
| 619 | self.added_tokens_dict[tok] = tokidx |
| 620 | self.added_tokens_ids.add(tokidx) |
| 621 | |
| 622 | # Store special tokens and their IDs |