| 16 | } // namespace |
| 17 | |
| 18 | BPETokenizer::BPETokenizer() |
| 19 | : vocab_size_(0), unk_token_id_(0), bos_token_id_(1), eos_token_id_(2), |
| 20 | vocab_mmap_ptr_(nullptr), vocab_mmap_size_(0), |
| 21 | merges_mmap_ptr_(nullptr), merges_mmap_size_(0) { |
| 22 | has_chat_template_ = false; |
| 23 | } |
| 24 | |
| 25 | BPETokenizer::~BPETokenizer() { |
| 26 | cleanup_mmap(); |
nothing calls this directly
no outgoing calls
no test coverage detected