Class StringRef

transformers/llm/engine/src/tokenizer/tokenizer.cpp:1472–1477

}

struct StringRef {
    const char* ptr;
    uint16_t len;
    std::string str() const { return std::string(ptr, len); }
    bool operator==(const StringRef& o) const { return len == o.len && memcmp(ptr, o.ptr, len) == 0; }
};

struct VocabEntry {
    const char* str;