Method set

transformers/llm/engine/src/tokenizer/unicode.cpp:117–117

struct CharSet {
    uint64_t bits[4] = {};
    void set(uint8_t b) { bits[b >> 6] |= 1ULL << (b & 63); }
    bool test(uint8_t b) const { return (bits[b >> 6] >> (b & 63)) & 1; }
    void negate() { for (auto& w : bits) w = ~w; }
    void merge(const CharSet& o) { for (int i = 0; i < 4; i++) bits[i] |= o.bits[i]; }