Method InitFromMemory

library/cpp/text_processing/dictionary/bpe_dictionary.cpp:403–420

}

void TMMapBpeDictionary::InitFromMemory(const void* data, size_t size) {
    const ui8* ptr = reinterpret_cast<const ui8*>(data);
    Y_ENSURE(!std::memcmp(ptr, BPE_MAGIC, BPE_MAGIC_SIZE));
    ptr += 16;

    BpeSize = *reinterpret_cast<const ui64*>(ptr);
    ptr += 16;

    ui64 sourceTokenIdsToTokenIdSize = *reinterpret_cast<const ui64*>(ptr);
    ptr += 8;
    SourceTokenIdsToTokenIdSeed = *reinterpret_cast<const ui64*>(ptr);
    ptr += 8;

    const TBucket* bucketDataBegin = reinterpret_cast<const TBucket*>(ptr);
    const TBucket* bucketDataEnd = reinterpret_cast<const TBucket*>(ptr + sourceTokenIdsToTokenIdSize);
    SourceTokenIdsToTokenId = MakeArrayRef(bucketDataBegin, bucketDataEnd);
    Y_ENSURE(size == 16 + 16 + 16 + sourceTokenIdsToTokenIdSize);
}