Method ProcessIdeographs

library/cpp/tokenizer/nlpparser.cpp:430–437

}

void TNlpParser::ProcessIdeographs(const wchar16* ts, const wchar16* te) {
    for (const wchar16* p = ts; p != te; ++p) {
        Base.AddIdeograph(1);
        Y_ASSERT(Base.GetTokenCount() == 1);
        MakeMultitokenEntry(Base.GetToken(0), p);
        Base.ResetTokens();
    }
}

void TNlpParser::Execute(const wchar16* text, size_t len, const wchar16** textStart) {
    if (!len)