Function PrepareMultitoken

library/cpp/tokenizer/multitokenutil.cpp:150–196

}

NLP_TYPE PrepareMultitoken(TTokenStructure& subtokens, wchar16* buffer, size_t buflen, const wchar16* entry, size_t& len) {
    size_t srcpos = subtokens[0].EndPos(); // the beginning of source delimiters and tokens
    if (srcpos > buflen - 1) {
        srcpos = buflen - 1;
        subtokens[0].Len = srcpos - subtokens[0].Pos;
    }
    size_t destpos = srcpos;
    std::char_traits<wchar16>::copy(buffer, entry, srcpos);

    TCharSpan* const firsttok = &subtokens[0];
    TCharSpan* const lasttok = firsttok + subtokens.size();
    TCharSpan* desttok = firsttok;

    // lengths of tokens are not changed, tokens can be moved only to remove delimiters

    for (const TCharSpan* srctok = firsttok + 1; srctok != lasttok; ++srctok) {
        // Check available space; truncate input if not sufficient
        if (srctok->EndPos() >= buflen) {
            break;
        }

        if (srctok->EndPos() + srctok->SuffixLen >= buflen) {
            ProcessHyphenation(buffer, entry, desttok, srctok, destpos, srcpos);

            if (desttok->EndPos() + desttok->SuffixLen >= buflen) {
                Y_ASSERT(desttok->EndPos() < buflen);
                desttok->SuffixLen = 0; // cut off the suffix

                CopyTokenText(buffer, entry, srctok->Len, destpos, srcpos); // copy with no suffix
            } else
                CopyTokenText(buffer, entry, (srctok->Len + srctok->SuffixLen), destpos, srcpos); // copy the token with suffix

            break;
        }

        ProcessHyphenation(buffer, entry, desttok, srctok, destpos, srcpos);
        CopyTokenText(buffer, entry, (srctok->Len + srctok->SuffixLen), destpos, srcpos); // copy the token with suffix
    }

    Y_ASSERT(srcpos >= destpos);

    // Multitoken->Leng
    len = destpos;
    subtokens.resize(desttok - firsttok + 1);
    subtokens.back().TokenDelim = TOKDELIM_NULL;
    return DetectNLPType(subtokens); // after PrepareMultitoken nlpType can be chagned
}

size_t AdjustSubtokens(TTokenStructure& subtokens, size_t maxLen) {
    Y_ASSERT(!subtokens.empty());