Function AdjustSubtokens

library/cpp/tokenizer/multitokenutil.cpp:198–211

}

size_t AdjustSubtokens(TTokenStructure& subtokens, size_t maxLen) {
    Y_ASSERT(!subtokens.empty());

    TCharSpan* const first = &subtokens[0];
    TCharSpan* p = first + subtokens.size() - 1;
    while (p != first && p->Pos >= maxLen) {
        --p;
    }

    CutSubtoken(*p, maxLen);
    subtokens.resize(p - first + 1);
    p->TokenDelim = TOKDELIM_NULL;
    return p->EndPos() + p->SuffixLen;
}

size_t AdjustSubtokens(TTokenStructure& subtokens, size_t n, size_t maxLen) {
    Y_ASSERT(n > 0);