Method CorrectPositions

library/cpp/tokenizer/nlpparserbase.h:84–96

returns length of all subtokens including prefix of the first subtoken and suffix of the last subtoken

    }
    // returns length of all subtokens including prefix of the first subtoken and suffix of the last subtoken
    size_t CorrectPositions() {
        Y_ASSERT(!Subtokens.empty());
        // position of the first subtoken can be non-zero in case of non-first subtoken
        // but TWideToken::Token must point to the first character of the first subtoken (prefix included if any)
        const size_t pos = Subtokens[0].Pos - Subtokens[0].PrefixLen;
        if (pos) {
            const size_t n = Subtokens.size();
            for (size_t i = 0; i < n; ++i)
                Subtokens[i].Pos -= pos;
        }
        Y_ASSERT(Subtokens[0].Pos == Subtokens[0].PrefixLen); // PrefixLen can be equal to 0
        return Subtokens.back().EndPos() + Subtokens.back().SuffixLen;
    }
    void UpdateSuffix() {
        if (Subtokens.empty())
            Y_ASSERT(!"can't update suffix: no subtokens");