Function GetLetterIndices

library/cpp/text_processing/dictionary/util.h:20–38

    template <typename TTokenType>
    static void GetLetterIndices(const TTokenType& token, TVector<ui32>* letterStartIndices) {
        letterStartIndices->clear();
        const unsigned char* current = reinterpret_cast<const unsigned char*>(token.data());
        const ui32 tokenSize = token.size();
        const unsigned char* last = current + tokenSize;
        ui32 i = 0;
        while (i < tokenSize) {
            letterStartIndices->push_back(i);
            size_t length = 0;
            auto recodeResult = GetUTF8CharLen(length, current, last);
            if (recodeResult != RECODE_OK || length == 0) {
                letterStartIndices->clear();
                return;
            }
            i += length;
            current += length;
        }
        letterStartIndices->push_back(tokenSize);
    }

    template <bool needToAddEndOfWordToken, typename TTokenType, typename TVisitor>
    static void ApplyFuncToLetterNGramsImpl(