Method Execute

library/cpp/tokenizer/nlpparser.cpp:439–533

}

void TNlpParser::Execute(const wchar16* text, size_t len, const wchar16** textStart) {
    if (!len)
        return;
    const wchar16* p = text;
    const wchar16* e = p + len;
    wchar16* data = nullptr;
    wchar16* dest = nullptr;
    ExtraLen.clear();
    ExtraLenIndex = 0;

    while (p != e) {
        if (UrlDecode && *p == PERCENT_CHAR && (p + 3) <= e && IsHexdigit(p[1]) && IsHexdigit(p[2])) {
            if (!dest) {
                Buffer = TTempArray<wchar16>(len + 1);
                data = Buffer.Data();
                dest = data;
                const size_t n = p - text;
                std::char_traits<wchar16>::copy(dest, text, n);
                dest += n;
            }

            const wchar16* start = p; // in case if UTF8 is bad
            TTempBuf buf(e - p);      // for UTF8
            char* const utf8 = buf.Data();
            size_t i = 0;
            while (p != e && *p == PERCENT_CHAR && (p + 3) <= e && IsHexdigit(p[1]) && IsHexdigit(p[2])) {
                const char c = (HexToChar(char(p[1])) << 4) | HexToChar(char(p[2]));
                utf8[i++] = ((unsigned char)c < 0x20 ? ' ' : c); // replace all controlling characters with ' '
                p += 3;
            }

            bool decoded = false;
            // convert at least 2 UTF8 bytes
            if (i > 1) {
                decoded = true;
                Y_ABORT_UNLESS(size_t(p - start) == 3 * i);
                size_t written = 0;
                const size_t extraLenRollback = ExtraLen.size();
                for (size_t j = 0; j < i;) {
                    size_t stepRead = 0;
                    if (RECODE_OK != GetUTF8CharLen(stepRead, reinterpret_cast<const unsigned char*>(utf8) + j, reinterpret_cast<const unsigned char*>(utf8) + i)) {
                        decoded = false;
                        break;
                    }
                    Y_ABORT_UNLESS(stepRead && j + stepRead <= i);
                    size_t stepWritten = 0;
                    if (!UTF8ToWide(utf8 + j, stepRead, dest + written, stepWritten)) {
                        decoded = false;
                        break;
                    }
                    written += stepWritten;
                    ExtraLen.push_back(std::make_pair<ui32>(dest + written - data, 3 * stepRead - stepWritten));
                    j += stepRead;
                }
                if (decoded) {
                    dest += written;
                } else {
                    ExtraLen.resize(extraLenRollback);