Method matchTokens

src/Storages/MergeTree/MergeTreeIndexText.cpp:613–644

}

std::pair<std::vector<size_t>, NameSet> MergeTreeIndexGranuleText::matchTokens(const ColumnString & all_tokens, std::vector<std::string_view> needed_tokens)
{
    NameSet missing_tokens;
    std::vector<size_t> matched_indices;
    matched_indices.reserve(needed_tokens.size());

    size_t num_tokens = all_tokens.size();
    auto idx_range = collections::range(0, num_tokens);
    auto it_begin = idx_range.begin();

    /// Sort tokens lexicographically for correct binary search in the dictionary.
    std::sort(needed_tokens.begin(), needed_tokens.end());

    for (const auto & token : needed_tokens)
    {
        /// Use binary search to find indices of needed tokens in the block.
        auto it = std::lower_bound(it_begin, idx_range.end(), token, [&all_tokens](size_t lhs_idx, std::string_view rhs_ref)
        {
            return all_tokens.getDataAt(lhs_idx) < rhs_ref;
        });

        it_begin = it;
        size_t idx_in_block = it - idx_range.begin();

        if (idx_in_block < num_tokens && all_tokens.getDataAt(idx_in_block) == token)
            matched_indices.emplace_back(idx_in_block);
        else
            missing_tokens.insert(String(token));
    }

    return {std::move(matched_indices), std::move(missing_tokens)};
}

std::shared_ptr<TextIndexHeader> MergeTreeIndexGranuleText::loadHeader(MergeTreeIndexReaderStream & header_stream, MergeTreeIndexDeserializationState & state)
{