Function textIndexValidator

src/Storages/MergeTree/MergeTreeIndexText.cpp:1776–1830

}

void textIndexValidator(const IndexDescription & index, bool /*attach*/, const MergeTreeSettings & settings)
{
    auto options = convertArgumentsToOptionsMap(index.arguments);

    auto tokenizer_ast = extractASTOption(options, ARGUMENT_TOKENIZER, true);
    auto preprocessor_ast = extractASTOption(options, ARGUMENT_PREPROCESSOR, false);
    TokenizerFactory::instance().get(tokenizer_ast);

    UInt64 dictionary_block_size = extractFieldOption<UInt64>(options, ARGUMENT_DICTIONARY_BLOCK_SIZE)
        .value_or(settings[MergeTreeSetting::text_index_dictionary_block_size]);

    if (dictionary_block_size == 0)
        throw Exception(ErrorCodes::BAD_ARGUMENTS, "Text index argument '{}' must be greater than 0, but got {}", ARGUMENT_DICTIONARY_BLOCK_SIZE, dictionary_block_size);

    UInt64 dictionary_block_use_fc_compression = extractFieldOption<UInt64>(options, ARGUMENT_DICTIONARY_BLOCK_FRONTCODING_COMPRESSION)
        .value_or(settings[MergeTreeSetting::text_index_dictionary_block_frontcoding_compression]);

    if (dictionary_block_use_fc_compression > 1)
        throw Exception(ErrorCodes::BAD_ARGUMENTS, "Text index argument '{}' must be 0 or 1, but got {}", ARGUMENT_DICTIONARY_BLOCK_FRONTCODING_COMPRESSION, dictionary_block_use_fc_compression);

    UInt64 posting_list_block_size = extractFieldOption<UInt64>(options, ARGUMENT_POSTING_LIST_BLOCK_SIZE)
        .value_or(settings[MergeTreeSetting::text_index_posting_list_block_size]);

    if (posting_list_block_size == 0)
        throw Exception(ErrorCodes::BAD_ARGUMENTS, "Text index argument '{}' must be greater than 0, but got {}", ARGUMENT_POSTING_LIST_BLOCK_SIZE, posting_list_block_size);

    String posting_list_codec_name = extractFieldOption<String>(options, ARGUMENT_POSTING_LIST_CODEC)
        .value_or(settings[MergeTreeSetting::text_index_posting_list_codec].toString());

    PostingListCodecFactory::createPostingListCodec(posting_list_codec_name, index.name);

    if (!options.empty())
        throw Exception(ErrorCodes::BAD_ARGUMENTS, "Unexpected text index arguments: {}", fmt::join(std::views::keys(options), ", "));

    /// Check that the index is created on a single column
    if (index.column_names.size() != 1 || index.data_types.size() != 1)
        throw Exception(ErrorCodes::INCORRECT_NUMBER_OF_COLUMNS, "Text index must be created on a single column");

    DataTypePtr index_data_type = index.data_types[0];
    WhichDataType which_data_type(MergeTreeIndexText::getNestedDataType(index_data_type));

    if (!which_data_type.isString() && !which_data_type.isFixedString())
    {
        throw Exception(
            ErrorCodes::BAD_ARGUMENTS,
            "Text index must be created on columns of type with base type of String or FixedString, got: {}",
            index_data_type->getName());
    }

    /// Create the preprocessor for validation.
    /// For very strict validation of the expression we fully parse it here.
    /// However it will be parsed again for index construction, generally immediately after this call.
    /// This is a bit redundant but that doesn't impact performance anyhow because the expression is intended to be simple enough.
    MergeTreeIndexTextPreprocessor preprocessor(preprocessor_ast, index);
}

}