Function remove_variables_using_fast_correlation

autoviz/AutoViz_Utils.py:1859–1931

#### THIS METHOD IS KNOWN AS SULO METHOD in HONOR OF my mother SULOCHANA SESHADRI ####### This highly efficient method removes variables that are highly correlated using a series of pair-wise correlation knockout rounds. It is extremely fast and hence can work on thousands of va

(df, numvars, modeltype, target,
                                            corr_limit=0.70, verbose=0)

##################################################################################
def remove_variables_using_fast_correlation(df, numvars, modeltype, target,
                                            corr_limit=0.70, verbose=0):
    """
    #### THIS METHOD IS KNOWN AS SULO METHOD in HONOR OF my mother SULOCHANA SESHADRI #######
    This highly efficient method removes variables that are highly correlated using a series of
    pair-wise correlation knockout rounds. It is extremely fast and hence can work on thousands
    of variables in less than a minute, even on a laptop. You need to send in a list of numeric
    variables and that's all! The method defines high Correlation as anything over 0.70 (absolute)
    but this can be changed. If two variables have absolute correlation higher than this, they
    will be marked, and using a process of elimination, one of them will get knocked out:
    To decide order of variables to keep, we use mutual information score to select. MIS returns
    a ranked list of these correlated variables: when we select one, we knock out other variables
    it is correlated to. Then we select next var. This way we remove correlated variables.
    Finally, we are left with uncorrelated variables that are also highly important (mutual score).
    ##############  YOU MUST INCLUDE THE ABOVE MESSAGE IF YOU COPY THIS CODE IN YOUR LIBRARY #####
    """
    print('    Removing correlated variables from %d numerics using SULO method' % len(numvars))
    correlation_dataframe = df[numvars].corr()
    a = correlation_dataframe.values
    col_index = correlation_dataframe.columns.tolist()
    index_triupper = list(zip(np.triu_indices_from(a, k=1)[0], np.triu_indices_from(a, k=1)[1]))
    high_corr_index_list = [x for x in np.argwhere(abs(a[np.triu_indices(len(a), k=1)]) >= corr_limit)]
    tuple_list = [y for y in [index_triupper[x[0]] for x in high_corr_index_list]]
    correlated_pair = [(col_index[local_tuple[0]], col_index[local_tuple[1]]) for local_tuple in tuple_list]
    corr_pair_dict = dict(return_dictionary_list(correlated_pair))
    keys_in_dict = list(corr_pair_dict.keys())
    reverse_correlated_pair = [(y, x) for (x, y) in correlated_pair]
    reverse_corr_pair_dict = dict(return_dictionary_list(reverse_correlated_pair))
    for key, val in reverse_corr_pair_dict.items():
        if key in keys_in_dict:
            if len(key) > 1:
                corr_pair_dict[key] += val
        else:
            corr_pair_dict[key] = val
    flat_corr_pair_list = [item for sublist in correlated_pair for item in sublist]
    #### You can make it a dictionary or a tuple of lists. We have chosen the latter here to keep order intact.
    corr_pair_count_dict = count_freq_in_list(flat_corr_pair_list)
    corr_list = [k for (k, v) in corr_pair_count_dict]
    ###### This is for ordering the variables from highest to lowest importance by target ###
    if len(corr_list) == 0:
        print('Selecting all (%d) variables since none of them are highly correlated...' % len(numvars))
        return numvars
    else:
        max_feats = len(corr_list)
        if modeltype == 'Regression':
            sel_function = mutual_info_regression
            fs = SelectKBest(score_func=sel_function, k=max_feats)
            fs.fit(df[corr_list], df[target])
            mutual_info = dict(zip(corr_list, fs.scores_))
        else:
            sel_function = mutual_info_classif
            fs = SelectKBest(score_func=sel_function, k=max_feats)
            fs.fit(df[corr_list], df[target])
            mutual_info = dict(zip(corr_list, fs.scores_))
        #### The first variable in list has the highest correlation to the target variable ###
        sorted_by_mutual_info = [key for (key, val) in sorted(mutual_info.items(), key=lambda kv: kv[1], reverse=True)]
        #####   Now we select the final list of correlated variables ###########
        selected_corr_list = []