This actually does Exploratory data analysis - it means this function performs EDA ###################################################################################### Takes a dataframe containing only predictors to be classified into various types. DO NOT SEND IN A TARGET COL
(df_preds, verbose=0)
| 66 | |
| 67 | #################################################################################### |
| 68 | def classify_columns(df_preds, verbose=0): |
| 69 | """ |
| 70 | This actually does Exploratory data analysis - it means this function performs EDA |
| 71 | ###################################################################################### |
| 72 | Takes a dataframe containing only predictors to be classified into various types. |
| 73 | DO NOT SEND IN A TARGET COLUMN since it will try to include that into various columns. |
| 74 | Returns a data frame containing columns and the class it belongs to such as numeric, |
| 75 | categorical, date or id column, boolean, nlp, discrete_string and cols to delete... |
| 76 | ####### Returns a dictionary with 10 kinds of vars like the following: # continuous_vars,int_vars |
| 77 | # cat_vars,factor_vars, bool_vars,discrete_string_vars,nlp_vars,date_vars,id_vars,cols_delete |
| 78 | """ |
| 79 | train = copy.deepcopy(df_preds) |
| 80 | #### If there are 30 chars are more in a discrete_string_var, it is then considered an NLP variable |
| 81 | max_nlp_char_size = 30 |
| 82 | max_cols_to_print = 30 |
| 83 | print('#######################################################################################') |
| 84 | print('######################## C L A S S I F Y I N G V A R I A B L E S ####################') |
| 85 | print('#######################################################################################') |
| 86 | print('Classifying variables in data set...') |
| 87 | #### Cat_Limit defines the max number of categories a column can have to be called a categorical colum |
| 88 | cat_limit = 35 |
| 89 | float_limit = 15 #### Make this limit low so that float variables below this limit become cat vars ### |
| 90 | |
| 91 | def add(a, b): |
| 92 | return a + b |
| 93 | |
| 94 | sum_all_cols = dict() |
| 95 | orig_cols_total = train.shape[1] |
| 96 | # Types of columns |
| 97 | cols_delete = [] |
| 98 | cols_delete = [col for col in list(train) if (len(train[col].value_counts()) == 1) |
| 99 | | (train[col].isnull().sum() / len(train) >= 0.90)] |
| 100 | inf_cols = EDA_find_remove_columns_with_infinity(train, remove=False, verbose=verbose) |
| 101 | mixed_cols = [x for x in list(train) if len(train[x].dropna().apply(type).value_counts()) > 1] |
| 102 | if len(mixed_cols) > 0: |
| 103 | print(' Removing %s column(s) due to mixed data type detected...' % mixed_cols) |
| 104 | cols_delete += mixed_cols |
| 105 | cols_delete += inf_cols |
| 106 | train = train[left_subtract(list(train), cols_delete)] |
| 107 | var_df = pd.Series(dict(train.dtypes)).reset_index(drop=False).rename( |
| 108 | columns={0: 'type_of_column'}) |
| 109 | sum_all_cols['cols_delete'] = cols_delete |
| 110 | |
| 111 | var_df['bool'] = var_df.apply( |
| 112 | lambda x: 1 if x['type_of_column'] in ['bool', 'object'] and len(train[x['index']].value_counts()) == 2 else 0, |
| 113 | axis=1) |
| 114 | string_bool_vars = list(var_df[(var_df['bool'] == 1)]['index']) |
| 115 | sum_all_cols['string_bool_vars'] = string_bool_vars |
| 116 | var_df['num_bool'] = var_df.apply(lambda x: 1 if x['type_of_column'] in [np.uint8, |
| 117 | np.uint16, np.uint32, np.uint64, |
| 118 | 'int8', 'int16', 'int32', 'int64', |
| 119 | 'float16', 'float32', 'float64'] and len( |
| 120 | train[x['index']].value_counts()) == 2 else 0, axis=1) |
| 121 | num_bool_vars = list(var_df[(var_df['num_bool'] == 1)]['index']) |
| 122 | sum_all_cols['num_bool_vars'] = num_bool_vars |
| 123 | ###### This is where we take all Object vars and split them into diff kinds ### |
| 124 | discrete_or_nlp = var_df.apply(lambda x: 1 if x['type_of_column'] in ['object'] and x[ |
| 125 | 'index'] not in string_bool_vars + cols_delete else 0, axis=1) |
no test coverage detected