Class DataProcessing

data_processor/data_processing.py:6–105

class DataProcessing:
    def __init__(self, split):
        self.split = split
    
    def make_features(self, file_path, window, csv_path, make_y=True, verbose=True, save_csv=False):
        df = pd.read_csv(f"{file_path}", index_col=0)
        print(df.shape)
        cols = df.columns
        #print(type(df[cols[1]].iloc[1]))
        #print(df[cols[2]].head())
        for i in cols[1:]:
            print(i)
            df[f'{i}_ret1'] = np.log(df[i]/df[i].shift(1))
            #df[f'{i}_autocorr1'] = df[f'{i}_ret1'].corr(df[f'{i}_ret1'].shift(1))
            for j in range(2, window, 2):
                #print((df[i]/df[i].shift(1)).head())
                df[f'{i}_ret{j}'] = np.log(df[i]/df[i].shift(j))
                df[f'{i}_mavg{j}'] = df[f'{i}_ret1'].rolling(j).mean()
                df[f'{i}_ewm{j}'] = df[f'{i}_ret1'].ewm(span=j).mean()
                #df[f'{i}_autocorr{j}'] = df[f'{i}_ret1'].corr(df[f'{i}_ret1'].shift(j))
            if window > 10:
                for j in range(10, window, 5):
                    df[f'{i}_vol{j}'] = df[f'{i}_ret1'].rolling(j).std()
                    df[f'{i}_ewmvol{j}'] = df[f'{i}_ret1'].ewm(span=j).std()
                    break
        df['liq'] = df['close']*df['volume']
        df['liq_ret1'] = df['liq']/df['liq'].shift(1)
        #df['liq_autocorr1'] = df['liq'].corr(df['liq'].shift(1))
        for j in range(2, window, 2):
            df[f'liq_ret{j}'] = df['liq']/df['liq'].shift(j)
            df[f'liq_mavg{j}'] = df['liq'].rolling(j).mean()
            df[f'liq_ewm{j}'] = df['liq'].ewm(span=j).mean()
            #df[f'liq_autocorr{j}'] = df['liq'].corr(df['liq'].shift(j))
        if window > 10:
            for j in range(10, window, 5):
                #df[f'liq_vol{j}'] = df['liq'].rolling(j).std()
                #df[f'liq_ewmvol{j}'] = df['liq'].ewm(span=j).std()
                break
        if verbose:
            df = df.dropna()
            print(df.shape)
            print(df.tail())
            print(df.head())
        if save_csv:
            df.to_csv(f'{csv_path}/full_features.csv')
        return df

    def make_train_test(self, df_x, df_y, window, csv_path, has_y=False, binary_y=False, save_csv=False):
        """
        Splits the dataset into train and test
        :param df_x: dataframe of x variables
        :type df_x: pd.DataFrame
        :param df_y: dataframe of y values
        :type df_y: pd.DataFrame
        :param window: the prediction window
        :type window: int
        :param has_y: whether df_y exists separately or is a column in df_x (must be 'target' column)
        :type has_y: boolean