Function forward

current_data.py:92–150

Creates the forward sample by parsing the current data html files that we downloaded in check_yahoo(). :return: a pandas dataframe containing all of the current data for each ticker.

()

def forward():
    """
    Creates the forward sample by parsing the current data html files that we downloaded in check_yahoo().
    :return: a pandas dataframe containing all of the current data for each ticker.
    """
    # Creating an empty dataframe which we will later fill. In addition to the features, we need some index variables
    # (date, unix timestamp, ticker), and of course the dependent variables (prices).
    df_columns = [
        "Date",
        "Unix",
        "Ticker",
        "Price",
        "stock_p_change",
        "SP500",
        "SP500_p_change",
    ] + features

    df = pd.DataFrame(columns=df_columns)

    tickerfile_list = os.listdir("forward/")

    # Required in macOS to remove the hidden index file.
    if ".DS_Store" in tickerfile_list:
        tickerfile_list.remove(".DS_Store")

    # This is the actual parsing. This needs to be fixed every time yahoo changes their UI.
    for tickerfile in tqdm(tickerfile_list, desc="Parsing progress:", unit="tickers"):
        ticker = tickerfile.split(".html")[0].upper()
        source = open(f"forward/{tickerfile}").read()
        # Remove commas from the html to make parsing easier.
        source = source.replace(",", "")

        # Regex search for the different variables in the html file, then append to value_list
        value_list = []
        for variable in features:
            try:
                # Basically, look for the first number present after we an occurence of the variable
                regex = (
                    r">"
                    + re.escape(variable)
                    + r".*?(\-?\d+\.*\d*K?M?B?|N/A[\\n|\s]*|>0|NaN)%?"
                    r"(</td>|</span>)"
                )
                value = re.search(regex, source, flags=re.DOTALL).group(1)

                # Dealing with number formatting
                value_list.append(data_string_to_float(value))

            # The data may not be present. Process accordingly.
            except AttributeError:
                value_list.append("N/A")
                # print(ticker, variable)

        # Append the ticker and the features to the dataframe
        new_df_row = [0, 0, ticker, 0, 0, 0, 0] + value_list

        df = df.append(dict(zip(df_columns, new_df_row)), ignore_index=True)