| 552 | sentiments_df.to_parquet(os.path.join(outpath, "{}.parquet".format(stock)), index=False) |
| 553 | |
| 554 | def _process_economic(self, |
| 555 | stocks = None, |
| 556 | start_date = None, |
| 557 | end_date = None): |
| 558 | |
| 559 | start_date = datetime.strptime(start_date if start_date else self.start_date, "%Y-%m-%d") |
| 560 | end_date = datetime.strptime(end_date if end_date else self.end_date, "%Y-%m-%d") |
| 561 | |
| 562 | """ |
| 563 | GDP, realGDP, nominalPotentialGDP, realGDPPerCapita, federalFunds, CPI, inflationRate, inflation, retailSales, consumerSentiment, durableGoods, unemploymentRate, totalNonfarmPayroll, initialClaims, industrialProductionTotalIndex, newPrivatelyOwnedHousingUnitsStartedTotalUnits, totalVehicleSales, retailMoneyFunds, smoothedUSRecessionProbabilities, 3MonthOr90DayRatesAndYieldsCertificatesOfDeposit, commercialBankInterestRateOnCreditCardPlansAllAccounts, 30YearFixedRateMortgageAverage, 15YearFixedRateMortgageAverage |
| 564 | """ |
| 565 | |
| 566 | indicators = [ |
| 567 | "GDP", |
| 568 | "federalFunds", |
| 569 | "CPI", |
| 570 | "inflationRate", |
| 571 | "unemploymentRate", |
| 572 | ] |
| 573 | |
| 574 | type = self.path_params["economic"][0]["type"] |
| 575 | path = self.path_params["economic"][0]["path"] |
| 576 | |
| 577 | df = None |
| 578 | |
| 579 | for indicator in indicators: |
| 580 | |
| 581 | indicator_path = os.path.join(self.root, path, "{}.csv".format(indicator)) |
| 582 | assert os.path.exists(indicator_path), "indicator path {} does not exist".format(indicator_path) |
| 583 | |
| 584 | indicator_df = pd.read_csv(indicator_path) |
| 585 | |
| 586 | indicator_df = indicator_df.rename(columns={ |
| 587 | "GDP": "gdp", |
| 588 | "federalFunds": "federal_funds", |
| 589 | "CPI": "cpi", |
| 590 | "inflationRate": "inflation_rate", |
| 591 | "unemploymentRate": "unemployment_rate", |
| 592 | }) |
| 593 | |
| 594 | indicator_df["timestamp"] = pd.to_datetime(indicator_df["timestamp"]) |
| 595 | indicator_df = indicator_df[(indicator_df["timestamp"] >= start_date) & (indicator_df["timestamp"] < end_date)] |
| 596 | indicator_df = indicator_df.sort_values(by="timestamp") |
| 597 | |
| 598 | if df is None: |
| 599 | df = indicator_df |
| 600 | else: |
| 601 | df = pd.merge(df, indicator_df, on="timestamp", how="left") |
| 602 | |
| 603 | df = df.fillna(method="ffill") |
| 604 | df = df.fillna(method="bfill") |
| 605 | df = df.reset_index(drop=True) |
| 606 | df["type"] = "economic" |
| 607 | |
| 608 | df.to_parquet(os.path.join(self.root, self.workdir, self.tag, "economic.parquet"), index=False) |
| 609 | |
| 610 | def process(self, |
| 611 | stocks = None, |