| 381 | guidances_df.to_parquet(os.path.join(outpath, "{}.parquet".format(stock)), index=False) |
| 382 | |
| 383 | def _process_news(self, |
| 384 | stocks = None, |
| 385 | start_date = None, |
| 386 | end_date = None): |
| 387 | |
| 388 | start_date = datetime.strptime(start_date if start_date else self.start_date, "%Y-%m-%d") |
| 389 | end_date = datetime.strptime(end_date if end_date else self.end_date, "%Y-%m-%d") |
| 390 | |
| 391 | stocks = stocks if stocks else self.stocks |
| 392 | |
| 393 | news_columns = [ |
| 394 | "title", |
| 395 | "text", |
| 396 | "source", |
| 397 | "url" |
| 398 | ] |
| 399 | |
| 400 | for stock in tqdm(stocks): |
| 401 | |
| 402 | newses = self.path_params["news"] |
| 403 | newses_df = [] |
| 404 | |
| 405 | for news in newses: |
| 406 | news_type = news["type"] |
| 407 | news_path = news["path"] |
| 408 | |
| 409 | news_path = os.path.join(self.root, news_path, "{}.csv".format(stock)) |
| 410 | |
| 411 | if news_type == "fmp": |
| 412 | news_column_map = { |
| 413 | "title": "title", |
| 414 | "text": "text", |
| 415 | "site": "source", |
| 416 | "url": "url", |
| 417 | } |
| 418 | elif news_type == "yahoofinance": |
| 419 | news_column_map = { |
| 420 | "headline": "title", |
| 421 | "summary": "text", |
| 422 | "datetime": "timestamp", |
| 423 | "source": "source", |
| 424 | "url": "url", |
| 425 | } |
| 426 | else: |
| 427 | news_column_map = { |
| 428 | "title": "title", |
| 429 | "text": "text", |
| 430 | "site": "source", |
| 431 | "url": "url", |
| 432 | } |
| 433 | |
| 434 | assert os.path.exists(news_path), "News path {} does not exist".format(news_path) |
| 435 | |
| 436 | news_df = pd.read_csv(news_path) |
| 437 | news_df = news_df.rename(columns=news_column_map)[["timestamp"] + news_columns] |
| 438 | news_df["timestamp"] = pd.to_datetime(news_df["timestamp"]) |
| 439 | |
| 440 | news_df = news_df[(news_df["timestamp"] >= start_date) & (news_df["timestamp"] < end_date)] |