stocktwits_posts,twitter_posts, stocktwits_comments,twitter_comments, stocktwits_likes,twitter_likes, stocktwits_impressions,twitter_impressions, stocktwits_sentiment,twitter_sentiment
(self,
stocks = None,
start_date = None,
end_date = None)
| 472 | newses_df.to_parquet(os.path.join(outpath, "{}.parquet".format(stock)), index=False) |
| 473 | |
| 474 | def _process_sentiment(self, |
| 475 | stocks = None, |
| 476 | start_date = None, |
| 477 | end_date = None): |
| 478 | """ |
| 479 | stocktwits_posts,twitter_posts, |
| 480 | stocktwits_comments,twitter_comments, |
| 481 | stocktwits_likes,twitter_likes, |
| 482 | stocktwits_impressions,twitter_impressions, |
| 483 | stocktwits_sentiment,twitter_sentiment |
| 484 | """ |
| 485 | |
| 486 | start_date = datetime.strptime(start_date if start_date else self.start_date, "%Y-%m-%d") |
| 487 | end_date = datetime.strptime(end_date if end_date else self.end_date, "%Y-%m-%d") |
| 488 | |
| 489 | stocks = stocks if stocks else self.stocks |
| 490 | |
| 491 | sentiment_columns = [ |
| 492 | "stocktwits_posts", |
| 493 | "stocktwits_comments", |
| 494 | "stocktwits_likes", |
| 495 | "stocktwits_impressions", |
| 496 | "stocktwits_sentiment", |
| 497 | ] |
| 498 | |
| 499 | for stock in tqdm(stocks): |
| 500 | |
| 501 | sentiments = self.path_params["sentiment"] |
| 502 | sentiments_df = [] |
| 503 | |
| 504 | for sentiment in sentiments: |
| 505 | sentiment_type = sentiment["type"] |
| 506 | sentiment_path = sentiment["path"] |
| 507 | |
| 508 | sentiment_path = os.path.join(self.root, sentiment_path, "{}.csv".format(stock)) |
| 509 | |
| 510 | if sentiment_type == "fmp": |
| 511 | sentiment_column_map = {} |
| 512 | |
| 513 | assert os.path.exists(sentiment_path), "sentiment path {} does not exist".format(sentiment_path) |
| 514 | |
| 515 | sentiment_df = pd.read_csv(sentiment_path) |
| 516 | sentiment_df = sentiment_df.rename(columns=sentiment_column_map)[["timestamp"] + sentiment_columns] |
| 517 | sentiment_df["timestamp"] = pd.to_datetime(sentiment_df["timestamp"]) |
| 518 | |
| 519 | sentiment_df = sentiment_df[ (sentiment_df["timestamp"] >= start_date) & (sentiment_df["timestamp"] < end_date)] |
| 520 | sentiment_df = sentiment_df.sort_values(by="timestamp") |
| 521 | sentiment_df["timestamp"] = pd.to_datetime(sentiment_df["timestamp"]).apply(lambda x: x.strftime("%Y-%m-%d")) |
| 522 | |
| 523 | if sentiment_type == "rapidapi_seekingalpha": |
| 524 | sentiment_df["type"] = "rapidapi" |
| 525 | sentiment_df["source"] = "seekingalpha" |
| 526 | |
| 527 | sentiment_df = cal_sentiment(sentiment_df, sentiment_columns) |
| 528 | sentiment_df = sentiment_df.drop_duplicates(subset=["timestamp"], keep="first") |
| 529 | sentiment_df = sentiment_df.reset_index(drop=True) |
| 530 | sentiment_df["timestamp"] = pd.to_datetime(sentiment_df["timestamp"]).apply(lambda x: x.strftime("%Y-%m-%d")) |
| 531 | sentiments_df.append(sentiment_df) |
no test coverage detected