| 308 | features_df.to_parquet(os.path.join(outpath, "{}.parquet".format(stock)), index=False) |
| 309 | |
| 310 | def _process_guidance(self, |
| 311 | stocks = None, |
| 312 | start_date = None, |
| 313 | end_date = None): |
| 314 | start_date = datetime.strptime(start_date if start_date else self.start_date, "%Y-%m-%d") |
| 315 | end_date = datetime.strptime(end_date if end_date else self.end_date, "%Y-%m-%d") |
| 316 | |
| 317 | stocks = stocks if stocks else self.stocks |
| 318 | |
| 319 | guidance_columns = [ |
| 320 | "title", |
| 321 | "text", |
| 322 | "sentiment", |
| 323 | "url", |
| 324 | ] |
| 325 | |
| 326 | for stock in tqdm(stocks): |
| 327 | |
| 328 | guidances = self.path_params["guidance"] |
| 329 | guidances_df = [] |
| 330 | |
| 331 | for guidance in guidances: |
| 332 | guidance_type = guidance["type"] |
| 333 | guidance_path = guidance["path"] |
| 334 | |
| 335 | guidance_path = os.path.join(self.root, guidance_path, "{}.csv".format(stock)) |
| 336 | |
| 337 | if guidance_type == "rapidapi_seekingalpha": |
| 338 | guidance_column_map = { |
| 339 | "title": "title", |
| 340 | "summary": "text", |
| 341 | "sentiment": "sentiment", |
| 342 | "url": "url", |
| 343 | } |
| 344 | |
| 345 | assert os.path.exists(guidance_path), "guidance path {} does not exist".format(guidance_path) |
| 346 | |
| 347 | guidance_df = pd.read_csv(guidance_path) |
| 348 | guidance_df = guidance_df.rename(columns=guidance_column_map)[["timestamp"] + guidance_columns] |
| 349 | guidance_df["timestamp"] = pd.to_datetime(guidance_df["timestamp"]) |
| 350 | |
| 351 | guidance_df = guidance_df[(guidance_df["timestamp"] >= start_date) & (guidance_df["timestamp"] < end_date)] |
| 352 | guidance_df = guidance_df.sort_values(by="timestamp") |
| 353 | guidance_df = guidance_df.drop_duplicates(subset=["timestamp", "title", "text"], keep="first") |
| 354 | |
| 355 | if guidance_type == "rapidapi_seekingalpha": |
| 356 | guidance_df["type"] = "rapidapi" |
| 357 | guidance_df["source"] = "seekingalpha" |
| 358 | |
| 359 | guidance_df = guidance_df.reset_index(drop=True) |
| 360 | guidance_df = cal_guidance(guidance_df) |
| 361 | guidance_df["timestamp"] = pd.to_datetime(guidance_df["timestamp"]).apply(lambda x: x.strftime("%Y-%m-%d")) |
| 362 | guidances_df.append(guidance_df) |
| 363 | |
| 364 | guidances_df = pd.concat(guidances_df) |
| 365 | |
| 366 | if self.if_parse_url: |
| 367 | urls = guidances_df["url"].values |