| 76 | return stock_data |
| 77 | |
| 78 | def process_stock(s): |
| 79 | files = sorted(glob.glob(f"../data/nasdaq/unscaled_data/{s}/*")) |
| 80 | num_workers = 10 |
| 81 | |
| 82 | # Splitting files into chunks for each process |
| 83 | file_chunks = np.array_split(files, num_workers) |
| 84 | |
| 85 | with multiprocessing.Pool(num_workers) as pool: |
| 86 | chunk_results = pool.map(process_stock_files, file_chunks) |
| 87 | |
| 88 | # Aggregating results from all chunks |
| 89 | stock_data = { |
| 90 | 'Mids': [], 'Spreads': [], 'Best_Ask_Volume': [], 'Best_Bid_Volume': [], |
| 91 | 'Volatility_10': [], 'Volatility_50': [], 'Volatility_100': [], |
| 92 | 'Levels_Ask_Side': [], 'Levels_Bid_Side': [], 'Seconds_Horizon_10': [], |
| 93 | 'Seconds_Horizon_50': [], 'Seconds_Horizon_100': [] |
| 94 | } |
| 95 | for chunk in chunk_results: |
| 96 | for key in stock_data: |
| 97 | stock_data[key].extend(chunk[key]) |
| 98 | |
| 99 | return s, stock_data |
| 100 | |
| 101 | if __name__ == "__main__": |
| 102 | stocks = ["BAC", "CHTR", "CSCO", "GOOG", "GS", "IBM", "MCD", "NVDA", "ORCL", "PFE", "PM", "VZ"] #"ABBV", "KO", "AAPL", |