Split the data into training, validation and test sets based on the training, validation and test ratios. Args: dataset (str): The considered dataset (i.e. nasdaq, lse, ...). training_stocks (list): The list of stocks to be used for training. target_stock (list):
(
dataset: str,
training_stocks: list[str],
target_stock: list[str],
training_ratio: float,
validation_ratio: float,
include_target_stock_in_training: bool,
)
| 28 | |
| 29 | |
| 30 | def data_split( |
| 31 | dataset: str, |
| 32 | training_stocks: list[str], |
| 33 | target_stock: list[str], |
| 34 | training_ratio: float, |
| 35 | validation_ratio: float, |
| 36 | include_target_stock_in_training: bool, |
| 37 | ) -> None: |
| 38 | """ |
| 39 | Split the data into training, validation and test sets based on the training, validation and test ratios. |
| 40 | |
| 41 | Args: |
| 42 | dataset (str): The considered dataset (i.e. nasdaq, lse, ...). |
| 43 | training_stocks (list): The list of stocks to be used for training. |
| 44 | target_stock (list): The list of stocks to be used for validation and test. |
| 45 | training_ratio (float): The ratio of training data. |
| 46 | validation_ratio (float): The ratio of validation data. |
| 47 | include_target_stock_in_training (bool): Including or not the target stock in the training set. |
| 48 | |
| 49 | Returns: |
| 50 | None. |
| 51 | """ |
| 52 | # List of target_stocks contains stocks that must be split into training, validation and test sets. |
| 53 | # If requested, target stocks are removed from the training set in a second stage. |
| 54 | for stock in target_stock: |
| 55 | # Sorted list of scaled data. |
| 56 | files_scaled = sorted(glob.glob(f"./data/{dataset}/scaled_data/{stock}/*.csv")) |
| 57 | # Sorted list of unscaled data. |
| 58 | files_unscaled = sorted( |
| 59 | glob.glob(f"./data/{dataset}/unscaled_data/{stock}/*.csv") |
| 60 | ) |
| 61 | |
| 62 | # Sanity check to make sure that the number of files in the scaled and unscaled folders is the same. |
| 63 | assert len(files_scaled) == len( |
| 64 | files_unscaled |
| 65 | ), "The number of files in the scaled and unscaled folders must be the same." |
| 66 | |
| 67 | # Number of training files (based on training ratio). |
| 68 | num_training_files = int(len(files_scaled) * training_ratio) |
| 69 | # Number of validation files (based on validation ratio). |
| 70 | num_validation_files = int(len(files_scaled) * validation_ratio) |
| 71 | # Number of test files (based on test ratio). |
| 72 | num_test_files = len(files_scaled) - num_training_files - num_validation_files |
| 73 | |
| 74 | # Create the training folder (scaled data) if it does not exist. |
| 75 | if not os.path.exists(f"./data/{dataset}/scaled_data/training"): |
| 76 | os.makedirs(f"./data/{dataset}/scaled_data/training") |
| 77 | # Create the validation folder (scaled data) if it does not exist. |
| 78 | if not os.path.exists(f"./data/{dataset}/scaled_data/validation"): |
| 79 | os.makedirs(f"./data/{dataset}/scaled_data/validation") |
| 80 | # Create the test folder (scaled data) if it does not exist. |
| 81 | if not os.path.exists(f"./data/{dataset}/scaled_data/test"): |
| 82 | os.makedirs(f"./data/{dataset}/scaled_data/test") |
| 83 | |
| 84 | # Create the training folder (unscaled data) if it does not exist. |
| 85 | if not os.path.exists(f"./data/{dataset}/unscaled_data/training"): |
| 86 | os.makedirs(f"./data/{dataset}/unscaled_data/training") |
| 87 | # Create the validation folder (unscaled data) if it does not exist. |