Save all the days used in the training, validation and test sets. Args: experiment_id (str): ID of the experiment. general_hyperparameters (dict): General hyperparameters. Returns: None.
(
experiment_id: str,
general_hyperparameters: dict[str, Any],
)
| 219 | |
| 220 | |
| 221 | def save_dataset_info( |
| 222 | experiment_id: str, |
| 223 | general_hyperparameters: dict[str, Any], |
| 224 | ) -> None: |
| 225 | """ |
| 226 | Save all the days used in the training, validation and test sets. |
| 227 | Args: |
| 228 | experiment_id (str): ID of the experiment. |
| 229 | general_hyperparameters (dict): General hyperparameters. |
| 230 | |
| 231 | Returns: |
| 232 | None. |
| 233 | """ |
| 234 | # Access the training data folder and list all the files. |
| 235 | training_days_temp = glob.glob( |
| 236 | f"./data/{general_hyperparameters['dataset']}/scaled_data/training/*.csv" |
| 237 | ) |
| 238 | # Access the validation data folder and list all the files. |
| 239 | validation_days_temp = glob.glob( |
| 240 | f"./data/{general_hyperparameters['dataset']}/scaled_data/validation/*.csv" |
| 241 | ) |
| 242 | # Access the test data folder and list all the files. |
| 243 | test_days_temp = glob.glob( |
| 244 | f"./data/{general_hyperparameters['dataset']}/scaled_data/test/*.csv" |
| 245 | ) |
| 246 | |
| 247 | training_days = [] |
| 248 | validation_days = [] |
| 249 | test_days = [] |
| 250 | |
| 251 | # Extract the dates from the file names (training data). |
| 252 | for i in training_days_temp: |
| 253 | i = i.split("/")[-1].split("_")[-1] |
| 254 | training_days.append(i) |
| 255 | |
| 256 | # Extract the dates from the file names (validation data). |
| 257 | for i in validation_days_temp: |
| 258 | i = i.split("/")[-1].split("_")[-1] |
| 259 | validation_days.append(i) |
| 260 | |
| 261 | # Extract the dates from the file names (test data). |
| 262 | for i in test_days_temp: |
| 263 | i = i.split("/")[-1].split("_")[-1] |
| 264 | test_days.append(i) |
| 265 | |
| 266 | # Create a dictionary containing the training, validation and test days. |
| 267 | dataset_info = { |
| 268 | "training_days": sorted(set(training_days)), |
| 269 | "validation_days": sorted(set(validation_days)), |
| 270 | "test_days": sorted(set(test_days)), |
| 271 | } |
| 272 | |
| 273 | # Save the dictionary as a YAML file. |
| 274 | logger.logger( |
| 275 | experiment_id=experiment_id, |
| 276 | header="dataset_info", |
| 277 | contents=dataset_info, |
| 278 | ) |