(
self,
dataset,
learning_stage,
window_size,
shuffling_seed,
cache_size,
lighten,
threshold,
all_horizons,
prediction_horizon,
targets_type,
balanced_dataloader=False,
backtest=False,
training_stocks=None,
validation_stocks=None,
target_stocks=None
)
| 14 | |
| 15 | class CustomDataset(Dataset): |
| 16 | def __init__( |
| 17 | self, |
| 18 | dataset, |
| 19 | learning_stage, |
| 20 | window_size, |
| 21 | shuffling_seed, |
| 22 | cache_size, |
| 23 | lighten, |
| 24 | threshold, |
| 25 | all_horizons, |
| 26 | prediction_horizon, |
| 27 | targets_type, |
| 28 | balanced_dataloader=False, |
| 29 | backtest=False, |
| 30 | training_stocks=None, |
| 31 | validation_stocks=None, |
| 32 | target_stocks=None |
| 33 | ): |
| 34 | self.learning_stage = learning_stage # The current learning stage (training, validation or testing). |
| 35 | self.shuffling_seed = ( |
| 36 | shuffling_seed # The seed for the random shuffling of the datasets. |
| 37 | ) |
| 38 | self.balanced_dataloader = balanced_dataloader # Whether to use a balanced dataloader or not. This option is available only for training. |
| 39 | self.backtest = backtest |
| 40 | self.targets_type = targets_type |
| 41 | |
| 42 | if self.learning_stage == "training": |
| 43 | file_patterns = [f"./data/{dataset}/scaled_data/{self.learning_stage}/{element}_orderbooks*.csv" for element in training_stocks] |
| 44 | self.csv_files = [] |
| 45 | for pattern in file_patterns: |
| 46 | self.csv_files.extend(glob.glob(pattern.format(dataset=dataset, self=self))) |
| 47 | |
| 48 | random.seed(self.shuffling_seed) |
| 49 | random.shuffle(self.csv_files) |
| 50 | else: |
| 51 | # During the validation and testing stages it is fundamental to read the datasets in chronological order. |
| 52 | if self.learning_stage == 'validation': |
| 53 | file_patterns = [f"./data/{dataset}/scaled_data/{self.learning_stage}/{element}_orderbooks*.csv" for element in validation_stocks] |
| 54 | else: |
| 55 | file_patterns = [f"./data/{dataset}/scaled_data/{self.learning_stage}/{element}_orderbooks*.csv" for element in target_stocks] |
| 56 | |
| 57 | self.csv_files = [] |
| 58 | for pattern in file_patterns: |
| 59 | self.csv_files.extend(glob.glob(pattern.format(dataset=dataset, self=self))) |
| 60 | self.csv_files = sorted(self.csv_files) |
| 61 | |
| 62 | self.window_size = window_size # The number of time steps in each window. |
| 63 | self.lighten = lighten # Whether to use the light version of the dataset. |
| 64 | self.threshold = threshold # The threshold for the classification task. |
| 65 | self.prediction_horizon = ( |
| 66 | prediction_horizon # The prediction horizon for the classification task. |
| 67 | ) |
| 68 | self.all_horizons = ( |
| 69 | all_horizons # List of all the possible prediction horizons. |
| 70 | ) |
| 71 | |
| 72 | self.cumulative_lengths = [0] # Store cumulative lengths of datasets. |
| 73 | self.cache_size = cache_size # The number of datasets to cache in memory. |
nothing calls this directly
no test coverage detected