| 16 | |
| 17 | class BasePreprocessing: |
| 18 | def __init__( |
| 19 | self, |
| 20 | data_dir: str = "./data/raw/", |
| 21 | save_dir: str = "./data/processed/", |
| 22 | modes: tuple = ("train", "validation", "test"), |
| 23 | n_jobs: int = -1, |
| 24 | ): |
| 25 | self.data_dir = Path(data_dir) |
| 26 | self.save_dir = Path(save_dir) |
| 27 | self.n_jobs = n_jobs |
| 28 | self.modes = modes |
| 29 | |
| 30 | if not self.data_dir.exists(): |
| 31 | logger.error("data folder doesn't exist") |
| 32 | raise FileNotFoundError |
| 33 | if self.save_dir.exists() is False: |
| 34 | self.save_dir.mkdir(parents=True, exist_ok=True) |
| 35 | |
| 36 | self.files = {} |
| 37 | for data_type in self.modes: |
| 38 | self.files.update({data_type: []}) |
| 39 | |
| 40 | @logger.catch |
| 41 | def preprocess(self): |