(
self,
unregistered_specifier: str = "",
registered_specifier: str = "",
stats_dir: Union[str, Path] = None,
loader_state: Optional[Path] = None,
collate_fn: Optional[Callable] = None,
batchfy_method: str = "bucket",
batch_size: int = 1000,
num_workers: int = 4,
rank: int = 0,
world_size: int = 1,
shuffle: bool = False,
sequential_load: bool = False,
save_loader_state: bool = False,
seed: int = 42,
)
| 65 | """ |
| 66 | |
| 67 | def __init__( |
| 68 | self, |
| 69 | unregistered_specifier: str = "", |
| 70 | registered_specifier: str = "", |
| 71 | stats_dir: Union[str, Path] = None, |
| 72 | loader_state: Optional[Path] = None, |
| 73 | collate_fn: Optional[Callable] = None, |
| 74 | batchfy_method: str = "bucket", |
| 75 | batch_size: int = 1000, |
| 76 | num_workers: int = 4, |
| 77 | rank: int = 0, |
| 78 | world_size: int = 1, |
| 79 | shuffle: bool = False, |
| 80 | sequential_load: bool = False, |
| 81 | save_loader_state: bool = False, |
| 82 | seed: int = 42, |
| 83 | ): |
| 84 | self.collate_fn = collate_fn |
| 85 | self.num_workers = num_workers |
| 86 | self.loader_state = loader_state |
| 87 | self.save_loader_state = save_loader_state |
| 88 | self.rank = rank |
| 89 | self.world_size = world_size |
| 90 | self.shuffle = shuffle |
| 91 | self.sequential_load = sequential_load |
| 92 | self.seed = seed |
| 93 | |
| 94 | # Convert stats_dir to Path if it's a string |
| 95 | if isinstance(stats_dir, str): |
| 96 | stats_dir = Path(stats_dir) |
| 97 | |
| 98 | # (1) parse data specifier |
| 99 | cache_unregistered, cache_registered = _parse_data_specifier( |
| 100 | unregistered_specifier, |
| 101 | registered_specifier, |
| 102 | ) |
| 103 | |
| 104 | # (2) build dataset |
| 105 | # Extract (name, data_json) tuples for unregistered datasets |
| 106 | dataset_unregistered = list( |
| 107 | set((name, data_json) for _, name, data_json, _ in cache_unregistered) |
| 108 | ) |
| 109 | # Extract (name,) tuples for registered datasets |
| 110 | dataset_registered = list(set(name for _, name, _ in cache_registered)) |
| 111 | logging.info( |
| 112 | f"Building dataset with unregistered={dataset_unregistered}, " |
| 113 | f"registered={dataset_registered}" |
| 114 | ) |
| 115 | dataset = CombinedDataset( |
| 116 | dataset_unregistered, |
| 117 | dataset_registered, |
| 118 | num_worker=num_workers, |
| 119 | rank=rank, |
| 120 | world_size=world_size, |
| 121 | ) |
| 122 | logging.info("Dataset construction completed") |
| 123 | |
| 124 | # Store dataset for later use |
nothing calls this directly
no test coverage detected