Datset builder class.
| 14 | |
| 15 | |
| 16 | class DatasetFactory(object): |
| 17 | """ Datset builder class. """ |
| 18 | |
| 19 | @staticmethod |
| 20 | def get_default_config(updates=None): |
| 21 | config = ConfigDict() |
| 22 | config.type = 'huggingface' |
| 23 | config.text_processor = TextProcessor.get_default_config() |
| 24 | config.huggingface_dataset = HuggingfaceDataset.get_default_config() |
| 25 | config.json_dataset = JsonDataset.get_default_config() |
| 26 | |
| 27 | config.vision_text_processor = VisionTextProcessor.get_default_config() |
| 28 | config.json_vision_dataset = JsonVisionDataset.get_default_config() |
| 29 | |
| 30 | if updates is not None: |
| 31 | config.update(ConfigDict(updates).copy_and_resolve_references()) |
| 32 | return config |
| 33 | |
| 34 | @classmethod |
| 35 | def load_dataset(cls, config, tokenizer, **kwargs): |
| 36 | config = cls.get_default_config(config) |
| 37 | if config.type == 'huggingface': |
| 38 | text_processor = TextProcessor(config.text_processor, tokenizer) |
| 39 | return HuggingfaceDataset( |
| 40 | config.huggingface_dataset, tokenizer, text_processor, **kwargs |
| 41 | ) |
| 42 | elif config.type == 'json': |
| 43 | text_processor = TextProcessor(config.text_processor, tokenizer) |
| 44 | return JsonDataset(config.json_dataset, tokenizer, text_processor, **kwargs) |
| 45 | elif config.type == 'json_vision': |
| 46 | vision_text_processor = VisionTextProcessor(config.vision_text_processor, tokenizer) |
| 47 | return JsonVisionDataset(config.json_vision_dataset, tokenizer, vision_text_processor, **kwargs) |
| 48 | else: |
| 49 | raise ValueError(f'Unknown dataset type: {config.type}') |
| 50 | |
| 51 | def __init__(self): |
| 52 | raise ValueError('DatasetFactory is a static class and should not be instantiated.') |
| 53 | |
| 54 | |
| 55 | class TextProcessor(object): |
nothing calls this directly
no outgoing calls
no test coverage detected