Load dataset. Args: path(str): Data path of the dataset. libs(Optional[Union[str, list]]): All the libraries to be tested. Defaults to None, which means all. mode(str): Dataset mode. Only `Insertion` and `Completion` are su
(self,
path: str,
libs: Optional[Union[str, list]] = None,
mode: str = 'Insertion')
| 87 | return data |
| 88 | |
| 89 | def load(self, |
| 90 | path: str, |
| 91 | libs: Optional[Union[str, list]] = None, |
| 92 | mode: str = 'Insertion'): |
| 93 | """Load dataset. |
| 94 | |
| 95 | Args: |
| 96 | path(str): Data path of the dataset. |
| 97 | libs(Optional[Union[str, list]]): All the libraries to be |
| 98 | tested. Defaults to None, which means all. |
| 99 | mode(str): Dataset mode. Only `Insertion` and `Completion` |
| 100 | are supported. Defaults to `Insertion`. |
| 101 | """ |
| 102 | |
| 103 | path = get_data_path(path, local_mode=True) |
| 104 | if isinstance(libs, str): |
| 105 | libs = [libs] |
| 106 | |
| 107 | if libs: |
| 108 | for lib in libs: |
| 109 | assert lib in self.ALL_LIBS, \ |
| 110 | f'Please choose libs in {self.ALL_LIBS}' |
| 111 | else: |
| 112 | libs = self.ALL_LIBS |
| 113 | |
| 114 | data_list = [] |
| 115 | for lib in libs: |
| 116 | source_path = Path(path) / lib / mode |
| 117 | problems = sorted(os.listdir(source_path), |
| 118 | key=lambda x: int(str(x).replace('q', ''))) |
| 119 | for problem in problems: |
| 120 | new_problem = self.get_data(source_path / problem) |
| 121 | data_list.append(new_problem) |
| 122 | |
| 123 | dataset = Dataset.from_list(data_list) |
| 124 | return DatasetDict({'test': dataset}) |
| 125 | |
| 126 | |
| 127 | @TEXT_POSTPROCESSORS.register_module('ds1000') |
no test coverage detected