(code_data_path, args_opt, process_fn=None, scale=1, skip_num=0)
| 171 | |
| 172 | |
| 173 | def get_code_data_train(code_data_path, args_opt, process_fn=None, scale=1, skip_num=0): |
| 174 | datasets = [] |
| 175 | paths = [] |
| 176 | |
| 177 | for dir in sorted(os.listdir(code_data_path)): |
| 178 | sub_dirs = os.listdir(os.path.join(code_data_path, dir)) |
| 179 | for sub_dir in sub_dirs: |
| 180 | if os.path.exists(os.path.join(code_data_path, dir, sub_dir, 'data.mdb')) and os.path.exists( |
| 181 | os.path.join(code_data_path, dir, sub_dir, 'lock.mdb')): |
| 182 | paths.append(os.path.join(code_data_path, dir, sub_dir)) |
| 183 | |
| 184 | for full_path in paths: |
| 185 | if os.path.isdir(full_path): |
| 186 | print(f"Loading code data {full_path}") |
| 187 | data = LMDBDataset( |
| 188 | full_path, |
| 189 | process_fn=process_fn, |
| 190 | ) |
| 191 | data = PadDataset( |
| 192 | data, |
| 193 | args_opt.seq_length, |
| 194 | args_opt.eod_id, |
| 195 | ) |
| 196 | datasets.append(data) |
| 197 | datasets = ConcatDataset(datasets, skip_num=skip_num) |
| 198 | return datasets |
| 199 | |
| 200 | |
| 201 | def get_code_data_eval(code_data_path, args_opt, process_fn=None, scale=1): |
no test coverage detected