| 1150 | |
| 1151 | |
| 1152 | def _add_data_args(parser): |
| 1153 | group = parser.add_argument_group(title="data and dataloader") |
| 1154 | |
| 1155 | group.add_argument( |
| 1156 | "--data-path", |
| 1157 | nargs="*", |
| 1158 | default=None, |
| 1159 | help="Path to the training dataset. Accepted format:" |
| 1160 | "1) a single data path, 2) multiple datasets in the" |
| 1161 | "form: dataset1-weight dataset1-path dataset2-weight " |
| 1162 | "dataset2-path ...", |
| 1163 | ) |
| 1164 | group.add_argument( |
| 1165 | "--valid-data-path", |
| 1166 | nargs="*", |
| 1167 | default=None, |
| 1168 | help="Path to the validation dataset. Accepted format:" |
| 1169 | "1) a single data path, 2) multiple datasets in the" |
| 1170 | "form: dataset1-weight dataset1-path dataset2-weight " |
| 1171 | "dataset2-path ...;" |
| 1172 | "when co-evaluation is enabled, the form will be dataset1-tag dataset1-path ...", |
| 1173 | ) |
| 1174 | group.add_argument("--index-cache-dir", type=str, default=None, help="Path to the index cache") |
| 1175 | group.add_argument( |
| 1176 | "--test-data-path", |
| 1177 | nargs="*", |
| 1178 | default=None, |
| 1179 | help="Path to the test dataset. Accepted format:" |
| 1180 | "1) a single data path, 2) multiple datasets in the" |
| 1181 | "form: dataset1-tag dataset1-path dataset2-tag " |
| 1182 | "dataset2-path ...", |
| 1183 | ) |
| 1184 | group.add_argument( |
| 1185 | "--split", |
| 1186 | type=str, |
| 1187 | default="969, 30, 1", |
| 1188 | help="Comma-separated list of proportions for training," |
| 1189 | " validation, and test split. For example the split " |
| 1190 | "`90,5,5` will use 90%% of data for training, 5%% for " |
| 1191 | "validation and 5%% for test.", |
| 1192 | ) |
| 1193 | group.add_argument( |
| 1194 | "--vocab-file", |
| 1195 | type=str, |
| 1196 | default=None, |
| 1197 | help="Path to the vocab file.", |
| 1198 | ) |
| 1199 | group.add_argument( |
| 1200 | "--merge-file", |
| 1201 | type=str, |
| 1202 | default=None, |
| 1203 | help="Path to the BPE merge file.", |
| 1204 | ) |
| 1205 | group.add_argument( |
| 1206 | "--tokenizer-path", |
| 1207 | type=str, |
| 1208 | default=None, |
| 1209 | help="Path to the tokenizer dir.", |