(
name, tokenizer, train_size=128, val_size=64,seed=0, seqlen=2048, test_only=False
)
| 156 | |
| 157 | |
| 158 | def get_loaders( |
| 159 | name, tokenizer, train_size=128, val_size=64,seed=0, seqlen=2048, test_only=False |
| 160 | ): |
| 161 | if 'wikitext2' in name: |
| 162 | return get_wikitext2(tokenizer,train_size,val_size,seed,seqlen,test_only) |
| 163 | elif 'c4' in name: |
| 164 | return get_c4(tokenizer,train_size,val_size,seed,seqlen,test_only) |
| 165 | elif 'redpajama' in name: |
| 166 | return get_redpajama(tokenizer,train_size,val_size,seed,seqlen) |
| 167 | else: |
| 168 | raise NotImplementedError |
| 169 | |
| 170 | |
| 171 |