(tokenizer, num_samples=8192)
| 76 | } |
| 77 | |
| 78 | def create_all_datasets(tokenizer, num_samples=8192): |
| 79 | return { |
| 80 | 'fixed_short': create_fixed_short_dataset(tokenizer, num_samples), |
| 81 | 'variable_short': create_variable_short_dataset(tokenizer, num_samples), |
| 82 | 'fixed_long': create_fixed_long_dataset(tokenizer, num_samples), |
| 83 | 'variable_long': create_variable_long_dataset(tokenizer, num_samples) |
| 84 | } |
| 85 | |
| 86 | def test_batch_size_worker(q, model_name, input_ids, attention_mask, bsize, device, use_xformers): |
| 87 | """ |
no test coverage detected