(max_sample)
| 109 | return extract_random_dataset(sources, targets, max_sample) |
| 110 | |
| 111 | def get_redpajama_dataset(max_sample): |
| 112 | # wiki_dataset = load_dataset("wikitext", 'wikitext-2-raw-v1', split='train') |
| 113 | wiki_dataset = load_dataset('/root/model/datasets/RedPajama-Data-1T-Sample', split='train') |
| 114 | |
| 115 | wiki_long = [] |
| 116 | for text in wiki_dataset['text']: |
| 117 | if len(text) > 512: |
| 118 | wiki_long.append(text) |
| 119 | wiki_front = [long[:128] for long in wiki_long] |
| 120 | |
| 121 | targets = sources = wiki_front |
| 122 | |
| 123 | return extract_random_dataset(sources, targets, max_sample) |
| 124 | |
| 125 | |
| 126 |
no test coverage detected