(path, data_size)
| 50 | |
| 51 | |
| 52 | def construct_dataset(path, data_size): |
| 53 | dataset = read_jsonl(path) |
| 54 | chunks = [] |
| 55 | for sample in dataset: |
| 56 | output = sample['dialogs'][1]['content'] |
| 57 | think = output.split('</think>')[0] |
| 58 | think = think.replace('<think>', '').strip() |
| 59 | |
| 60 | |
| 61 | thinks = think.split('\n\n') |
| 62 | thinks = think.split(' ') |
| 63 | max_len = 60 |
| 64 | if len(thinks) < max_len: |
| 65 | continue |
| 66 | |
| 67 | for i in range(100): |
| 68 | beg_idx = random.randint(0, len(thinks) - max_len) |
| 69 | chunks.append({ |
| 70 | 'id_ddm': sample['id_ddm'], |
| 71 | 'text': ' '.join(thinks[beg_idx: beg_idx+max_len]) |
| 72 | }) |
| 73 | |
| 74 | thinks = think.split('\n') |
| 75 | if '' in thinks: |
| 76 | thinks.remove('') |
| 77 | |
| 78 | if len(chunks) > data_size: |
| 79 | continue |
| 80 | |
| 81 | if len(chunks) > data_size: |
| 82 | chunks = random.sample(chunks, data_size) |
| 83 | return chunks |
| 84 | |
| 85 | |
| 86 | def build_prompt(think_text: str) -> str: |
no test coverage detected