(conversations, tokenizer, args)
| 25 | |
| 26 | def process_file(lines, rank, args): |
| 27 | def build_input(conversations, tokenizer, args): |
| 28 | zero_width_chars = ["\u200b", "\u200c", "\u200d", "\ufeff"] # filter null characters |
| 29 | for conv in conversations: |
| 30 | if conv['role'] == "assistant": |
| 31 | for char in zero_width_chars: |
| 32 | conv['content'] = conv['content'].replace(char, '') |
| 33 | |
| 34 | if len(conversations) == 0: |
| 35 | return None |
| 36 | |
| 37 | input_ids = [] |
| 38 | starts = [] |
| 39 | ends = [] |
| 40 | for item in conversations: |
| 41 | content = item["content"] |
| 42 | role = item["role"] |
| 43 | if role == 'assistant' and content != '': |
| 44 | starts.append(len(input_ids)) |
| 45 | input_ids.extend(tokenizer.build_single_message(role, item.get("metadata", ""), content)) |
| 46 | if role == 'assistant' and content != '': |
| 47 | ends.append(len(input_ids)) |
| 48 | input_ids.append(EOS_ID) |
| 49 | input_ids = tokenizer.batch_encode_plus([input_ids], return_tensors="pt", is_split_into_words=True) |
| 50 | inputs = input_ids.input_ids[0] |
| 51 | labels = torch.full_like(inputs, -100) |
| 52 | for start, end in zip(starts, ends): |
| 53 | labels[start+3:end+3] = inputs[start+3:end+3] |
| 54 | |
| 55 | if inputs.shape[0] > max_length: |
| 56 | print("exceed_length") |
| 57 | if skip_exceed_length_case: |
| 58 | return None |
| 59 | if truncate_side == 'right': |
| 60 | inputs = inputs[:max_length] |
| 61 | labels = labels[:max_length] |
| 62 | elif truncate_side == 'left': |
| 63 | cut_num = inputs.shape[0] - max_length |
| 64 | inputs = torch.cat([inputs[:2], inputs[2 + cut_num:]], dim=0) |
| 65 | labels = torch.cat([labels[:2], labels[2 + cut_num:]], dim=0) |
| 66 | else: |
| 67 | raise ValueError('truncate_side must be "right" or "left"') |
| 68 | return inputs, labels |
| 69 | |
| 70 | try: |
| 71 | final_inputs = torch.full((len(lines), max_length), PAD_ID, dtype=torch.int64) |
no test coverage detected