(
sources,
tokenizer: transformers.PreTrainedTokenizer,
conv_template = "vicuna-1.1",
mask_user = True
)
| 77 | |
| 78 | |
| 79 | def preprocess( |
| 80 | sources, |
| 81 | tokenizer: transformers.PreTrainedTokenizer, |
| 82 | conv_template = "vicuna-1.1", |
| 83 | mask_user = True |
| 84 | ) -> Dict: |
| 85 | |
| 86 | conv = get_conv_template(conv_template) |
| 87 | roles = {"human": conv.roles[0], "gpt": conv.roles[1]} |
| 88 | |
| 89 | # Apply prompt templates |
| 90 | conversations = [] |
| 91 | for i, source in enumerate(sources): |
| 92 | if roles[source[0]["from"]] != conv.roles[0]: |
| 93 | # Skip the first one if it is not from human |
| 94 | source = source[1:] |
| 95 | |
| 96 | conv.messages = [] |
| 97 | for j, sentence in enumerate(source): |
| 98 | role = roles[sentence["from"]] |
| 99 | # assert role == conv.roles[j % 2], f"{i}" |
| 100 | assert role == conv.roles[j % 2], breakpoint() |
| 101 | conv.append_message(role, sentence["value"]) |
| 102 | conversations.append(conv.get_prompt()) |
| 103 | |
| 104 | # Tokenize conversations |
| 105 | # input_ids = tokenizer(conversations[:2],return_tensors="pt",padding="max_length",max_length=4096,truncation=True,).input_ids |
| 106 | # input_ids = tokenizer(conversations[:1][0][:100],return_tensors="pt",padding="max_length",max_length=tokenizer.model_max_length,truncation=True,).input_ids |
| 107 | input_ids = tokenizer( |
| 108 | conversations, |
| 109 | return_tensors="pt", |
| 110 | padding="max_length", |
| 111 | max_length=tokenizer.model_max_length, |
| 112 | truncation=True, |
| 113 | ).input_ids |
| 114 | |
| 115 | # input_ids = tokenizer(conversations[:2],return_tensors="pt",padding="max_length",max_length=tokenizer.model_max_length,truncation=True,).input_ids |
| 116 | |
| 117 | # block_size = args.pt_context_len |
| 118 | # column_names = list(dataset["train"].features) |
| 119 | # text_column_name = "text" if "text" in column_names else column_names[0] |
| 120 | |
| 121 | # def tokenize_function(examples): |
| 122 | # output = tokenizer(examples[text_column_name]) |
| 123 | # return output |
| 124 | # tokenized_datasets = dataset.map( |
| 125 | # tokenize_function, |
| 126 | # batched=True, |
| 127 | # remove_columns=column_names, |
| 128 | # num_proc=args.preprocessing_num_workers, |
| 129 | # load_from_cache_file=not args.overwrite_cache, |
| 130 | # desc="Running tokenizer on dataset", |
| 131 | # ) |
| 132 | |
| 133 | |
| 134 | targets = input_ids.clone() |
| 135 | |
| 136 | assert (conv.sep_style == SeparatorStyle.ADD_COLON_TWO) or (conv.sep_style == SeparatorStyle.CHATML) or (conv.sep_style == SeparatorStyle.LLAMA2) |
no test coverage detected