(examples, tokenizer, query, response)
| 207 | return dict(input_ids=input_ids, labels=labels) |
| 208 | |
| 209 | def train_tokenize_function(examples, tokenizer, query, response): |
| 210 | sources = [PROMPT.format_map(dict(instruction=instruction)) for instruction in examples[query]] |
| 211 | targets = [f"{output}{tokenizer.eos_token}" for output in examples[response]] |
| 212 | data_dict = preprocess(sources, targets, tokenizer) |
| 213 | return data_dict |
| 214 | |
| 215 | def load_and_preprocess_it(tokenizer, args): |
| 216 |
nothing calls this directly
no test coverage detected