Tokenize the prompt.
(prompt, tokenizer, args, add_eos_token=True)
| 277 | {data_point["output"]}""" |
| 278 | |
| 279 | def tokenize_cr(prompt, tokenizer, args, add_eos_token=True): |
| 280 | """Tokenize the prompt.""" |
| 281 | result = tokenizer( |
| 282 | prompt, |
| 283 | truncation=True, |
| 284 | max_length=args.max_seq_length, |
| 285 | padding=False, |
| 286 | return_tensors=None, |
| 287 | ) |
| 288 | |
| 289 | if ( |
| 290 | result["input_ids"][-1] != tokenizer.eos_token_id |
| 291 | and len(result["input_ids"]) < args.max_seq_length |
| 292 | and add_eos_token |
| 293 | ): |
| 294 | result["input_ids"].append(tokenizer.eos_token_id) |
| 295 | if "chatglm" not in args.model: |
| 296 | result["attention_mask"].append(1) |
| 297 | |
| 298 | result["labels"] = result["input_ids"].copy() |
| 299 | |
| 300 | if "chatglm" in args.model: |
| 301 | return {"input_ids": result["input_ids"], "labels": result["labels"]} |
| 302 | else: |
| 303 | return result |
| 304 | |
| 305 | def generate_and_tokenize_prompt_cr(data_point, tokenizer, args): |
| 306 | """Generate and tokenize prompt with proper labels.""" |
no outgoing calls
no test coverage detected