Format original data Args: ori_data (dict): input data sample. Returns: dict: data sample with chat format.
(ori_data)
| 26 | |
| 27 | |
| 28 | def get_chat_format_data(ori_data): |
| 29 | """Format original data |
| 30 | |
| 31 | Args: |
| 32 | ori_data (dict): input data sample. |
| 33 | |
| 34 | Returns: |
| 35 | dict: data sample with chat format. |
| 36 | """ |
| 37 | input_str = ori_data["input"] |
| 38 | instruction_str = ori_data["instruction"] |
| 39 | output_str = ori_data["output"] |
| 40 | data = dict() |
| 41 | if input_str != "": |
| 42 | data["user"] = f"<|User|>:{instruction_str}\n{input_str}" |
| 43 | else: |
| 44 | data["user"] = f"<|User|>:{instruction_str}" |
| 45 | data["bot"] = f"<|Bot|>:{output_str}" |
| 46 | return data |
| 47 | |
| 48 | |
| 49 | def tokenize(sample, sp_model): |