(prompt_template='chatml')
| 64 | |
| 65 | |
| 66 | def format_rows(prompt_template='chatml'): |
| 67 | if prompt_template == 'chatml': |
| 68 | def _format_rows(row: NectarRow) -> FormattedDatasetRow: |
| 69 | chosen = row['answers'][0]['answer'] |
| 70 | # chosen = choose_top_answer( |
| 71 | # answers=row['answers'], |
| 72 | # filter_out_gpt=False, |
| 73 | # ) |
| 74 | |
| 75 | # https://arxiv.org/pdf/2310.16944 |
| 76 | # We construct binary preferences from UltraFeedback by selecting the |
| 77 | # highest mean score as the “chosen” response and one of the remaining three at random as |
| 78 | # “rejected”. We opted for random selection instead of selecting the lowest-scored response |
| 79 | # to encourage diversity and make the DPO objective more challenging. As noted above, this |
| 80 | # step is computed offline and does not involve any sampling from the reference model. |
| 81 | rejected_pool = row['answers'][1:] |
| 82 | rejected = random.choice(rejected_pool)['answer'] # Randomize the rejected answer |
| 83 | |
| 84 | # The DPOTrainer tokenization will not add a stop token. So we have to add it here ourselves. |
| 85 | # and have the model learn to output the end token. |
| 86 | return { |
| 87 | 'chosen': f'{chosen}<|im_end|>', |
| 88 | 'rejected': f'{rejected}<|im_end|>', |
| 89 | 'prompt': format_prompt(row['prompt'], 'chatml'), |
| 90 | } |
| 91 | |
| 92 | return _format_rows |
| 93 | |
| 94 | raise NotImplementedError(f'Prompt template [{prompt_template}] not implemented.') |
| 95 | |
| 96 | |
| 97 | def choose_top_answer(answers: List[NectarAnswer], filter_out_gpt: bool) -> str: |
no outgoing calls
no test coverage detected