Class for an example in a preference or SFT dataset. If you want each prompt to be uniquely associated with an Example instance, save it in a dict.
| 36 | |
| 37 | @dataclass |
| 38 | class Example: |
| 39 | """ |
| 40 | Class for an example in a preference or SFT dataset. If you want each prompt to be uniquely associated with an Example instance, save it in a dict. |
| 41 | """ |
| 42 | prompt: List = field(default_factory=list) # list of turns, each with two keys: "role" and "content" |
| 43 | prompt_id: int = -1 # unique identifier for prompt (optional) |
| 44 | generations: List = field(default_factory=list) # list of list of turns (the output sequences to predict) |
| 45 | sft_index: int = -1 # which response in self.generations should be generated for SFT |
| 46 | scores: List[float] = field(default_factory=list) # score for each generation |
| 47 | pairs: List[Tuple[int, int]] = field(default_factory=list) # for preference feedback data: indices in responses, where i > j in pair (i,j) is a preference |
| 48 | desirable: List[bool] = field(default_factory=list) # for binary feedback data: whether the generation at the corresponding index in self.generations is desirable |
| 49 | dataset_name: str = '' |
| 50 | original_prompt: str = '' # the unformatted prompt (needed to recover instruction for AlpacaEval) |
| 51 | |
| 52 | def __setattr__(self, name, value): |
| 53 | """Set prompt ID automatically.""" |
| 54 | if name == 'prompt' and value is not None: |
| 55 | content = '' |
| 56 | |
| 57 | for turn in value: |
| 58 | if "role" not in turn: |
| 59 | raise ValueError("every turn in an example must have a 'role' field") |
| 60 | |
| 61 | if "content" not in turn: |
| 62 | raise ValueError("every turn in an example must have a 'content' field") |
| 63 | |
| 64 | content = content + turn['content'] |
| 65 | |
| 66 | self.prompt_id = hashlib.sha256(content.encode()).hexdigest() |
| 67 | |
| 68 | super().__setattr__(name, value) |
| 69 | |
| 70 | def __getitem__(self, key): |
| 71 | """ |
| 72 | Get an attribute of the Example by its key. |
| 73 | |
| 74 | Args: |
| 75 | key: The attribute name to retrieve |
| 76 | |
| 77 | Returns: |
| 78 | The value of the attribute |
| 79 | |
| 80 | Raises: |
| 81 | AttributeError: If the attribute doesn't exist |
| 82 | """ |
| 83 | if hasattr(self, key): |
| 84 | return getattr(self, key) |
| 85 | else: |
| 86 | raise AttributeError(f"'{self.__class__.__name__}' object has no attribute '{key}'") |
| 87 | |
| 88 | def num_generations(self): |
| 89 | return len(self.generations) |
| 90 | |
| 91 | def remove_extra_spaces(self): |
| 92 | """ |
| 93 | Remove double spaces in the prompt and generations to standardize spacing. |
| 94 | """ |
| 95 | def clean(text: str) -> str: |