Preprocess the dataset into a list of (text, label) tuples.
(self, split: str)
| 51 | return map(self._process_doc, dataset) |
| 52 | |
| 53 | def _preprocess_dataset(self, split: str): |
| 54 | """Preprocess the dataset into a list of (text, label) tuples.""" |
| 55 | d = pd.DataFrame(self.dataset[split]) |
| 56 | text = d["text"] |
| 57 | labels = np.round(((d["toxicity_ai"] + d["toxicity_human"]) > 5.5), 0).astype( |
| 58 | np.int32 |
| 59 | ) |
| 60 | return [[x, y] for x, y in zip(text, labels)] |
| 61 | |
| 62 | def _process_doc(self, doc): |
| 63 | return { |
no outgoing calls