Tokenize the full dataset. Parameters ------------ dataset : lmflow.datasets.Dataset. args : Optional. Positional arguments. kwargs : Optional. Keyword arguments. Returns ------------ tokenized_datas
(self, dataset: Dataset, add_special_tokens=True, *args, **kwargs)
| 81 | HFModelMixin.__init__(self, model_args=model_args, do_train=do_train, device=device, **kwargs) |
| 82 | |
| 83 | def tokenize(self, dataset: Dataset, add_special_tokens=True, *args, **kwargs) -> Dataset: |
| 84 | """ |
| 85 | Tokenize the full dataset. |
| 86 | |
| 87 | Parameters |
| 88 | ------------ |
| 89 | dataset : lmflow.datasets.Dataset. |
| 90 | |
| 91 | args : Optional. |
| 92 | Positional arguments. |
| 93 | |
| 94 | kwargs : Optional. |
| 95 | Keyword arguments. |
| 96 | |
| 97 | Returns |
| 98 | ------------ |
| 99 | tokenized_datasets : |
| 100 | The tokenized dataset, without any leading or trailing special |
| 101 | tokens (normally they are Begin-Of-Sentence or End-Of-Sentence |
| 102 | tokens). |
| 103 | """ |
| 104 | # Preprocessing the datasets. |
| 105 | # First we tokenize all the texts. |
| 106 | if dataset.get_backend() != "huggingface": |
| 107 | raise NotImplementedError("tokenization of datasets with non-huggingface backend arenot supported yet") |
| 108 | |
| 109 | dataset_type = dataset.get_type() |
| 110 | model_args = self.model_args |
| 111 | raw_datasets = dataset |
| 112 | hf_raw_datasets = dataset.get_backend_dataset() |
| 113 | column_names = list(hf_raw_datasets.features) |
| 114 | data_args = raw_datasets.get_data_args() |
| 115 | |
| 116 | # Requires three types of information for tokenizing different datasets |
| 117 | # 1) Which fields require tokenization, e.g. |
| 118 | # "text2float": "text", but not "float" |
| 119 | # "text2text": both "input" and "output" |
| 120 | # 2) How will there tokenized sequence concatenated together, e.g. |
| 121 | # "text_only": "text" -> "text" |
| 122 | # "text2text": "input", "output" -> "input" + "output" |
| 123 | # 3) Which fields require loss in final computation, e.g. |
| 124 | # "text_only": "text" |
| 125 | # "text2text": "output" only |
| 126 | tokenized_column_order = None # Handles 1) and 2) |
| 127 | label_columns = None # Handles 3) |
| 128 | if dataset_type == "text_only": |
| 129 | tokenized_column_order = ["text"] |
| 130 | label_columns = ["text"] |
| 131 | elif dataset_type == "text2text": |
| 132 | tokenized_column_order = ["input", "output"] |
| 133 | label_columns = ["output"] |
| 134 | add_special_tokens = False |
| 135 | elif dataset_type == "conversation": |
| 136 | if data_args.conversation_template: |
| 137 | if data_args.conversation_template in PRESET_TEMPLATES.keys(): |
| 138 | conversation_template = PRESET_TEMPLATES[data_args.conversation_template] |
| 139 | else: |
| 140 | raise NotImplementedError( |