r""" Initializes a HFDecoderModel instance. Parameters ------------ model_args : Model arguments such as model name, path, revision, etc. do_train : bool, default True Determines whether to prepare the model for training, including distribtued env, model placem
| 57 | |
| 58 | |
| 59 | class HFDecoderModel(DecoderModel, HFModelMixin, Tunable): |
| 60 | r""" |
| 61 | Initializes a HFDecoderModel instance. |
| 62 | |
| 63 | Parameters |
| 64 | ------------ |
| 65 | |
| 66 | model_args : |
| 67 | Model arguments such as model name, path, revision, etc. |
| 68 | |
| 69 | do_train : bool, default True |
| 70 | Determines whether to prepare the model for training, including distribtued env, model placement, quantization, |
| 71 | lora, etc. |
| 72 | |
| 73 | args : Optional. |
| 74 | Positional arguments. |
| 75 | |
| 76 | kwargs : Optional. |
| 77 | Keyword arguments. |
| 78 | """ |
| 79 | |
| 80 | def __init__(self, model_args, do_train=True, device="gpu", **kwargs): |
| 81 | HFModelMixin.__init__(self, model_args=model_args, do_train=do_train, device=device, **kwargs) |
| 82 | |
| 83 | def tokenize(self, dataset: Dataset, add_special_tokens=True, *args, **kwargs) -> Dataset: |
| 84 | """ |
| 85 | Tokenize the full dataset. |
| 86 | |
| 87 | Parameters |
| 88 | ------------ |
| 89 | dataset : lmflow.datasets.Dataset. |
| 90 | |
| 91 | args : Optional. |
| 92 | Positional arguments. |
| 93 | |
| 94 | kwargs : Optional. |
| 95 | Keyword arguments. |
| 96 | |
| 97 | Returns |
| 98 | ------------ |
| 99 | tokenized_datasets : |
| 100 | The tokenized dataset, without any leading or trailing special |
| 101 | tokens (normally they are Begin-Of-Sentence or End-Of-Sentence |
| 102 | tokens). |
| 103 | """ |
| 104 | # Preprocessing the datasets. |
| 105 | # First we tokenize all the texts. |
| 106 | if dataset.get_backend() != "huggingface": |
| 107 | raise NotImplementedError("tokenization of datasets with non-huggingface backend arenot supported yet") |
| 108 | |
| 109 | dataset_type = dataset.get_type() |
| 110 | model_args = self.model_args |
| 111 | raw_datasets = dataset |
| 112 | hf_raw_datasets = dataset.get_backend_dataset() |
| 113 | column_names = list(hf_raw_datasets.features) |
| 114 | data_args = raw_datasets.get_data_args() |
| 115 | |
| 116 | # Requires three types of information for tokenizing different datasets |
no outgoing calls