This function tokenizes the source strings given the tokenizer and returns a dictionary containing the tokenized inputs and labels. Args: strings (List[str]): The list of input strings. tokenizer (PreTrainedTokenizer): The tokenizer to use.
(self, strings, tokenizer)
| 99 | return dict(input_ids=input_ids, labels=labels) |
| 100 | |
| 101 | def tokenize(self, strings, tokenizer): |
| 102 | """ |
| 103 | This function tokenizes the source strings given the tokenizer and returns a dictionary containing the |
| 104 | tokenized inputs and labels. |
| 105 | Args: |
| 106 | strings (List[str]): The list of input strings. |
| 107 | tokenizer (PreTrainedTokenizer): The tokenizer to use. |
| 108 | max_input_length (Optional[int]): The maximum length of the input sequences. |
| 109 | max_target_length (Optional[int]): The maximum length of the target sequences. |
| 110 | Returns: |
| 111 | Dict[str, Any]: A dictionary containing input_ids, labels, input_ids_lens and labels_lens. |
| 112 | """ |
| 113 | |
| 114 | tokenized_list = [ |
| 115 | tokenizer( |
| 116 | text, |
| 117 | return_tensors='np', |
| 118 | padding='max_length', |
| 119 | max_length=self.max_padding_length, |
| 120 | truncation=True, |
| 121 | ) for text in strings |
| 122 | ] |
| 123 | input_ids = labels = [ |
| 124 | tokenized.input_ids[0] for tokenized in tokenized_list |
| 125 | ] |
| 126 | input_ids_lens = labels_lens = [ |
| 127 | (tokenized.input_ids != tokenizer.pad_token_id).sum().item() |
| 128 | for tokenized in tokenized_list |
| 129 | ] |
| 130 | return dict( |
| 131 | input_ids=input_ids, |
| 132 | labels=labels, |
| 133 | input_ids_lens=input_ids_lens, |
| 134 | labels_lens=labels_lens, |
| 135 | ) |
| 136 | |
| 137 | def gpt_convert_example_to_feature(self, sample): |
| 138 | input_ids, labels = sample |