Preprocesses the raw input text by adding the prefix and tokenizing it. Args: raw_text (str): The raw input text. Returns: Tuple[torch.Tensor, torch.Tensor]: A tuple containing the tokenized input IDs and attention mask.
(self, raw_text: str)
| 125 | return model |
| 126 | |
| 127 | def preprocess(self, raw_text: str) -> Tuple[torch.Tensor, torch.Tensor]: |
| 128 | """ |
| 129 | Preprocesses the raw input text by adding the prefix and tokenizing it. |
| 130 | |
| 131 | Args: |
| 132 | raw_text (str): The raw input text. |
| 133 | |
| 134 | Returns: |
| 135 | Tuple[torch.Tensor, torch.Tensor]: A tuple containing the tokenized input IDs and attention mask. |
| 136 | """ |
| 137 | text = self.prefix + raw_text |
| 138 | |
| 139 | tokens = self.tokenizer.batch_encode_plus([text], return_tensors="pt") |
| 140 | input_ids, attention_mask = tokens['input_ids'], tokens['attention_mask'] |
| 141 | |
| 142 | return input_ids, attention_mask |
| 143 | |
| 144 | def forward( |
| 145 | self, data: str, paras: Optional[Dict[str, float]] = None |