(
sources: Sequence[str],
tokenizer: transformers.PreTrainedTokenizer,
has_image: bool = False
)
| 248 | |
| 249 | |
| 250 | def preprocess( |
| 251 | sources: Sequence[str], |
| 252 | tokenizer: transformers.PreTrainedTokenizer, |
| 253 | has_image: bool = False |
| 254 | ) -> Dict: |
| 255 | if conversation_lib.default_conversation.sep_style == conversation_lib.SeparatorStyle.PLAIN: |
| 256 | return preprocess_plain(sources, tokenizer) |
| 257 | |
| 258 | if conversation_lib.default_conversation.version == "colongpt": |
| 259 | return preprocess_colongpt(sources, tokenizer, has_image=has_image) |
| 260 | |
| 261 | |
| 262 | class LazySupervisedDataset(Dataset): |
no test coverage detected