(text, text_pair)
| 56 | |
| 57 | |
| 58 | def _encode(text, text_pair): |
| 59 | tokenizer = get_tokenizer() |
| 60 | encoded_input = tokenizer(text, text_pair, max_length=args.sample_length, padding='max_length', truncation='only_first') |
| 61 | seq_len = len(encoded_input['input_ids']) |
| 62 | position_ids = torch.arange(seq_len) |
| 63 | return dict(input_ids=encoded_input['input_ids'], position_ids=position_ids.numpy(), token_type_ids=encoded_input['token_type_ids'], attention_mask=encoded_input['attention_mask']) |
| 64 | |
| 65 | from sat.data_utils import load_hf_dataset |
| 66 | def create_dataset_function(path, args): |
no test coverage detected