| 71 | from transformers.models.roberta.modeling_roberta import create_position_ids_from_input_ids |
| 72 | |
| 73 | def _encode(text, text_pair): |
| 74 | encoded_input = tokenizer(text, text_pair, max_length=args.sample_length, padding='max_length', truncation='only_first') |
| 75 | position_ids = create_position_ids_from_input_ids(torch.tensor([encoded_input['input_ids']]), 1, 0) |
| 76 | return dict(input_ids=encoded_input['input_ids'], position_ids=position_ids[0].numpy(), attention_mask=encoded_input['attention_mask']) |
| 77 | |
| 78 | from sat.data_utils import load_hf_dataset |
| 79 | def create_dataset_function(path, args): |