| 106 | from transformers.models.roberta.modeling_roberta import create_position_ids_from_input_ids |
| 107 | |
| 108 | def _encode(text, text_pair): |
| 109 | encoded_input = tokenizer(text, text_pair, max_length=args.sample_length, padding='max_length', truncation='only_first') |
| 110 | position_ids = create_position_ids_from_input_ids(torch.tensor([encoded_input['input_ids']]), 1, 0) |
| 111 | return dict(input_ids=encoded_input['input_ids'], position_ids=position_ids[0].numpy(), attention_mask=encoded_input['attention_mask']) |
| 112 | |
| 113 | from sat.data_utils import load_hf_dataset |
| 114 | def create_dataset_function(path, args): |