(text, text_pair)
| 70 | return total_loss, {'acc': acc} |
| 71 | |
| 72 | def _encode(text, text_pair): |
| 73 | tokenizer = get_tokenizer() |
| 74 | encoded_input = tokenizer(text, text_pair, max_length=args.sample_length, padding='max_length', truncation='only_first') |
| 75 | seq_len = len(encoded_input['input_ids']) |
| 76 | position_ids = torch.arange(seq_len) |
| 77 | return dict(input_ids=encoded_input['input_ids'], position_ids=position_ids.numpy(), token_type_ids=encoded_input['token_type_ids'], attention_mask=encoded_input['attention_mask']) |
| 78 | |
| 79 | from sat.data_utils import load_hf_dataset |
| 80 | def create_dataset_function(path, args): |
no test coverage detected