| 8 | |
| 9 | |
| 10 | def load_and_cache_examples(data_file, local_rank, max_seq_length, tokenizer, evaluate=False, |
| 11 | input_label="Body", target_label='Linear_Formula'): |
| 12 | |
| 13 | if local_rank not in [-1, 0] and not evaluate: |
| 14 | torch.distributed.barrier() # Make sure only the first process in distributed training process the dataset, and the others will use the cache |
| 15 | nr_examples = 0 |
| 16 | examples = [] |
| 17 | labels = [] |
| 18 | for i, pair in enumerate(json.load(open(data_file, 'r'))): |
| 19 | if (not "Body" in pair) or not pair[input_label]: |
| 20 | if evaluate == False: |
| 21 | text_a = pair['Body'] + pair['Question'] |
| 22 | else: |
| 23 | text_a = pair['Body'] #+ pair['Question'] |
| 24 | else: |
| 25 | if evaluate == False: |
| 26 | text_a = pair["Body"] + pair["Question"] |
| 27 | else: |
| 28 | text_a = pair[input_label] #+ pair['Question'] |
| 29 | |
| 30 | label = str(pair[target_label]) |
| 31 | label = label.replace("'","") |
| 32 | |
| 33 | try: |
| 34 | encoded_reconstr_code = get_encoded_code_tokens(label) |
| 35 | except: |
| 36 | print("Error related to brackets", label) |
| 37 | continue |
| 38 | |
| 39 | label = ' '.join(encoded_reconstr_code) |
| 40 | labels.append(label) |
| 41 | guid = str(i) |
| 42 | ex = InputExample(guid=guid, text_a=text_a, text_b=None, label=label) |
| 43 | examples.append(ex) |
| 44 | nr_examples += 1 |
| 45 | #if evaluate==False and nr_examples==1000: |
| 46 | # break |
| 47 | print('number of examples:', nr_examples) |
| 48 | tokenized_inputs = tokenizer.batch_encode_plus( |
| 49 | [ex.text_a for ex in examples], |
| 50 | padding="longest", |
| 51 | max_length=max_seq_length, |
| 52 | pad_to_max_length = True, |
| 53 | truncation=True, |
| 54 | return_tensors="pt", |
| 55 | ) |
| 56 | # tokenize targets |
| 57 | tokenized_targets = tokenizer.batch_encode_plus( |
| 58 | [ex.label for ex in examples], |
| 59 | padding='longest', |
| 60 | max_length=max_seq_length, |
| 61 | pad_to_max_length = True, |
| 62 | truncation=True, |
| 63 | return_tensors="pt", |
| 64 | ) |
| 65 | |
| 66 | if local_rank == 0 and not evaluate: |
| 67 | torch.distributed.barrier() # Make sure only the first process in distributed training process the dataset, and the others will use the cache |