(data)
| 41 | label2id = json.load(open(args.label_path)) |
| 42 | |
| 43 | def process_data(data): |
| 44 | features = [] |
| 45 | for example in data: |
| 46 | feature = tokenizer( |
| 47 | example["text"], |
| 48 | padding="max_length", |
| 49 | max_length=args.max_seq_length, |
| 50 | return_token_type_ids=True, |
| 51 | truncation=True |
| 52 | ) |
| 53 | feature["labels"] = label2id[example["label"]] |
| 54 | features.append(feature) |
| 55 | |
| 56 | return features |
| 57 | |
| 58 | train_features = process_data(train_json) |
| 59 | val_features = process_data(val_json) |