Function make_data

train.py:34–61

(args, tokenizer)

def make_data(args, tokenizer):
    # train_json = [json.loads(line) for line in open(args.train_data_path).readlines()]
    # val_json = [json.loads(line) for line in open(args.val_data_path).readlines()]
    # test_json = [json.loads(line) for line in open(args.val_data_path).readlines()]
    train_json = json.load(open(args.train_data_path))
    test_json = json.load(open(args.val_data_path))
    val_json = json.load(open(args.val_data_path))
    label2id = json.load(open(args.label_path))

    def process_data(data):
        features = []
        for example in data:
            feature = tokenizer(
                example["text"],
                padding="max_length",
                max_length=args.max_seq_length,
                return_token_type_ids=True,
                truncation=True
            )
            feature["labels"] = label2id[example["label"]]
            features.append(feature)

        return features

    train_features = process_data(train_json)
    val_features = process_data(val_json)
    test_features = process_data(test_json)
    return myDataset(train_features), myDataset(val_features), myDataset(test_features)


model_path = "./bert-base-uncased/"   #https://huggingface.co/google-bert/bert-base-uncased