Class SnliProcessor

src/process_data.py:234–258

class SnliProcessor(DataProcessor):
    dataset_project = {"train": "snli_1.0_train",
                        "dev": "snli_1.0_train",
                        "test": "snli_1.0_dev"
                        }
    def __init__(self):
        super().__init__()
        self.labels = ["contradiction", "entailment", "neutral"]

    def get_examples(self, data_dir, split):
        path = os.path.join(data_dir, "{}.jsonl".format(self.dataset_project[split]))
        examples = []
        with open(path)as f:
            lines = f.readlines()
            for idx, line in enumerate(lines):
                data = json.loads(line)
                text_a = data["sentence1"]
                text_b = data["sentence2"]
                if data["gold_label"] not in self.labels:
                    continue
                label = self.get_label_id(data["gold_label"])
                example = InputExample(
                        guid=str(idx), text_a=text_a, text_b=text_b, label=label)
                examples.append(example)
        return examples

class YelpProcessor(DataProcessor):
    dataset_project = {"train": "train",