Processor for the X-Stance data set.
| 1113 | |
| 1114 | |
| 1115 | class XStanceProcessor(DataProcessor): |
| 1116 | """Processor for the X-Stance data set.""" |
| 1117 | |
| 1118 | def __init__(self, args, language: str = None): |
| 1119 | super().__init__(args) |
| 1120 | if language is not None: |
| 1121 | assert language in ['de', 'fr'] |
| 1122 | self.language = language |
| 1123 | |
| 1124 | def get_train_examples(self, data_dir): |
| 1125 | return self._create_examples(os.path.join(data_dir, "train.jsonl")) |
| 1126 | |
| 1127 | def get_dev_examples(self, data_dir, for_train=False): |
| 1128 | return self._create_examples(os.path.join(data_dir, "test.jsonl")) |
| 1129 | |
| 1130 | def get_test_examples(self, data_dir) -> List[InputExample]: |
| 1131 | raise NotImplementedError() |
| 1132 | |
| 1133 | def get_unlabeled_examples(self, data_dir) -> List[InputExample]: |
| 1134 | return self.get_train_examples(data_dir) |
| 1135 | |
| 1136 | def get_labels(self): |
| 1137 | return ["FAVOR", "AGAINST"] |
| 1138 | |
| 1139 | def _create_examples(self, path: str) -> List[InputExample]: |
| 1140 | examples = [] |
| 1141 | |
| 1142 | with open(path, encoding='utf8') as f: |
| 1143 | for line in f: |
| 1144 | example_json = json.loads(line) |
| 1145 | label = example_json['label'] |
| 1146 | id_ = example_json['id'] |
| 1147 | text_a = punctuation_standardization(example_json['question']) |
| 1148 | text_b = punctuation_standardization(example_json['comment']) |
| 1149 | language = example_json['language'] |
| 1150 | |
| 1151 | if self.language is not None and language != self.language: |
| 1152 | continue |
| 1153 | |
| 1154 | example = InputExample(guid=id_, text_a=text_a, text_b=text_b, label=label) |
| 1155 | examples.append(example) |
| 1156 | |
| 1157 | return examples |
| 1158 | |
| 1159 | |
| 1160 | class Sst2Processor(DataProcessor): |