Process CWQ split.
(
raw_samples: List[Dict[str, Any]],
spacy_model: Any,
tokenizer: Any,
)
| 146 | |
| 147 | |
| 148 | def process_data( |
| 149 | raw_samples: List[Dict[str, Any]], |
| 150 | spacy_model: Any, |
| 151 | tokenizer: Any, |
| 152 | ) -> Tuple[List[Dict[str, Any]], int]: |
| 153 | """Process CWQ split.""" |
| 154 | |
| 155 | processed_samples = [] |
| 156 | skipped = 0 |
| 157 | for sample in tqdm.tqdm(raw_samples): |
| 158 | processed_sample = process_sample(sample, spacy_model, tokenizer) |
| 159 | if processed_sample is None: |
| 160 | skipped += 1 |
| 161 | else: |
| 162 | processed_samples.append(processed_sample) |
| 163 | logging.info('Skipped %s samples', skipped) |
| 164 | return processed_samples, skipped |
| 165 | |
| 166 | |
| 167 | def main(argv: Sequence[str]) -> None: |
no test coverage detected
searching dependent graphs…