Process WebRED split and updates relation vocabulary with new relations.
(
raw_samples: List[Dict[str, Any]],
relation_vocab: Dict[str, int],
spacy_model: Any,
tokenizer: Any,
)
| 243 | |
| 244 | |
| 245 | def process_data( |
| 246 | raw_samples: List[Dict[str, Any]], |
| 247 | relation_vocab: Dict[str, int], |
| 248 | spacy_model: Any, |
| 249 | tokenizer: Any, |
| 250 | ) -> Tuple[List[Dict[str, Any]], Dict[str, int]]: |
| 251 | """Process WebRED split and updates relation vocabulary with new relations.""" |
| 252 | |
| 253 | skipped = 0 |
| 254 | processed_samples = [] |
| 255 | for sample in tqdm.tqdm(raw_samples): |
| 256 | processed_sample, relation_vocab = process_sample( |
| 257 | sample=sample, |
| 258 | relation_vocab=relation_vocab, |
| 259 | spacy_model=spacy_model, |
| 260 | tokenizer=tokenizer, |
| 261 | ) |
| 262 | if processed_sample is not None: |
| 263 | processed_samples.append(processed_sample) |
| 264 | else: |
| 265 | skipped += 1 |
| 266 | |
| 267 | logging.info('Skipped %s samples', skipped) |
| 268 | |
| 269 | return processed_samples, relation_vocab |
| 270 | |
| 271 | |
| 272 | def main(argv: Sequence[str]) -> None: |
no test coverage detected
searching dependent graphs…