Processes HoVeR splits. Given claims and accompanying labels, tokenizes the claims and annotates claim with mention positions corresponding to noun chunks. Args: data: List of raw HoVeR samples. Each raw HoVeR sample contains: claim, text that describes the claim to be classified.
(data: Sequence[Dict[Text, Any]], spacy_model: Any,
tokenizer: Any)
| 47 | |
| 48 | |
| 49 | def process_data(data: Sequence[Dict[Text, Any]], spacy_model: Any, |
| 50 | tokenizer: Any): |
| 51 | """Processes HoVeR splits. |
| 52 | |
| 53 | Given claims and accompanying labels, tokenizes the claims and annotates |
| 54 | claim with mention positions corresponding to noun chunks. |
| 55 | |
| 56 | Args: |
| 57 | data: List of raw HoVeR samples. Each raw HoVeR sample contains: claim, text |
| 58 | that describes the claim to be classified. label, string label, SUPPORTED |
| 59 | or NOT_SUPPORTED. |
| 60 | spacy_model: Spacy model object. |
| 61 | tokenizer: BERT tokenizer. |
| 62 | |
| 63 | Returns: |
| 64 | List of processed HoVeR samples. Each sample contains: |
| 65 | text_ids: dense token ids. |
| 66 | text_mask: dense padding mask. |
| 67 | mention_start_positions: sparse array of mention starting positions. |
| 68 | mention_end_positions: sparse array of mention end positions. |
| 69 | mention_mask: mention padding mask. |
| 70 | target: string labels converted to 0 or 1. |
| 71 | """ |
| 72 | processed_data = [] |
| 73 | for sample in tqdm.tqdm(data): |
| 74 | new_sample = {} |
| 75 | parsed_claim = spacy_model(sample['claim']) |
| 76 | mention_char_spans = [] |
| 77 | for chunk in parsed_claim.noun_chunks: |
| 78 | span_start_char = parsed_claim[chunk.start].idx |
| 79 | span_last_token = parsed_claim[chunk.end - 1] |
| 80 | span_end_char = span_last_token.idx + len(span_last_token.text) - 1 |
| 81 | mention_char_spans.append((span_start_char, span_end_char)) |
| 82 | |
| 83 | _, text_ids, text_mask, mention_spans, _ = tokenization_utils.tokenize_with_mention_spans( |
| 84 | tokenizer=tokenizer, |
| 85 | sentence=sample['claim'], |
| 86 | spans=mention_char_spans, |
| 87 | max_length=FLAGS.max_length, |
| 88 | add_bert_tokens=True, |
| 89 | allow_truncated_spans=True) |
| 90 | |
| 91 | new_sample['text_ids'] = text_ids |
| 92 | new_sample['text_mask'] = text_mask |
| 93 | |
| 94 | mention_spans = np.array(mention_spans) |
| 95 | if mention_spans.size == 0: |
| 96 | mention_start_positions = np.array([], dtype=np.int64) |
| 97 | mention_end_positions = np.array([], dtype=np.int64) |
| 98 | else: |
| 99 | mention_start_positions = mention_spans[:, 0] |
| 100 | mention_end_positions = mention_spans[:, 1] |
| 101 | |
| 102 | mention_start_positions = mention_start_positions[:FLAGS.max_mentions] |
| 103 | mention_end_positions = mention_end_positions[:FLAGS.max_mentions] |
| 104 | |
| 105 | mention_pad_shape = (0, FLAGS.max_mentions - len(mention_start_positions)) |
| 106 |