Processes CWQ sample. To process a raw CWQ example, we first strip special tokens and convert tokens back to text. Then a spacy model extracts noun chunks, and the standard BERT tokenizer tokenizes again. Finally we append a mask token to the text, and add the answer entity id as a mention
(
sample: Dict[str, Any],
spacy_model: Any,
tokenizer: Any,
)
| 42 | |
| 43 | |
| 44 | def process_sample( |
| 45 | sample: Dict[str, Any], |
| 46 | spacy_model: Any, |
| 47 | tokenizer: Any, |
| 48 | ) -> Optional[Dict[str, Any]]: |
| 49 | """Processes CWQ sample. |
| 50 | |
| 51 | To process a raw CWQ example, we first strip special tokens and convert tokens |
| 52 | back to text. Then a spacy model extracts noun chunks, and the standard BERT |
| 53 | tokenizer tokenizes again. Finally we append a mask token to the text, and add |
| 54 | the answer entity id as a mention at the position of the mask token. |
| 55 | |
| 56 | Args: |
| 57 | sample: raw CWQ sample. Needs to contain following fields: text_ids, |
| 58 | target_mention. |
| 59 | spacy_model: spacy model used to extract mentions. |
| 60 | tokenizer: BERT tokenizer. |
| 61 | |
| 62 | Returns: |
| 63 | Processed CWQ sample. |
| 64 | """ |
| 65 | numpy_sample = {key: value.numpy() for key, value in sample.items()} |
| 66 | processed_sample = {} |
| 67 | |
| 68 | text_ids = numpy_sample['text_ids'] |
| 69 | target_mention = numpy_sample['target_mention'] |
| 70 | # Skip samples for which the answer is not present in entity vocabulary |
| 71 | if target_mention == 0: |
| 72 | return None |
| 73 | |
| 74 | text_ids = text_ids[text_ids.nonzero()] |
| 75 | |
| 76 | # Remove special tokens |
| 77 | special_tokens_to_remove = [101, 102, 103, 104, 105, 106] |
| 78 | keep_positions = np.logical_not(np.isin(text_ids, special_tokens_to_remove)) |
| 79 | text_ids = text_ids[keep_positions] |
| 80 | |
| 81 | def detokenize(text_ids, tokenizer): |
| 82 | text_ids = text_ids[np.nonzero(text_ids)] |
| 83 | text_tokens = tokenizer.convert_ids_to_tokens(text_ids) |
| 84 | text_tokens = [ |
| 85 | t[2:] if t.startswith('##') else ' ' + t for t in text_tokens |
| 86 | ] |
| 87 | return ''.join(text_tokens) |
| 88 | |
| 89 | # Convert back to text |
| 90 | text = detokenize(text_ids, tokenizer) |
| 91 | |
| 92 | # Annotate noun chunks |
| 93 | parsed_text = spacy_model(text) |
| 94 | mention_char_spans = [] |
| 95 | for chunk in parsed_text.noun_chunks: |
| 96 | span_start_char = parsed_text[chunk.start].idx |
| 97 | span_last_token = parsed_text[chunk.end - 1] |
| 98 | span_end_char = span_last_token.idx + len(span_last_token.text) - 1 |
| 99 | mention_char_spans.append((span_start_char, span_end_char)) |
| 100 | |
| 101 | # Tokenize again. Set max length to 1 less than max length as we still have to |
no test coverage detected
searching dependent graphs…