Check if this is the 'max context' doc span for the token.
(doc_spans, cur_span_index, position)
| 48 | |
| 49 | |
| 50 | def _check_is_max_context(doc_spans, cur_span_index, position): |
| 51 | """Check if this is the 'max context' doc span for the token.""" |
| 52 | |
| 53 | # Because of the sliding window approach taken to scoring documents, a single |
| 54 | # token can appear in multiple documents. E.g. |
| 55 | # Doc: the man went to the store and bought a gallon of milk |
| 56 | # Span A: the man went to the |
| 57 | # Span B: to the store and bought |
| 58 | # Span C: and bought a gallon of |
| 59 | # ... |
| 60 | # |
| 61 | # Now the word 'bought' will have two scores from spans B and C. We only |
| 62 | # want to consider the score with "maximum context", which we define as |
| 63 | # the *minimum* of its left and right context (the *sum* of left and |
| 64 | # right context will always be the same, of course). |
| 65 | # |
| 66 | # In the example the maximum context for 'bought' would be span C since |
| 67 | # it has 1 left context and 3 right context, while span B has 4 left context |
| 68 | # and 0 right context. |
| 69 | best_score = None |
| 70 | best_span_index = None |
| 71 | for (span_index, doc_span) in enumerate(doc_spans): |
| 72 | end = doc_span.start + doc_span.length - 1 |
| 73 | if position < doc_span.start: |
| 74 | continue |
| 75 | if position > end: |
| 76 | continue |
| 77 | num_left_context = position - doc_span.start |
| 78 | num_right_context = end - position |
| 79 | score = min(num_left_context, num_right_context) + 0.01 * doc_span.length |
| 80 | if best_score is None or score > best_score: |
| 81 | best_score = score |
| 82 | best_span_index = span_index |
| 83 | |
| 84 | return cur_span_index == best_span_index |
| 85 | |
| 86 | |
| 87 | def convert_example_to_features(doc_tokens, question_text, tokenizer, max_seq_length, |
no test coverage detected