Project the tokenized prediction back to the original text.
(pred_text, orig_text, do_lower_case, verbose=False)
| 791 | |
| 792 | |
| 793 | def get_final_text(pred_text, orig_text, do_lower_case, verbose=False): |
| 794 | """Project the tokenized prediction back to the original text.""" |
| 795 | |
| 796 | # When we created the data, we kept track of the alignment between original |
| 797 | # (whitespace tokenized) tokens and our WordPiece tokenized tokens. So |
| 798 | # now `orig_text` contains the span of our original text corresponding to the |
| 799 | # span that we predicted. |
| 800 | # |
| 801 | # However, `orig_text` may contain extra characters that we don't want in |
| 802 | # our prediction. |
| 803 | # |
| 804 | # For example, let's say: |
| 805 | # pred_text = steve smith |
| 806 | # orig_text = Steve Smith's |
| 807 | # |
| 808 | # We don't want to return `orig_text` because it contains the extra "'s". |
| 809 | # |
| 810 | # We don't want to return `pred_text` because it's already been normalized |
| 811 | # (the SQuAD eval script also does punctuation stripping/lower casing but |
| 812 | # our tokenizer does additional normalization like stripping accent |
| 813 | # characters). |
| 814 | # |
| 815 | # What we really want to return is "Steve Smith". |
| 816 | # |
| 817 | # Therefore, we have to apply a semi-complicated alignment heruistic between |
| 818 | # `pred_text` and `orig_text` to get a character-to-character alignment. This |
| 819 | # can fail in certain cases in which case we just return `orig_text`. |
| 820 | |
| 821 | def _strip_spaces(text): |
| 822 | ns_chars = [] |
| 823 | ns_to_s_map = collections.OrderedDict() |
| 824 | for (i, c) in enumerate(text): |
| 825 | if c == " ": |
| 826 | continue |
| 827 | ns_to_s_map[len(ns_chars)] = i |
| 828 | ns_chars.append(c) |
| 829 | ns_text = "".join(ns_chars) |
| 830 | return (ns_text, ns_to_s_map) |
| 831 | |
| 832 | # We first tokenize `orig_text`, strip whitespace from the result |
| 833 | # and `pred_text`, and check if they are the same length. If they are |
| 834 | # NOT the same length, the heuristic has failed. If they are the same |
| 835 | # length, we assume the characters are one-to-one aligned. |
| 836 | tokenizer = tokenization.BasicTokenizer(do_lower_case=do_lower_case) |
| 837 | |
| 838 | tok_text = " ".join(tokenizer.tokenize(orig_text)) |
| 839 | |
| 840 | start_position = tok_text.find(pred_text) |
| 841 | if start_position == -1: |
| 842 | if verbose: |
| 843 | logging.info("Unable to find text: '%s' in '%s'", pred_text, orig_text) |
| 844 | return orig_text |
| 845 | end_position = start_position + len(pred_text) - 1 |
| 846 | |
| 847 | (orig_ns_text, orig_ns_to_s_map) = _strip_spaces(orig_text) |
| 848 | (tok_ns_text, tok_ns_to_s_map) = _strip_spaces(tok_text) |
| 849 | |
| 850 | if len(orig_ns_text) != len(tok_ns_text): |
no test coverage detected