Function _is_whitespace

tasks/seq2seq/dataset.py:236–239

(c)

    # Use heuristics to construct the token to char mapping for BertTokenizer

    def _is_whitespace(c):
        if c == " " or c == "\t" or c == "\r" or c == "\n" or ord(c) == 0x202F:
            return True
        return False

    def _normalize(s):
        # if s in tokenizer.command_token_map: