File tokenization_utils_base.py

bert/tokenization_utils_base.py:None–None

# coding=utf-8
# Copyright 2020 The HuggingFace Inc. team.
#
# Licensed under the Apache License, Version 2.0 (the "License");