Function _is_end_of_word

bert/tokenization_utils.py:86–89

Checks whether the last character in text is one of a punctuation, control or whitespace character.

(text)

def _is_end_of_word(text):
    """Checks whether the last character in text is one of a punctuation, control or whitespace character."""
    last_char = text[-1]
    return bool(_is_control(last_char) | _is_punctuation(last_char) | _is_whitespace(last_char))


def _is_start_of_word(text):