Function _is_start_of_word

bert/tokenization_utils.py:92–95

Checks whether the first character in text is one of a punctuation, control or whitespace character.

(text)

def _is_start_of_word(text):
    """Checks whether the first character in text is one of a punctuation, control or whitespace character."""
    first_char = text[0]
    return bool(_is_control(first_char) | _is_punctuation(first_char) | _is_whitespace(first_char))


class PreTrainedTokenizer(PreTrainedTokenizerBase):