Check if the current word piece is the starting piece (BERT).
(piece)
| 161 | |
| 162 | |
| 163 | def is_start_piece(piece): |
| 164 | """Check if the current word piece is the starting piece (BERT).""" |
| 165 | # When a word has been split into |
| 166 | # WordPieces, the first token does not have any marker and any subsequence |
| 167 | # tokens are prefixed with ##. So whenever we see the ## token, we |
| 168 | # append it to the previous set of word indexes. |
| 169 | return not piece.startswith("##") |
| 170 | |
| 171 | |
| 172 | def create_masked_lm_predictions( |
no outgoing calls
no test coverage detected