Segements text into pieces. This method is used together with sentence piece tokenizer and is forked from: https://github.com/google-research/google-research/blob/e1f6fa00/albert/tokenization.py Args: sp_model: A spm.SentencePieceProcessor object. text: The input text to be segement
(sp_model, text, sample=False)
| 447 | |
| 448 | |
| 449 | def encode_pieces(sp_model, text, sample=False): |
| 450 | """Segements text into pieces. |
| 451 | |
| 452 | This method is used together with sentence piece tokenizer and is forked from: |
| 453 | https://github.com/google-research/google-research/blob/e1f6fa00/albert/tokenization.py |
| 454 | |
| 455 | |
| 456 | Args: |
| 457 | sp_model: A spm.SentencePieceProcessor object. |
| 458 | text: The input text to be segemented. |
| 459 | sample: Whether to randomly sample a segmentation output or return a |
| 460 | deterministic one. |
| 461 | |
| 462 | Returns: |
| 463 | A list of token pieces. |
| 464 | """ |
| 465 | if six.PY2 and isinstance(text, six.text_type): |
| 466 | text = six.ensure_binary(text, "utf-8") |
| 467 | |
| 468 | if not sample: |
| 469 | pieces = sp_model.EncodeAsPieces(text) |
| 470 | else: |
| 471 | pieces = sp_model.SampleEncodeAsPieces(text, 64, 0.1) |
| 472 | new_pieces = [] |
| 473 | for piece in pieces: |
| 474 | piece = printable_text(piece) |
| 475 | if len(piece) > 1 and piece[-1] == "," and piece[-2].isdigit(): |
| 476 | cur_pieces = sp_model.EncodeAsPieces(piece[:-1].replace( |
| 477 | SPIECE_UNDERLINE, "")) |
| 478 | if piece[0] != SPIECE_UNDERLINE and cur_pieces[0][0] == SPIECE_UNDERLINE: |
| 479 | if len(cur_pieces[0]) == 1: |
| 480 | cur_pieces = cur_pieces[1:] |
| 481 | else: |
| 482 | cur_pieces[0] = cur_pieces[0][1:] |
| 483 | cur_pieces.append(piece[-1]) |
| 484 | new_pieces.extend(cur_pieces) |
| 485 | else: |
| 486 | new_pieces.append(piece) |
| 487 | |
| 488 | return new_pieces |
| 489 | |
| 490 | |
| 491 | def encode_ids(sp_model, text, sample=False): |
no test coverage detected