(text)
| 75 | sp_tokenizer = get_sentencepiece_tokenizer(vocab_path) |
| 76 | |
| 77 | def tokenizer(text): |
| 78 | split_on_punctuation = re.split( |
| 79 | r"\s+|\s*([%s])\s*" % re.escape(string.punctuation), text) |
| 80 | spaced = " ".join(x for x in split_on_punctuation if x) |
| 81 | tokens = sp_tokenizer.EncodeAsPieces(spaced) |
| 82 | tokens = [t.lstrip("▁") for t in tokens if t and t != "▁"] |
| 83 | return tokens |
| 84 | |
| 85 | return tokenizer |
| 86 |
no test coverage detected
searching dependent graphs…