tokenize sentence and get token types
(self, sent, sentence_num=0, beginning=False, ending=False)
| 935 | return rtn |
| 936 | |
| 937 | def sentence_tokenize(self, sent, sentence_num=0, beginning=False, ending=False): |
| 938 | """tokenize sentence and get token types""" |
| 939 | tokens = self.tokenizer.EncodeAsIds(sent).tokenization |
| 940 | str_type = 'str' + str(sentence_num) |
| 941 | token_types = [self.tokenizer.get_type(str_type).Id] * len(tokens) |
| 942 | return tokens, token_types |
| 943 | |
| 944 | def get_doc(self, idx): |
| 945 | """gets text of document corresponding to idx""" |
no test coverage detected