split document into sentences
(self, document)
| 924 | return sample |
| 925 | |
| 926 | def sentence_split(self, document): |
| 927 | """split document into sentences""" |
| 928 | lines = document.split('\n') |
| 929 | if self.presplit_sentences: |
| 930 | return [line for line in lines if line] |
| 931 | rtn = [] |
| 932 | for line in lines: |
| 933 | if line != '': |
| 934 | rtn.extend(tokenize.sent_tokenize(line)) |
| 935 | return rtn |
| 936 | |
| 937 | def sentence_tokenize(self, sent, sentence_num=0, beginning=False, ending=False): |
| 938 | """tokenize sentence and get token types""" |
no test coverage detected