Split words. Args: text: Text tensor or string input. jieba_usr_dict: TODO. **kwargs: Additional keyword arguments.
(text: str, jieba_usr_dict=None, **kwargs)
| 27 | |
| 28 | |
| 29 | def split_words(text: str, jieba_usr_dict=None, **kwargs): |
| 30 | """Split words. |
| 31 | |
| 32 | Args: |
| 33 | text: Text tensor or string input. |
| 34 | jieba_usr_dict: TODO. |
| 35 | **kwargs: Additional keyword arguments. |
| 36 | """ |
| 37 | if jieba_usr_dict: |
| 38 | input_list = text.split() |
| 39 | token_list_all = [] |
| 40 | langauge_list = [] |
| 41 | token_list_tmp = [] |
| 42 | language_flag = None |
| 43 | for token in input_list: |
| 44 | if isEnglish(token) and language_flag == "Chinese": |
| 45 | token_list_all.append(token_list_tmp) |
| 46 | langauge_list.append("Chinese") |
| 47 | token_list_tmp = [] |
| 48 | elif not isEnglish(token) and language_flag == "English": |
| 49 | token_list_all.append(token_list_tmp) |
| 50 | langauge_list.append("English") |
| 51 | token_list_tmp = [] |
| 52 | |
| 53 | token_list_tmp.append(token) |
| 54 | |
| 55 | if isEnglish(token): |
| 56 | language_flag = "English" |
| 57 | else: |
| 58 | language_flag = "Chinese" |
| 59 | |
| 60 | if token_list_tmp: |
| 61 | token_list_all.append(token_list_tmp) |
| 62 | langauge_list.append(language_flag) |
| 63 | |
| 64 | result_list = [] |
| 65 | for token_list_tmp, language_flag in zip(token_list_all, langauge_list): |
| 66 | if language_flag == "English": |
| 67 | result_list.extend(token_list_tmp) |
| 68 | else: |
| 69 | seg_list = jieba_usr_dict.cut(join_chinese_and_english(token_list_tmp), HMM=False) |
| 70 | result_list.extend(seg_list) |
| 71 | |
| 72 | return result_list |
| 73 | |
| 74 | else: |
| 75 | words = [] |
| 76 | segs = text.split() |
| 77 | for seg in segs: |
| 78 | # There is no space in seg. |
| 79 | current_word = "" |
| 80 | for c in seg: |
| 81 | if len(c.encode()) == 1: |
| 82 | # This is an ASCII char. |
| 83 | current_word += c |
| 84 | else: |
| 85 | # This is a Chinese char. |
| 86 | if len(current_word) > 0: |
no test coverage detected
searching dependent graphs…