| 105 | |
| 106 | @staticmethod |
| 107 | def _split_sentences(path): |
| 108 | en_max_len, cn_max_len = 0, 0 |
| 109 | en_sts, cn_sts = [], [] |
| 110 | with open(path, 'r', encoding='utf-8') as f: |
| 111 | for line in f: |
| 112 | line_split = line.split('\t') |
| 113 | line_split[0] = re.sub(r'[^\w\s\'-]', '', line_split[0]) |
| 114 | line_split[0] = line_split[0].lower() |
| 115 | # [\u4e00-\u9fa5] matching Chinese characters |
| 116 | line_split[1] = re.sub("[^\u4e00-\u9fa5]", "", line_split[1]) |
| 117 | |
| 118 | en_stc = line_split[0].split(' ') |
| 119 | cn_stc = [word for word in line_split[1]] |
| 120 | en_sts.append(en_stc) |
| 121 | cn_sts.append(cn_stc) |
| 122 | en_max_len = max(en_max_len, len(en_stc)) |
| 123 | cn_max_len = max(cn_max_len, len(cn_stc)) |
| 124 | return en_max_len, cn_max_len, en_sts, cn_sts |
| 125 | |
| 126 | @staticmethod |
| 127 | def _encoding_stc(src_tokens, tgt_tokens, src_max_len, tgt_max_len, src_vocab, tgt_vocab): |