()
| 87 | |
| 88 | |
| 89 | def main(): |
| 90 | parser = get_parser() |
| 91 | args = parser.parse_args() |
| 92 | |
| 93 | rs = [] |
| 94 | if args.non_lang_syms is not None: |
| 95 | with codecs.open(args.non_lang_syms, 'r', encoding="utf-8") as f: |
| 96 | nls = [x.rstrip() for x in f.readlines()] |
| 97 | rs = [re.compile(re.escape(x)) for x in nls] |
| 98 | |
| 99 | if args.bpe_model is not None: |
| 100 | import sentencepiece as spm |
| 101 | sp = spm.SentencePieceProcessor() |
| 102 | sp.load(args.bpe_model) |
| 103 | |
| 104 | if args.text: |
| 105 | f = codecs.open(args.text, encoding="utf-8") |
| 106 | else: |
| 107 | f = codecs.getreader("utf-8")( |
| 108 | sys.stdin if is_python2 else sys.stdin.buffer) |
| 109 | |
| 110 | sys.stdout = codecs.getwriter("utf-8")( |
| 111 | sys.stdout if is_python2 else sys.stdout.buffer) |
| 112 | line = f.readline() |
| 113 | n = args.nchar |
| 114 | while line: |
| 115 | x = line.split() |
| 116 | print(' '.join(x[:args.skip_ncols]), end=" ") |
| 117 | a = ' '.join(x[args.skip_ncols:]) |
| 118 | |
| 119 | # get all matched positions |
| 120 | match_pos = [] |
| 121 | for r in rs: |
| 122 | i = 0 |
| 123 | while i >= 0: |
| 124 | m = r.search(a, i) |
| 125 | if m: |
| 126 | match_pos.append([m.start(), m.end()]) |
| 127 | i = m.end() |
| 128 | else: |
| 129 | break |
| 130 | |
| 131 | if len(match_pos) > 0: |
| 132 | chars = [] |
| 133 | i = 0 |
| 134 | while i < len(a): |
| 135 | start_pos, end_pos = exist_or_not(i, match_pos) |
| 136 | if start_pos is not None: |
| 137 | chars.append(a[start_pos:end_pos]) |
| 138 | i = end_pos |
| 139 | else: |
| 140 | chars.append(a[i]) |
| 141 | i += 1 |
| 142 | a = chars |
| 143 | |
| 144 | if (args.trans_type == "phn"): |
| 145 | a = a.split(" ") |
| 146 | elif args.trans_type == "cn_char_en_bpe": |
no test coverage detected