| 33 | |
| 34 | |
| 35 | def read_uttid2text(filename, do_tn=False, rm_special=False): |
| 36 | uttid2text = OrderedDict() |
| 37 | with open(filename, "r", encoding="utf8") as fin: |
| 38 | for i, line in enumerate(fin): |
| 39 | cols = line.split() |
| 40 | if len(cols) == 0: |
| 41 | print("[WARN] empty line, continue", i, flush=True) |
| 42 | continue |
| 43 | assert cols[0] not in uttid2text, f"repeated uttid: {line}" |
| 44 | if len(cols) == 1: |
| 45 | uttid2text[cols[0]] = "" |
| 46 | continue |
| 47 | txt = " ".join(cols[1:]) |
| 48 | if rm_special: |
| 49 | txt = " ".join([t for t in re.split("<\|.*?\|>", txt) if t.strip() != ""]) |
| 50 | if do_tn: |
| 51 | import cn2an |
| 52 | txt = cn2an.transform(txt, "an2cn") |
| 53 | uttid2text[cols[0]] = txt |
| 54 | return uttid2text |
| 55 | |
| 56 | |
| 57 | def text2tokens(text): |