Generates examples from TMX file.
(path)
| 1041 | |
| 1042 | |
| 1043 | def _parse_tmx(path): |
| 1044 | """Generates examples from TMX file.""" |
| 1045 | |
| 1046 | def _get_tuv_lang(tuv): |
| 1047 | for k, v in tuv.items(): |
| 1048 | if k.endswith("}lang"): |
| 1049 | return v |
| 1050 | raise AssertionError("Language not found in `tuv` attributes.") |
| 1051 | |
| 1052 | def _get_tuv_seg(tuv): |
| 1053 | segs = tuv.findall("seg") |
| 1054 | assert len(segs) == 1, "Invalid number of segments: %d" % len(segs) |
| 1055 | return segs[0].text |
| 1056 | |
| 1057 | with epath.Path(path).open("rb") as f: |
| 1058 | utf_f = codecs.getreader("utf-8")(f) |
| 1059 | for line_id, (_, elem) in enumerate(ElementTree.iterparse(utf_f)): # pytype: disable=wrong-arg-types |
| 1060 | if elem.tag == "tu": |
| 1061 | yield line_id, { |
| 1062 | _get_tuv_lang(tuv): _get_tuv_seg(tuv) |
| 1063 | for tuv in elem.iterfind("tuv") |
| 1064 | } |
| 1065 | elem.clear() |
| 1066 | |
| 1067 | |
| 1068 | def _parse_tsv(path, language_pair=None): |
nothing calls this directly
no test coverage detected