Yields lines from the text file. Performs lowercasing and white-space tokenization on each line before returning. Args: text_file: String filename. multiple: Whether multiple references / generations are expected in a line.
(text_file, multiple=False)
| 76 | |
| 77 | |
| 78 | def _text_reader(text_file, multiple=False): |
| 79 | """Yields lines from the text file. |
| 80 | |
| 81 | Performs lowercasing and white-space tokenization on each line before |
| 82 | returning. |
| 83 | |
| 84 | Args: |
| 85 | text_file: String filename. |
| 86 | multiple: Whether multiple references / generations are expected in a line. |
| 87 | """ |
| 88 | with io.open(text_file) as f: |
| 89 | for line in f: |
| 90 | if multiple: |
| 91 | yield [item.lower().split() for item in line.strip().split("\t")] |
| 92 | else: |
| 93 | yield line.strip().lower().split() |
| 94 | |
| 95 | |
| 96 | def _table_reader(table_file): |