()
| 110 | return doc_count |
| 111 | |
| 112 | def sample_europarl(): |
| 113 | |
| 114 | # download europarl.tgz v7, if not already here (in cwd): |
| 115 | file_name = 'europarl.tgz' |
| 116 | if not os.path.exists(file_name): |
| 117 | print('Download %s to %s...' % (EUROPARL_V7_URL, file_name)) |
| 118 | urllib.request.urlretrieve(EUROPARL_V7_URL, file_name + '.tmp') |
| 119 | os.rename(file_name + '.tmp', file_name) |
| 120 | else: |
| 121 | print('%s already here; skipping download...' % file_name) |
| 122 | |
| 123 | if not DEBUG: |
| 124 | tmp_dir_path = tempfile.mkdtemp() |
| 125 | else: |
| 126 | tmp_dir_path = '/tmp/tmp31ekzg75' |
| 127 | print('Using tmp dir "%s"...' % tmp_dir_path) |
| 128 | try: |
| 129 | if not DEBUG: |
| 130 | cmd = 'tar xzf %s -C %s' % (file_name, tmp_dir_path) |
| 131 | print('Run: %s' % cmd) |
| 132 | subprocess.run(cmd, shell=True) |
| 133 | |
| 134 | doc_count = 0 |
| 135 | skip_count = 0 |
| 136 | file_count = 0 |
| 137 | |
| 138 | all_txt_file_name = '%s/all.txt' % tmp_dir_path |
| 139 | |
| 140 | print('Extract text...') |
| 141 | |
| 142 | start_time = time.time() |
| 143 | next_print_time = start_time + 3 |
| 144 | # normalize text a bit and concatenate all lines into single file, counting total lines/bytes |
| 145 | with open(all_txt_file_name, 'w', encoding='utf-8') as all_out: |
| 146 | for dir_path, dir_names, file_names in os.walk('%s/txt' % tmp_dir_path): |
| 147 | for file_name in file_names: |
| 148 | if file_name.endswith('.txt'): |
| 149 | file_count += 1 |
| 150 | |
| 151 | year, month, day = (int(x) for x in file_name[3:-4].split('-')[:3]) |
| 152 | if year >= 50: |
| 153 | year = 1900 + year |
| 154 | else: |
| 155 | year = 2000 + year |
| 156 | |
| 157 | date_string = '%04d-%02d-%02d' % (year, month, day) |
| 158 | |
| 159 | # unfortunately we need errors='ignore' since in Europarl v7, one file (pl/ep-09-10-22-009.txt) has invalid utf-8: |
| 160 | chapter_count = 0 |
| 161 | with open('%s/%s' % (dir_path, file_name), 'r', encoding='utf-8', errors='ignore') as f_in: |
| 162 | last_text = [] |
| 163 | last_title = None |
| 164 | while True: |
| 165 | line = f_in.readline() |
| 166 | if line == '': |
| 167 | break |
| 168 | line = line.strip() |
| 169 | if line.startswith('<CHAPTER '): |
no test coverage detected
searching dependent graphs…