Splits docs into smallish (~1 KB) sized docs, repeating same title and date
(all_out, title_string, date_string, body_string)
| 82 | EUROPARL_V7_URL = 'https://www.statmt.org/europarl/v7/europarl.tgz' |
| 83 | |
| 84 | def split_docs(all_out, title_string, date_string, body_string): |
| 85 | |
| 86 | ''' |
| 87 | Splits docs into smallish (~1 KB) sized docs, repeating same title and date |
| 88 | ''' |
| 89 | |
| 90 | doc_count = 0 |
| 91 | while len(body_string) > 0: |
| 92 | char_count = int(random.gauss(TARGET_DOC_CHARS, TARGET_DOC_CHARS/4)) |
| 93 | if char_count < 64: |
| 94 | # trimmed normal? |
| 95 | continue |
| 96 | |
| 97 | m = re_next_non_word_character.search(body_string, char_count) |
| 98 | if m is not None: |
| 99 | char_count = m.start(0) |
| 100 | else: |
| 101 | char_count = len(body_string) |
| 102 | |
| 103 | body_string_fragment = body_string[:char_count].strip() |
| 104 | |
| 105 | #print('write title %d, body %d' % (len(title_string), len(body_string_fragment))) |
| 106 | all_out.write('%s\t%s\t%s\n' % (title_string, date_string, body_string_fragment)) |
| 107 | body_string = body_string[char_count:] |
| 108 | doc_count += 1 |
| 109 | |
| 110 | return doc_count |
| 111 | |
| 112 | def sample_europarl(): |
| 113 |
no test coverage detected
searching dependent graphs…