(documents: list, pid: int)
| 66 | return hex_dig[:6] |
| 67 | |
| 68 | def process_data(documents: list, pid: int): |
| 69 | # add newwords |
| 70 | t0 = time.time() |
| 71 | new_words = load_newwords() |
| 72 | for w in new_words: |
| 73 | jieba.add_word(w, tag='n') |
| 74 | |
| 75 | stop_words = load_stopwords() |
| 76 | print('{} start..'.format(pid)) |
| 77 | bad_patterns = [image_name, chapter2, chapter3] |
| 78 | |
| 79 | for filename,filepath in documents: |
| 80 | d = '' |
| 81 | with open(filepath) as f: |
| 82 | d = f.read() |
| 83 | # use half content |
| 84 | head_length = int(len(d) * 0.8) |
| 85 | d = d[0:head_length] |
| 86 | |
| 87 | cuts = [w.word for w in jp.cut(d)] |
| 88 | |
| 89 | filtered = [] |
| 90 | for c in cuts: |
| 91 | c = c.strip() |
| 92 | if c in stop_words: |
| 93 | continue |
| 94 | |
| 95 | if 'images' == c: |
| 96 | continue |
| 97 | |
| 98 | skip = False |
| 99 | for bad_pattern in bad_patterns: |
| 100 | if bad_pattern.match(c): |
| 101 | skip = True |
| 102 | break |
| 103 | if skip: |
| 104 | continue |
| 105 | |
| 106 | filtered.append(c) |
| 107 | |
| 108 | if len(filtered) < 1: |
| 109 | continue |
| 110 | new_content = ' '.join(filtered) |
| 111 | |
| 112 | if len(new_content) < 300: |
| 113 | continue |
| 114 | dirname = os.path.join('preprocess', str(pid)) |
| 115 | if not os.path.exists(dirname): |
| 116 | os.makedirs(dirname) |
| 117 | |
| 118 | hashname = content_hash(new_content) |
| 119 | outfilepath = os.path.join(dirname, hashname + '.md') |
| 120 | |
| 121 | with open('name_map.txt', 'a') as f: |
| 122 | f.write('{}\t {}'.format(hashname, filepath)) |
| 123 | f.write('\n') |
| 124 | |
| 125 | with open(outfilepath, 'w') as f: |
no test coverage detected