找出窗口 :param content_lst: :param window_size: :return:
(content_lst, window_size)
| 17 | |
| 18 | |
| 19 | def get_window(content_lst, window_size): |
| 20 | """ |
| 21 | 找出窗口 |
| 22 | :param content_lst: |
| 23 | :param window_size: |
| 24 | :return: |
| 25 | """ |
| 26 | word_window_freq = defaultdict(int) # w(i) 单词在窗口单位内出现的次数 |
| 27 | word_pair_count = defaultdict(int) # w(i, j) |
| 28 | windows_len = 0 |
| 29 | for words in tqdm(content_lst, desc="Split by window"): |
| 30 | windows = list() |
| 31 | |
| 32 | if isinstance(words, str): |
| 33 | words = words.split() |
| 34 | length = len(words) |
| 35 | |
| 36 | if length <= window_size: |
| 37 | windows.append(words) |
| 38 | else: |
| 39 | for j in range(length - window_size + 1): |
| 40 | window = words[j: j + window_size] |
| 41 | windows.append(list(set(window))) |
| 42 | |
| 43 | for window in windows: |
| 44 | for word in window: |
| 45 | word_window_freq[word] += 1 |
| 46 | |
| 47 | for word_pair in itertools.combinations(window, 2): |
| 48 | word_pair_count[word_pair] += 1 |
| 49 | |
| 50 | windows_len += len(windows) |
| 51 | return word_window_freq, word_pair_count, windows_len |
| 52 | |
| 53 | |
| 54 | def cal_pmi(W_ij, W, word_freq_1, word_freq_2): |