| 34 | plot_lock = asyncio.Lock() |
| 35 | |
| 36 | class AsyncWordCloudGenerator: |
| 37 | def __init__(self): |
| 38 | logging.getLogger('jieba').setLevel(logging.WARNING) |
| 39 | self.stop_words_file = config.STOP_WORDS_FILE |
| 40 | self.lock = asyncio.Lock() |
| 41 | self.stop_words = self.load_stop_words() |
| 42 | self.custom_words = config.CUSTOM_WORDS |
| 43 | for word, group in self.custom_words.items(): |
| 44 | jieba.add_word(word) |
| 45 | |
| 46 | def load_stop_words(self): |
| 47 | with open(self.stop_words_file, 'r', encoding='utf-8') as f: |
| 48 | return set(f.read().strip().split('\n')) |
| 49 | |
| 50 | async def generate_word_frequency_and_cloud(self, data, save_words_prefix): |
| 51 | all_text = ' '.join(item['content'] for item in data) |
| 52 | words = [word for word in jieba.lcut(all_text) if word not in self.stop_words and len(word.strip()) > 0] |
| 53 | word_freq = Counter(words) |
| 54 | |
| 55 | # Save word frequency to file |
| 56 | freq_file = f"{save_words_prefix}_word_freq.json" |
| 57 | async with aiofiles.open(freq_file, 'w', encoding='utf-8') as file: |
| 58 | await file.write(json.dumps(word_freq, ensure_ascii=False, indent=4)) |
| 59 | |
| 60 | # Try to acquire the plot lock without waiting |
| 61 | if plot_lock.locked(): |
| 62 | utils.logger.info("Skipping word cloud generation as the lock is held.") |
| 63 | return |
| 64 | |
| 65 | await self.generate_word_cloud(word_freq, save_words_prefix) |
| 66 | |
| 67 | async def generate_word_cloud(self, word_freq, save_words_prefix): |
| 68 | await plot_lock.acquire() |
| 69 | top_20_word_freq = {word: freq for word, freq in |
| 70 | sorted(word_freq.items(), key=lambda item: item[1], reverse=True)[:20]} |
| 71 | wordcloud = WordCloud( |
| 72 | font_path=config.FONT_PATH, |
| 73 | width=800, |
| 74 | height=400, |
| 75 | background_color='white', |
| 76 | max_words=200, |
| 77 | stopwords=self.stop_words, |
| 78 | colormap='viridis', |
| 79 | contour_color='steelblue', |
| 80 | contour_width=1 |
| 81 | ).generate_from_frequencies(top_20_word_freq) |
| 82 | |
| 83 | # Save word cloud image |
| 84 | plt.figure(figsize=(10, 5), facecolor='white') |
| 85 | plt.imshow(wordcloud, interpolation='bilinear') |
| 86 | |
| 87 | plt.axis('off') |
| 88 | plt.tight_layout(pad=0) |
| 89 | plt.savefig(f"{save_words_prefix}_word_cloud.png", format='png', dpi=300) |
| 90 | plt.close() |
| 91 | |
| 92 | plot_lock.release() |