(doc_item)
| 596 | stemmer = SnowballStemmer("english") |
| 597 | |
| 598 | def process_doc(doc_item): |
| 599 | doc_id, doc = doc_item |
| 600 | text = f"{doc.get('title', '')} {doc['text']}" |
| 601 | tokens = SimpleTokenizer.tokenize(text) |
| 602 | terms = [ |
| 603 | stemmer.stem_word(t.lower()) |
| 604 | for t in tokens |
| 605 | if t.lower() not in STOPWORDS and t not in PUNCT and len(t) <= 40 |
| 606 | ] |
| 607 | return {"id": doc_id, "tokens": terms, "length": len(terms)} |
| 608 | |
| 609 | docs = [] |
| 610 | with ThreadPoolExecutor(max_workers=MAX_WORKERS) as ex: |