Function extract_keywords

skills/paper-processor/scripts/embed.py:99–143

Extract keywords Args: text: Text to extract from top_k: Number of keywords to return Returns: List of keywords

(text: str, top_k: int = 5)

def extract_keywords(text: str, top_k: int = 5) -> List[str]:
    """
    Extract keywords

    Args:
        text: Text to extract from
        top_k: Number of keywords to return

    Returns:
        List of keywords
    """
    # Tokenize
    words = re.findall(r'\b\w+\b', text.lower())

    # Remove stop words and short words
    words = [w for w in words if w not in STOP_WORDS and len(w) > 3]

    # Calculate word frequency
    word_freq = {}
    for word in words:
        word_freq[word] = word_freq.get(word, 0) + 1

    # Match domain keywords
    all_keywords = set()
    for keywords in DOMAIN_KEYWORDS.values():
        all_keywords.update(keywords)

    matched = []
    for keyword in all_keywords:
        if keyword in text.lower():
            matched.append(keyword)

    # Sort by frequency
    sorted_words = sorted(word_freq.items(), key=lambda x: x[1], reverse=True)

    # Combine results
    result = matched[:top_k]
    if len(result) < top_k:
        for word, _ in sorted_words:
            if word not in result:
                result.append(word)
            if len(result) >= top_k:
                break

    return result[:top_k]


def generate_embedding(text: str, model_name: str = "text-embedding-3-small") -> List[float]: