Function transform_sentence_to_vector

tests/test-tf-idf-bm25.py:129–184

(id, sentence, punctuations, stemmer, token_max_len=40)

def transform_sentence_to_vector(id, sentence, punctuations, stemmer, token_max_len=40):
    stopwords = {
        "a",
        "and",
        "are",
        "as",
        "at",
        "be",
        "but",
        "by",
        "for",
        "if",
        "in",
        "into",
        "is",
        "it",
        "no",
        "not",
        "of",
        "on",
        "or",
        "s",
        "such",
        "t",
        "that",
        "the",
        "their",
        "then",
        "there",
        "these",
        "they",
        "this",
        "to",
        "was",
        "will",
        "with",
        "www",
    }
    cleaned = remove_non_alphanumeric(sentence)
    tokens = SimpleTokenizer.tokenize(cleaned)
    processed_tokens = [
        stemmer.stem_word(token.lower())
        for token in tokens
        if token not in punctuations
        and token.lower() not in stopwords
        and len(token) <= token_max_len
    ]
    terms, length = construct_sparse_vector(processed_tokens)

    return {
        "id": id,
        "text": sentence,
        "indices": [term[0] for term in terms],
        "raw_term_frequencies": [term[1] for term in terms],
        "length": length,
    }