Function find_lemmata

pattern/text/en/__init__.py:76–89

Annotates the tokens with lemmata for plural nouns and conjugated verbs, where each token is a [word, part-of-speech] list.

(tokens)

#--- ENGLISH PARSER --------------------------------------------------------------------------------

def find_lemmata(tokens):
    """ Annotates the tokens with lemmata for plural nouns and conjugated verbs,
        where each token is a [word, part-of-speech] list.
    """
    for token in tokens:
        word, pos, lemma = token[0], token[1], token[0]
        # cats => cat
        if pos == "NNS":
            lemma = singularize(word)
        # sat => sit
        if pos.startswith(("VB", "MD")):
            lemma = conjugate(word, INFINITIVE) or word
        token.append(lemma.lower())
    return tokens
    
class Parser(_Parser):