Class Spelling

pattern/text/__init__.py:1711–1785

# Based on: Peter Norvig, "How to Write a Spelling Corrector", http://norvig.com/spell-correct.html

class Spelling(lazydict):
    
    ALPHA = "abcdefghijklmnopqrstuvwxyz"
    
    def __init__(self, path=""):
        self._path = path
    
    def load(self):
        for x in _read(self._path):
            x = x.split()
            dict.__setitem__(self, x[0], int(x[1]))
    
    @property
    def path(self):
        return self._path
        
    @property
    def language(self):
        return self._language

    @classmethod
    def train(self, s, path="spelling.txt"):
        """ Counts the words in the given string and saves the probabilities at the given path.
            This can be used to generate a new model for the Spelling() constructor.
        """
        model = {}
        for w in re.findall("[a-z]+", s.lower()):
            model[w] = w in model and model[w] + 1 or 1
        model = ("%s %s" % (k, v) for k, v in sorted(model.items()))
        model = "\n".join(model)
        f = open(path, "w")
        f.write(model)
        f.close()
        
    def _edit1(self, w):
        """ Returns a set of words with edit distance 1 from the given word.
        """
        # Of all spelling errors, 80% is covered by edit distance 1.
        # Edit distance 1 = one character deleted, swapped, replaced or inserted.
        split = [(w[:i], w[i:]) for i in range(len(w) + 1)]
        delete, transpose, replace, insert = (
            [a + b[1:] for a, b in split if b],
            [a + b[1] + b[0] + b[2:] for a, b in split if len(b) > 1],
            [a + c + b[1:] for a, b in split for c in Spelling.ALPHA if b],
            [a + c + b[0:] for a, b in split for c in Spelling.ALPHA]
        )
        return set(delete + transpose + replace + insert)
        
    def _edit2(self, w):
        """ Returns a set of words with edit distance 2 from the given word
        """
        # Of all spelling errors, 99% is covered by edit distance 2.
        # Only keep candidates that are actually known words (20% speedup).
        return set(e2 for e1 in self._edit1(w) for e2 in self._edit1(e1) if e2 in self)

    def _known(self, words=[]):
        """ Returns the given list of words filtered by known words.
        """