An unordered bag-of-words representation of the given string, list, dict or Sentence. Lists can contain tuples (of), strings or numbers. Dicts can contain tuples (of), strings or numbers as keys, and floats as values. Document.terms stores a dict of (word, count)
(self, string="", **kwargs)
| 322 | # type = None |
| 323 | # ) |
| 324 | def __init__(self, string="", **kwargs): |
| 325 | """ An unordered bag-of-words representation of the given string, list, dict or Sentence. |
| 326 | Lists can contain tuples (of), strings or numbers. |
| 327 | Dicts can contain tuples (of), strings or numbers as keys, and floats as values. |
| 328 | Document.terms stores a dict of (word, count)-items. |
| 329 | Document.vector stores a dict of (word, weight)-items, |
| 330 | where weight is the term frequency normalized (0.0-1.0) to remove document length bias. |
| 331 | Punctuation marks are stripped from the words. |
| 332 | Stop words in the exclude list are excluded from the document. |
| 333 | Only top words whose count exceeds the threshold are included in the document. |
| 334 | """ |
| 335 | kwargs.setdefault("filter", lambda w: w.lstrip("'").isalnum()) |
| 336 | kwargs.setdefault("threshold", 0) |
| 337 | kwargs.setdefault("dict", readonlydict) |
| 338 | # A string of words: map to read-only dict of (word, count)-items. |
| 339 | if string is None: |
| 340 | w = kwargs["dict"]() |
| 341 | v = None |
| 342 | elif isinstance(string, basestring): |
| 343 | w = words(string, **kwargs) |
| 344 | w = count(w, **kwargs) |
| 345 | v = None |
| 346 | # A list of words: map to read-only dict of (word, count)-items. |
| 347 | elif isinstance(string, (list, tuple)) and not string.__class__.__name__ == "Text": |
| 348 | w = string |
| 349 | w = count(w, **kwargs) |
| 350 | v = None |
| 351 | # A Vector of (word, weight)-items: copy as document vector. |
| 352 | elif isinstance(string, Vector): |
| 353 | w = string |
| 354 | w = kwargs["dict"](w) |
| 355 | v = Vector(w) |
| 356 | # A dict of (word, count)-items: make read-only. |
| 357 | elif isinstance(string, dict): |
| 358 | w = string |
| 359 | w = kwargs["dict"](w) |
| 360 | v = None |
| 361 | # pattern.en.Sentence with Word objects: can use stemmer=LEMMA. |
| 362 | elif string.__class__.__name__ == "Sentence": |
| 363 | w = string.words |
| 364 | w = [w for w in w if kwargs["filter"](w.string)] |
| 365 | w = count(w, **kwargs) |
| 366 | v = None |
| 367 | # pattern.en.Text with Sentence objects, can use stemmer=LEMMA. |
| 368 | elif string.__class__.__name__ == "Text": |
| 369 | w = []; [w.extend(sentence.words) for sentence in string] |
| 370 | w = [w for w in w if kwargs["filter"](w.string)] |
| 371 | w = count(w, **kwargs) |
| 372 | v = None |
| 373 | else: |
| 374 | raise TypeError, "document string is not str, unicode, list, dict, Vector, Sentence or Text." |
| 375 | self._id = _uid() # Document ID, used when comparing objects. |
| 376 | self._name = kwargs.get("name") # Name that describes the document content. |
| 377 | self._type = kwargs.get("type", # Type that describes the category or class of the document. |
| 378 | kwargs.get("label")) |
| 379 | self._language = kwargs.get("language") |
| 380 | self._description = kwargs.get("description", "") |
| 381 | self._terms = w # Dictionary of (word, count)-items. |