Returns an iterator over the lines in the file at the given path, strippping comments and decoding each line to Unicode.
(path, encoding="utf-8", comment=";;;")
| 191 | # Named entity rules are used to discover proper nouns (NNP's). |
| 192 | |
| 193 | def _read(path, encoding="utf-8", comment=";;;"): |
| 194 | """ Returns an iterator over the lines in the file at the given path, |
| 195 | strippping comments and decoding each line to Unicode. |
| 196 | """ |
| 197 | if path: |
| 198 | if isinstance(path, basestring) and os.path.exists(path): |
| 199 | # From file path. |
| 200 | f = open(path) |
| 201 | elif isinstance(path, basestring): |
| 202 | # From string. |
| 203 | f = path.splitlines() |
| 204 | elif hasattr(path, "read"): |
| 205 | # From string buffer. |
| 206 | f = path.read().splitlines() |
| 207 | else: |
| 208 | f = path |
| 209 | for i, line in enumerate(f): |
| 210 | line = line.strip(codecs.BOM_UTF8) if i == 0 and isinstance(line, str) else line |
| 211 | line = line.strip() |
| 212 | line = decode_utf8(line) |
| 213 | if not line or (comment and line.startswith(comment)): |
| 214 | continue |
| 215 | yield line |
| 216 | raise StopIteration |
| 217 | |
| 218 | class Lexicon(lazydict): |
| 219 |