Return an iterable of pygmars.Token built from a ``numbered_lines`` iterable of tuples of (line number, text). We perform a simple tokenization on spaces, tabs and some punctuation: =;
(numbered_lines, splitter=re.compile(r'[\t =;]+').split)
| 393 | |
| 394 | |
| 395 | def get_tokens(numbered_lines, splitter=re.compile(r'[\t =;]+').split): |
| 396 | """ |
| 397 | Return an iterable of pygmars.Token built from a ``numbered_lines`` iterable |
| 398 | of tuples of (line number, text). |
| 399 | |
| 400 | We perform a simple tokenization on spaces, tabs and some punctuation: =; |
| 401 | """ |
| 402 | last_line = "" |
| 403 | for start_line, line in numbered_lines: |
| 404 | pos = 0 |
| 405 | |
| 406 | if TRACE_TOK: |
| 407 | logger_debug(' get_tokens: bare line: ' + repr(line)) |
| 408 | |
| 409 | # keep or skip empty lines |
| 410 | if not line.strip(): |
| 411 | stripped = last_line.lower().strip(string.punctuation) |
| 412 | if ( |
| 413 | stripped.startswith("copyright") |
| 414 | or stripped.endswith(("by", "copyright", "0", "1", "2", "3", "4", "5", "6", "7", "8", "9")) |
| 415 | ): |
| 416 | continue |
| 417 | else: |
| 418 | yield Token(value="\n", label="EMPTY_LINE", start_line=start_line, pos=pos) |
| 419 | pos += 1 |
| 420 | last_line = "" |
| 421 | continue |
| 422 | |
| 423 | if TRACE_TOK: |
| 424 | logger_debug(' get_tokens: before preped line: ' + repr(line)) |
| 425 | |
| 426 | last_line = line |
| 427 | |
| 428 | if TRACE_TOK: |
| 429 | logger_debug(' get_tokens: preped line: ' + repr(line)) |
| 430 | |
| 431 | for tok in splitter(line): |
| 432 | # strip trailing quotes+comma |
| 433 | if tok.endswith("',"): |
| 434 | tok = tok.rstrip("',") |
| 435 | |
| 436 | tok = ( |
| 437 | tok |
| 438 | .strip("' ") # strip leading and trailing single quotes, and spaces |
| 439 | .rstrip(':') # strip trailing colons |
| 440 | .strip() |
| 441 | ) |
| 442 | |
| 443 | # the tokenizer allows a single colon or dot to be a token and we discard these |
| 444 | if tok and tok not in ':.': |
| 445 | yield Token(value=tok, start_line=start_line, pos=pos) |
| 446 | pos += 1 |
| 447 | |
| 448 | |
| 449 | class Detection: |