Yield tuples of (line number, text line) from the file at `location`. Return an empty iterator if no text content is extractible. Text extraction is based on detected file type. Long lines are broken down in chunks, therefore two items can have the same line number. line number
(
location,
demarkup=False,
plain_text=False,
start_line=1,
)
| 49 | |
| 50 | |
| 51 | def numbered_text_lines( |
| 52 | location, |
| 53 | demarkup=False, |
| 54 | plain_text=False, |
| 55 | start_line=1, |
| 56 | ): |
| 57 | """ |
| 58 | Yield tuples of (line number, text line) from the file at `location`. Return |
| 59 | an empty iterator if no text content is extractible. Text extraction is |
| 60 | based on detected file type. Long lines are broken down in chunks, therefore |
| 61 | two items can have the same line number. |
| 62 | |
| 63 | line numbers start at ``start_line`` which is 1-based by default. |
| 64 | |
| 65 | If `demarkup` is True, attempt to detect if a file contains HTML/XML-like |
| 66 | markup and cleanup this markup. |
| 67 | |
| 68 | If `plain_text` is True treat the file as a plain text file and do not |
| 69 | attempt to detect its type and extract its content with special procedures. |
| 70 | This is used mostly when loading license texts and rules. |
| 71 | |
| 72 | Note: For testing or building from strings, location can be a is a list of |
| 73 | unicode line strings. |
| 74 | """ |
| 75 | if not location: |
| 76 | return iter([]) |
| 77 | |
| 78 | if not isinstance(location, str): |
| 79 | # not a path: wrap an iterator on location which should be a sequence of lines |
| 80 | if TRACE: |
| 81 | logger_debug('numbered_text_lines:', 'location is not a file') |
| 82 | return enumerate(iter(location), start_line) |
| 83 | |
| 84 | if plain_text: |
| 85 | if TRACE: |
| 86 | logger_debug('numbered_text_lines:', 'plain_text') |
| 87 | return enumerate(unicode_text_lines(location), start_line) |
| 88 | |
| 89 | T = typecode.get_type(location) |
| 90 | |
| 91 | if TRACE: |
| 92 | logger_debug('numbered_text_lines: T.filetype_file:', T.filetype_file) |
| 93 | logger_debug('numbered_text_lines: T.is_text_with_long_lines:', T.is_text_with_long_lines) |
| 94 | logger_debug('numbered_text_lines: T.is_binary:', T.is_binary) |
| 95 | |
| 96 | # TODO: we should have a command line to force digging inside binaries |
| 97 | if not T.contains_text: |
| 98 | return iter([]) |
| 99 | |
| 100 | # Should we read this as some markup, pdf office doc, text or binary? |
| 101 | if T.is_pdf and T.is_pdf_with_text: |
| 102 | if TRACE: |
| 103 | logger_debug('numbered_text_lines:', 'is_pdf') |
| 104 | return enumerate(unicode_text_lines_from_pdf(location), start_line) |
| 105 | |
| 106 | if T.filetype_file.startswith('Spline Font Database'): |
| 107 | if TRACE: |
| 108 | logger_debug('numbered_text_lines:', 'Spline Font Database') |