Parse a CSV file into a list of records with type conversion.
(lines, select=None, types=None, has_headers=True, delimiter=',', silence_errors=False)
| 4 | log = logging.getLogger(__name__) |
| 5 | |
| 6 | def parse_csv(lines, select=None, types=None, has_headers=True, delimiter=',', silence_errors=False): |
| 7 | ''' |
| 8 | Parse a CSV file into a list of records with type conversion. |
| 9 | ''' |
| 10 | assert not (select and not has_headers), 'select requires column headers' |
| 11 | rows = csv.reader(lines, delimiter=delimiter) |
| 12 | |
| 13 | # Read the file headers (if any) |
| 14 | headers = next(rows) if has_headers else [] |
| 15 | |
| 16 | # If specific columns have been selected, make indices for filtering and set output columns |
| 17 | if select: |
| 18 | indices = [ headers.index(colname) for colname in select ] |
| 19 | headers = select |
| 20 | |
| 21 | records = [] |
| 22 | for rowno, row in enumerate(rows, 1): |
| 23 | if not row: # Skip rows with no data |
| 24 | continue |
| 25 | |
| 26 | # If specific column indices are selected, pick them out |
| 27 | if select: |
| 28 | row = [ row[index] for index in indices] |
| 29 | |
| 30 | # Apply type conversion to the row |
| 31 | if types: |
| 32 | try: |
| 33 | row = [func(val) for func, val in zip(types, row)] |
| 34 | except ValueError as e: |
| 35 | if not silence_errors: |
| 36 | log.warning("Row %d: Couldn't convert %s", rowno, row) |
| 37 | log.debug("Row %d: Reason %s", rowno, e) |
| 38 | continue |
| 39 | |
| 40 | # Make a dictionary or a tuple |
| 41 | if headers: |
| 42 | record = dict(zip(headers, row)) |
| 43 | else: |
| 44 | record = tuple(row) |
| 45 | records.append(record) |
| 46 | |
| 47 | return records |