Function flatten_scan

src/formattedcode/output_csv.py:107–223

Yield ordered dictionaries of key/values flattening the sequence data in a single line-separated value and keying always by path, given a ScanCode `scan` results list. Update the `headers` mapping sequences with seen keys as a side effect.

(scan, headers)

def flatten_scan(scan, headers):
    """
    Yield ordered dictionaries of key/values flattening the sequence
    data in a single line-separated value and keying always by path,
    given a ScanCode `scan` results list. Update the `headers` mapping
    sequences with seen keys as a side effect.
    """
    seen = set()

    def collect_keys(mapping, key_group):
        """Update the headers with new keys."""
        keys = mapping.keys()
        headers[key_group].extend(k for k in keys if k not in seen)
        seen.update(keys)

    for scanned_file in scan:
        path = scanned_file.pop('path')

        # removing any slash at the begening of the path
        path = path.lstrip('/')

        # use a trailing slash for directories
        if scanned_file.get('type') == 'directory' and not path.endswith('/'):
            path += '/'

        errors = scanned_file.pop('scan_errors', [])

        # FIXME: info are NOT lists: lists are the actual scans
        file_info = dict(path=path)
        file_info.update(
            (
                (k, v) for k, v in scanned_file.items()
                if not isinstance(v, (list, dict))
            )
        )
        # Scan errors are joined in a single multi-line value
        file_info['scan_errors'] = '\n'.join(errors)

        collect_keys(file_info, 'info')
        yield file_info

        for detection in scanned_file.get('license_detections', []):
            license_expression = detection["license_expression"]
            detection_log = detection.get("detection_log", []) or []
            detection_log = '\n'.join(detection_log)
            license_matches = detection["matches"]
            for match in license_matches:
                lic = dict(path=path)
                lic["license_expression"] = license_expression
                lic["detection_log"] = detection_log

                for k, val in match.items():
                    # do not include matched text for now.
                    if k == 'matched_text':
                        continue

                    if k == 'licenses':
                        license_keys = []