Function normalize_tar_archive

pythonbuild/utils.py:405–472

Normalize the contents of a tar archive. We want tar archives to be as deterministic as possible. This function will take tar archive data in a buffer and return a new buffer containing a more deterministic tar archive.

(data: io.BytesIO)

def normalize_tar_archive(data: io.BytesIO) -> io.BytesIO:
    """Normalize the contents of a tar archive.

    We want tar archives to be as deterministic as possible. This function will
    take tar archive data in a buffer and return a new buffer containing a more
    deterministic tar archive.
    """
    members = []

    with tarfile.open(fileobj=data) as tf:
        for ti in tf:
            # We don't care about directory entries. Tools can handle this fine.
            if ti.isdir():
                continue

            filedata = tf.extractfile(ti)
            if filedata is not None:
                filedata = io.BytesIO(filedata.read())

            members.append((ti, filedata))

    # Sort the archive members. We put PYTHON.json first so metadata can
    # be read without reading the entire archive.
    def sort_key(v):
        if v[0].name == "python/PYTHON.json":
            return 0, v[0].name
        else:
            return 1, v[0].name

    members.sort(key=sort_key)

    # Normalize attributes on archive members.
    for entry in members:
        ti = entry[0]

        # The pax headers attribute takes priority over the other named
        # attributes. To minimize potential for our assigns to no-op, we
        # clear out the pax headers. We can't reset all the pax headers,
        # as this would nullify symlinks.
        for a in ("mtime", "uid", "uname", "gid", "gname"):
            try:
                ti.pax_headers.__delattr__(a)
            except AttributeError:
                pass

        ti.pax_headers = {}

        ti.mtime = DEFAULT_MTIME
        ti.uid = 0
        ti.uname = "root"
        ti.gid = 0
        ti.gname = "root"

        # Give user/group read/write on all entries.
        ti.mode |= stat.S_IRUSR | stat.S_IWUSR | stat.S_IRGRP | stat.S_IWGRP

        # If user executable, give to group as well.
        if ti.mode & stat.S_IXUSR: