Function _visit_level

python/pyarrow/tests/parquet/test_dataset.py:562–592

(base_dir, level, part_keys)

    pathsep = getattr(fs, "pathsep", getattr(fs, "sep", "/"))

    def _visit_level(base_dir, level, part_keys):
        name, values = partition_spec[level]
        for value in values:
            this_part_keys = part_keys + [(name, value)]

            level_dir = pathsep.join([
                str(base_dir),
                f'{name}={value}'
            ])
            fs.create_dir(level_dir)

            if level == DEPTH - 1:
                # Generate example data
                from pyarrow.fs import FileType

                file_path = pathsep.join([level_dir, guid()])
                filtered_df = _filter_partition(df, this_part_keys)
                part_table = pa.Table.from_pandas(filtered_df)
                with fs.open_output_stream(file_path) as f:
                    _write_table(part_table, f)
                assert fs.get_file_info(file_path).type != FileType.NotFound
                assert fs.get_file_info(file_path).type == FileType.File

                file_success = pathsep.join([level_dir, '_SUCCESS'])
                with fs.open_output_stream(file_success) as f:
                    pass
            else:
                _visit_level(level_dir, level + 1, this_part_keys)
                file_success = pathsep.join([level_dir, '_SUCCESS'])
                with fs.open_output_stream(file_success) as f:
                    pass

    _visit_level(base_dir, 0, [])