| 2861 | |
| 2862 | |
| 2863 | def test_pyfilesystem_simple(tmp_path: pathlib.Path): |
| 2864 | zip_path = ( |
| 2865 | pathlib.Path("/".join(__file__.split("/")[:-1])) / "data" / "pyfs-testdata.zip" |
| 2866 | ) |
| 2867 | output_path = tmp_path / "output.txt" |
| 2868 | |
| 2869 | with open_fs("zip://" + str(zip_path)) as source: |
| 2870 | table = pw.io.pyfilesystem.read( |
| 2871 | source, |
| 2872 | mode="static", |
| 2873 | with_metadata=True, |
| 2874 | ) |
| 2875 | pw.io.jsonlines.write(table, output_path) |
| 2876 | pw.run(monitoring_level=pw.MonitoringLevel.NONE) |
| 2877 | |
| 2878 | paths = set() |
| 2879 | names = set() |
| 2880 | with open(output_path, "r") as f: |
| 2881 | for line in f: |
| 2882 | data = json.loads(line) |
| 2883 | assert "_metadata" in data |
| 2884 | assert "data" in data |
| 2885 | assert "diff" in data |
| 2886 | assert "time" in data |
| 2887 | metadata = data["_metadata"] |
| 2888 | paths.add(metadata["path"]) |
| 2889 | names.add(metadata["name"]) |
| 2890 | assert metadata["size"] == len(base64.b64decode(data["data"])) |
| 2891 | |
| 2892 | assert names == set(["a.txt", "b.txt"]) |
| 2893 | assert paths == set(["projects/a.txt", "projects/b.txt"]) |
| 2894 | |
| 2895 | |
| 2896 | @needs_multiprocessing_fork |