| 62 | |
| 63 | |
| 64 | def test_infer_storage_options(): |
| 65 | so = infer_storage_options("/mnt/datasets/test.csv") |
| 66 | assert so.pop("protocol") == "file" |
| 67 | assert so.pop("path") == "/mnt/datasets/test.csv" |
| 68 | assert not so |
| 69 | |
| 70 | assert infer_storage_options("./test.csv")["path"] == "./test.csv" |
| 71 | assert infer_storage_options("../test.csv")["path"] == "../test.csv" |
| 72 | |
| 73 | so = infer_storage_options("C:\\test.csv") |
| 74 | assert so.pop("protocol") == "file" |
| 75 | assert so.pop("path") == "C:\\test.csv" |
| 76 | assert not so |
| 77 | |
| 78 | assert infer_storage_options("d:\\test.csv")["path"] == "d:\\test.csv" |
| 79 | assert infer_storage_options("\\test.csv")["path"] == "\\test.csv" |
| 80 | assert infer_storage_options(".\\test.csv")["path"] == ".\\test.csv" |
| 81 | assert infer_storage_options("test.csv")["path"] == "test.csv" |
| 82 | |
| 83 | so = infer_storage_options( |
| 84 | "hdfs://username:pwd@Node:123/mnt/datasets/test.csv?q=1#fragm", |
| 85 | inherit_storage_options={"extra": "value"}, |
| 86 | ) |
| 87 | assert so.pop("protocol") == "hdfs" |
| 88 | assert so.pop("username") == "username" |
| 89 | assert so.pop("password") == "pwd" |
| 90 | assert so.pop("host") == "Node" |
| 91 | assert so.pop("port") == 123 |
| 92 | assert so.pop("path") == "/mnt/datasets/test.csv#fragm" |
| 93 | assert so.pop("url_query") == "q=1" |
| 94 | assert so.pop("url_fragment") == "fragm" |
| 95 | assert so.pop("extra") == "value" |
| 96 | assert not so |
| 97 | |
| 98 | so = infer_storage_options("hdfs://User-name@Node-name.com/mnt/datasets/test.csv") |
| 99 | assert so.pop("username") == "User-name" |
| 100 | assert so.pop("host") == "Node-name.com" |
| 101 | |
| 102 | u = "http://127.0.0.1:8080/test.csv" |
| 103 | assert infer_storage_options(u) == {"protocol": "http", "path": u} |
| 104 | |
| 105 | # For s3 and gcs the netloc is actually the bucket name, so we want to |
| 106 | # include it in the path. Test that: |
| 107 | # - Parsing doesn't lowercase the bucket |
| 108 | # - The bucket is included in path |
| 109 | for protocol in ["s3", "gcs", "gs"]: |
| 110 | options = infer_storage_options(f"{protocol}://Bucket-name.com/test.csv") |
| 111 | assert options["path"] == "Bucket-name.com/test.csv" |
| 112 | |
| 113 | with pytest.raises(KeyError): |
| 114 | infer_storage_options("file:///bucket/file.csv", {"path": "collide"}) |
| 115 | with pytest.raises(KeyError): |
| 116 | infer_storage_options("hdfs:///bucket/file.csv", {"protocol": "collide"}) |
| 117 | |
| 118 | |
| 119 | @pytest.mark.parametrize( |