(hdfs_connection, pickle_with_and_without_subtree_filesystem)
| 1602 | |
| 1603 | @pytest.mark.hdfs |
| 1604 | def test_hdfs_options(hdfs_connection, pickle_with_and_without_subtree_filesystem): |
| 1605 | pickle_module = pickle_with_and_without_subtree_filesystem |
| 1606 | from pyarrow.fs import HadoopFileSystem |
| 1607 | if not pa.have_libhdfs(): |
| 1608 | pytest.skip('Cannot locate libhdfs') |
| 1609 | |
| 1610 | host, port, user = hdfs_connection |
| 1611 | |
| 1612 | replication = 2 |
| 1613 | buffer_size = 64*1024 |
| 1614 | default_block_size = 128*1024**2 |
| 1615 | uri = ('hdfs://{}:{}/?user={}&replication={}&buffer_size={}' |
| 1616 | '&default_block_size={}') |
| 1617 | |
| 1618 | hdfs1 = HadoopFileSystem(host, port, user='libhdfs', |
| 1619 | replication=replication, buffer_size=buffer_size, |
| 1620 | default_block_size=default_block_size) |
| 1621 | hdfs2 = HadoopFileSystem.from_uri(uri.format( |
| 1622 | host, port, 'libhdfs', replication, buffer_size, default_block_size |
| 1623 | )) |
| 1624 | hdfs3 = HadoopFileSystem.from_uri(uri.format( |
| 1625 | host, port, 'me', replication, buffer_size, default_block_size |
| 1626 | )) |
| 1627 | hdfs4 = HadoopFileSystem.from_uri(uri.format( |
| 1628 | host, port, 'me', replication + 1, buffer_size, default_block_size |
| 1629 | )) |
| 1630 | hdfs5 = HadoopFileSystem(host, port) |
| 1631 | hdfs6 = HadoopFileSystem.from_uri(f'hdfs://{host}:{port}') |
| 1632 | hdfs7 = HadoopFileSystem(host, port, user='localuser') |
| 1633 | hdfs8 = HadoopFileSystem(host, port, user='localuser', |
| 1634 | kerb_ticket="cache_path") |
| 1635 | hdfs9 = HadoopFileSystem(host, port, user='localuser', |
| 1636 | kerb_ticket=pathlib.Path("cache_path")) |
| 1637 | hdfs10 = HadoopFileSystem(host, port, user='localuser', |
| 1638 | kerb_ticket="cache_path2") |
| 1639 | hdfs11 = HadoopFileSystem(host, port, user='localuser', |
| 1640 | kerb_ticket="cache_path", |
| 1641 | extra_conf={'hdfs_token': 'abcd'}) |
| 1642 | |
| 1643 | assert hdfs1 == hdfs2 |
| 1644 | assert hdfs5 == hdfs6 |
| 1645 | assert hdfs6 != hdfs7 |
| 1646 | assert hdfs2 != hdfs3 |
| 1647 | assert hdfs3 != hdfs4 |
| 1648 | assert hdfs7 != hdfs5 |
| 1649 | assert hdfs2 != hdfs3 |
| 1650 | assert hdfs3 != hdfs4 |
| 1651 | assert hdfs7 != hdfs8 |
| 1652 | assert hdfs8 == hdfs9 |
| 1653 | assert hdfs10 != hdfs9 |
| 1654 | assert hdfs11 != hdfs8 |
| 1655 | |
| 1656 | with pytest.raises(TypeError): |
| 1657 | HadoopFileSystem() |
| 1658 | with pytest.raises(TypeError): |
| 1659 | HadoopFileSystem.from_uri(3) |
| 1660 | |
| 1661 | for fs in [hdfs1, hdfs2, hdfs3, hdfs4, hdfs5, hdfs6, hdfs7, hdfs8, |
nothing calls this directly
no test coverage detected