MCPcopy Create free account
hub / github.com/apache/arrow / test_checksum_write_to_dataset

Function test_checksum_write_to_dataset

python/pyarrow/tests/parquet/test_basic.py:1018–1067  ·  view source on GitHub ↗

Check that checksum verification works for datasets created with pq.write_to_dataset

(tempdir)

Source from the content-addressed store, hash-verified

1016
1017@pytest.mark.dataset
1018def test_checksum_write_to_dataset(tempdir):
1019 """Check that checksum verification works for datasets created with
1020 pq.write_to_dataset"""
1021
1022 table_orig = pa.table({'a': [1, 2, 3, 4]})
1023
1024 # Write a sample dataset with page checksum enabled
1025 original_dir_path = tempdir / 'correct_dir'
1026 pq.write_to_dataset(table_orig,
1027 original_dir_path,
1028 write_page_checksum=True)
1029
1030 # Read file and verify that the data is correct
1031 original_file_path_list = list(original_dir_path.iterdir())
1032 assert len(original_file_path_list) == 1
1033 original_path = original_file_path_list[0]
1034 table_check = pq.read_table(original_path, page_checksum_verification=True)
1035 assert table_orig == table_check
1036
1037 # Read the original file as binary and swap the 31-th and 36-th bytes. This
1038 # should be equivalent to storing the following data:
1039 # pa.table({'a': [1, 3, 2, 4]})
1040 bin_data = bytearray(original_path.read_bytes())
1041
1042 # Swap two bytes to emulate corruption. Also, check that the two bytes are
1043 # different, otherwise no corruption occurs
1044 assert bin_data[31] != bin_data[36]
1045 bin_data[31], bin_data[36] = bin_data[36], bin_data[31]
1046
1047 # Write the corrupted data to another parquet dataset
1048 # Copy dataset dir (which should be just one file)
1049 corrupted_dir_path = tempdir / 'corrupted_dir'
1050 copytree(original_dir_path, corrupted_dir_path)
1051 # Corrupt just the one file with the dataset
1052 corrupted_file_path = corrupted_dir_path / original_path.name
1053 corrupted_file_path.write_bytes(bin_data)
1054
1055 # Case 1: Reading the corrupted file with read_table() and without page
1056 # checksum verification succeeds but yields corrupted data
1057 table_corrupt = pq.read_table(corrupted_file_path,
1058 page_checksum_verification=False)
1059 # The read should complete without error, but the table has different
1060 # content than the original file!
1061 assert table_corrupt != table_orig
1062 assert table_corrupt == pa.table({'a': [1, 3, 2, 4]})
1063
1064 # Case 2: Reading the corrupted file with read_table() and with page
1065 # checksum verification enabled raises an exception
1066 with pytest.raises(OSError, match="CRC checksum verification"):
1067 _ = pq.read_table(corrupted_file_path, page_checksum_verification=True)
1068
1069
1070@pytest.mark.parametrize(

Callers

nothing calls this directly

Calls 4

listFunction · 0.85
lenFunction · 0.85
read_tableMethod · 0.45
read_bytesMethod · 0.45

Tested by

no test coverage detected