MCPcopy Create free account
hub / github.com/apache/arrow / test_column_encoding

Function test_column_encoding

python/pyarrow/tests/parquet/test_basic.py:460–585  ·  view source on GitHub ↗
()

Source from the content-addressed store, hash-verified

458
459
460def test_column_encoding():
461 arr_float = pa.array(list(map(float, range(100))))
462 arr_int = pa.array(list(map(int, range(100))))
463 arr_bin = pa.array([str(x) for x in range(100)], type=pa.binary())
464 arr_flba = pa.array(
465 [str(x).zfill(10) for x in range(100)], type=pa.binary(10))
466 arr_bool = pa.array([False, True, False, False] * 25)
467 mixed_table = pa.Table.from_arrays(
468 [arr_float, arr_int, arr_bin, arr_flba, arr_bool],
469 names=['a', 'b', 'c', 'd', 'e'])
470
471 # Check "BYTE_STREAM_SPLIT" for columns 'a', 'b', 'd'
472 # and "PLAIN" column_encoding for column 'c'.
473 _check_roundtrip(mixed_table, expected=mixed_table, use_dictionary=False,
474 column_encoding={'a': "BYTE_STREAM_SPLIT",
475 'b': "BYTE_STREAM_SPLIT",
476 'c': "PLAIN",
477 'd': "BYTE_STREAM_SPLIT"})
478
479 # Check "PLAIN" for all columns.
480 _check_roundtrip(mixed_table, expected=mixed_table,
481 use_dictionary=False,
482 column_encoding="PLAIN")
483
484 # Check "DELTA_BINARY_PACKED" for integer columns.
485 _check_roundtrip(mixed_table, expected=mixed_table,
486 use_dictionary=False,
487 column_encoding={'a': "PLAIN",
488 'b': "DELTA_BINARY_PACKED",
489 'c': "PLAIN"})
490
491 # Check "DELTA_LENGTH_BYTE_ARRAY" for byte columns.
492 _check_roundtrip(mixed_table, expected=mixed_table,
493 use_dictionary=False,
494 column_encoding={'a': "PLAIN",
495 'b': "DELTA_BINARY_PACKED",
496 'c': "DELTA_LENGTH_BYTE_ARRAY"})
497
498 # Check "DELTA_BYTE_ARRAY" for byte columns.
499 _check_roundtrip(mixed_table, expected=mixed_table,
500 use_dictionary=False,
501 column_encoding={'a': "PLAIN",
502 'b': "DELTA_BINARY_PACKED",
503 'c': "DELTA_BYTE_ARRAY",
504 'd': "DELTA_BYTE_ARRAY"})
505
506 # Check "RLE" for boolean columns.
507 _check_roundtrip(mixed_table, expected=mixed_table,
508 use_dictionary=False,
509 column_encoding={'e': "RLE"})
510
511 # Try to pass "BYTE_STREAM_SPLIT" column encoding for boolean column 'e'.
512 # This should throw an error as it is does not support BOOLEAN.
513 with pytest.raises(IOError,
514 match="BYTE_STREAM_SPLIT only supports"):
515 _check_roundtrip(mixed_table, expected=mixed_table,
516 use_dictionary=False,
517 column_encoding={'a': "PLAIN",

Callers

nothing calls this directly

Calls 4

_check_roundtripFunction · 0.90
listFunction · 0.85
mapFunction · 0.85
arrayMethod · 0.45

Tested by

no test coverage detected