MCPcopy Create free account
hub / github.com/apache/arrow / test_dictionary_delta

Function test_dictionary_delta

python/pyarrow/tests/test_ipc.py:617–689  ·  view source on GitHub ↗
(format_fixture)

Source from the content-addressed store, hash-verified

615
616
617def test_dictionary_delta(format_fixture):
618 ty = pa.dictionary(pa.int8(), pa.utf8())
619 data = [["foo", "foo", None],
620 ["foo", "bar", "foo"], # potential delta
621 ["foo", "bar"], # nothing new
622 ["foo", None, "bar", "quux"], # potential delta
623 ["bar", "quux"], # replacement
624 ]
625 batches = [
626 pa.RecordBatch.from_arrays([pa.array(v, type=ty)], names=['dicts'])
627 for v in data]
628 batches_delta_only = batches[:4]
629 schema = batches[0].schema
630
631 def write_batches(batches, as_table=False):
632 with format_fixture._get_writer(pa.MockOutputStream(),
633 schema) as writer:
634 if as_table:
635 table = pa.Table.from_batches(batches)
636 writer.write_table(table)
637 else:
638 for batch in batches:
639 writer.write_batch(batch)
640 return writer.stats
641
642 if format_fixture.is_file:
643 # File format cannot handle replacement
644 with pytest.raises(pa.ArrowInvalid):
645 write_batches(batches)
646 # File format cannot handle delta if emit_deltas
647 # is not provided
648 with pytest.raises(pa.ArrowInvalid):
649 write_batches(batches_delta_only)
650 else:
651 st = write_batches(batches)
652 assert st.num_record_batches == 5
653 assert st.num_dictionary_batches == 4
654 assert st.num_replaced_dictionaries == 3
655 assert st.num_dictionary_deltas == 0
656
657 format_fixture.use_legacy_ipc_format = None
658 format_fixture.options = pa.ipc.IpcWriteOptions(
659 emit_dictionary_deltas=True)
660 if format_fixture.is_file:
661 # File format cannot handle replacement
662 with pytest.raises(pa.ArrowInvalid):
663 write_batches(batches)
664 else:
665 st = write_batches(batches)
666 assert st.num_record_batches == 5
667 assert st.num_dictionary_batches == 4
668 assert st.num_replaced_dictionaries == 1
669 assert st.num_dictionary_deltas == 2
670
671 st = write_batches(batches_delta_only)
672 assert st.num_record_batches == 4
673 assert st.num_dictionary_batches == 3
674 assert st.num_replaced_dictionaries == 0

Callers

nothing calls this directly

Calls 3

write_batchesFunction · 0.85
dictionaryMethod · 0.45
arrayMethod · 0.45

Tested by

no test coverage detected