(format_fixture)
| 591 | |
| 592 | |
| 593 | def test_dictionary_delta(format_fixture): |
| 594 | ty = pa.dictionary(pa.int8(), pa.utf8()) |
| 595 | data = [["foo", "foo", None], |
| 596 | ["foo", "bar", "foo"], # potential delta |
| 597 | ["foo", "bar"], # nothing new |
| 598 | ["foo", None, "bar", "quux"], # potential delta |
| 599 | ["bar", "quux"], # replacement |
| 600 | ] |
| 601 | batches = [ |
| 602 | pa.RecordBatch.from_arrays([pa.array(v, type=ty)], names=['dicts']) |
| 603 | for v in data] |
| 604 | batches_delta_only = batches[:4] |
| 605 | schema = batches[0].schema |
| 606 | |
| 607 | def write_batches(batches, as_table=False): |
| 608 | with format_fixture._get_writer(pa.MockOutputStream(), |
| 609 | schema) as writer: |
| 610 | if as_table: |
| 611 | table = pa.Table.from_batches(batches) |
| 612 | writer.write_table(table) |
| 613 | else: |
| 614 | for batch in batches: |
| 615 | writer.write_batch(batch) |
| 616 | return writer.stats |
| 617 | |
| 618 | if format_fixture.is_file: |
| 619 | # File format cannot handle replacement |
| 620 | with pytest.raises(pa.ArrowInvalid): |
| 621 | write_batches(batches) |
| 622 | # File format cannot handle delta if emit_deltas |
| 623 | # is not provided |
| 624 | with pytest.raises(pa.ArrowInvalid): |
| 625 | write_batches(batches_delta_only) |
| 626 | else: |
| 627 | st = write_batches(batches) |
| 628 | assert st.num_record_batches == 5 |
| 629 | assert st.num_dictionary_batches == 4 |
| 630 | assert st.num_replaced_dictionaries == 3 |
| 631 | assert st.num_dictionary_deltas == 0 |
| 632 | |
| 633 | format_fixture.use_legacy_ipc_format = None |
| 634 | format_fixture.options = pa.ipc.IpcWriteOptions( |
| 635 | emit_dictionary_deltas=True) |
| 636 | if format_fixture.is_file: |
| 637 | # File format cannot handle replacement |
| 638 | with pytest.raises(pa.ArrowInvalid): |
| 639 | write_batches(batches) |
| 640 | else: |
| 641 | st = write_batches(batches) |
| 642 | assert st.num_record_batches == 5 |
| 643 | assert st.num_dictionary_batches == 4 |
| 644 | assert st.num_replaced_dictionaries == 1 |
| 645 | assert st.num_dictionary_deltas == 2 |
| 646 | |
| 647 | st = write_batches(batches_delta_only) |
| 648 | assert st.num_record_batches == 4 |
| 649 | assert st.num_dictionary_batches == 3 |
| 650 | assert st.num_replaced_dictionaries == 0 |
nothing calls this directly
no test coverage detected