(format_fixture)
| 615 | |
| 616 | |
| 617 | def test_dictionary_delta(format_fixture): |
| 618 | ty = pa.dictionary(pa.int8(), pa.utf8()) |
| 619 | data = [["foo", "foo", None], |
| 620 | ["foo", "bar", "foo"], # potential delta |
| 621 | ["foo", "bar"], # nothing new |
| 622 | ["foo", None, "bar", "quux"], # potential delta |
| 623 | ["bar", "quux"], # replacement |
| 624 | ] |
| 625 | batches = [ |
| 626 | pa.RecordBatch.from_arrays([pa.array(v, type=ty)], names=['dicts']) |
| 627 | for v in data] |
| 628 | batches_delta_only = batches[:4] |
| 629 | schema = batches[0].schema |
| 630 | |
| 631 | def write_batches(batches, as_table=False): |
| 632 | with format_fixture._get_writer(pa.MockOutputStream(), |
| 633 | schema) as writer: |
| 634 | if as_table: |
| 635 | table = pa.Table.from_batches(batches) |
| 636 | writer.write_table(table) |
| 637 | else: |
| 638 | for batch in batches: |
| 639 | writer.write_batch(batch) |
| 640 | return writer.stats |
| 641 | |
| 642 | if format_fixture.is_file: |
| 643 | # File format cannot handle replacement |
| 644 | with pytest.raises(pa.ArrowInvalid): |
| 645 | write_batches(batches) |
| 646 | # File format cannot handle delta if emit_deltas |
| 647 | # is not provided |
| 648 | with pytest.raises(pa.ArrowInvalid): |
| 649 | write_batches(batches_delta_only) |
| 650 | else: |
| 651 | st = write_batches(batches) |
| 652 | assert st.num_record_batches == 5 |
| 653 | assert st.num_dictionary_batches == 4 |
| 654 | assert st.num_replaced_dictionaries == 3 |
| 655 | assert st.num_dictionary_deltas == 0 |
| 656 | |
| 657 | format_fixture.use_legacy_ipc_format = None |
| 658 | format_fixture.options = pa.ipc.IpcWriteOptions( |
| 659 | emit_dictionary_deltas=True) |
| 660 | if format_fixture.is_file: |
| 661 | # File format cannot handle replacement |
| 662 | with pytest.raises(pa.ArrowInvalid): |
| 663 | write_batches(batches) |
| 664 | else: |
| 665 | st = write_batches(batches) |
| 666 | assert st.num_record_batches == 5 |
| 667 | assert st.num_dictionary_batches == 4 |
| 668 | assert st.num_replaced_dictionaries == 1 |
| 669 | assert st.num_dictionary_deltas == 2 |
| 670 | |
| 671 | st = write_batches(batches_delta_only) |
| 672 | assert st.num_record_batches == 4 |
| 673 | assert st.num_dictionary_batches == 3 |
| 674 | assert st.num_replaced_dictionaries == 0 |
nothing calls this directly
no test coverage detected