(self)
| 571 | self.check_reader(reader, explicit_schema, [expected_data]) |
| 572 | |
| 573 | def test_inference(self): |
| 574 | rows = b'{"a": 0, "b": "foo" }\n\ |
| 575 | {"a": 1, "c": true }\n{"a": 2, "d": 4.0}' |
| 576 | expected_schema = pa.schema([ |
| 577 | ('a', pa.int64()), |
| 578 | ('b', pa.utf8()) |
| 579 | ]) |
| 580 | expected_data = {'a': [0], 'b': ["foo"]} |
| 581 | |
| 582 | read_options = ReadOptions(block_size=32) |
| 583 | parse_options = ParseOptions(unexpected_field_behavior="infer") |
| 584 | reader = self.open_bytes( |
| 585 | rows, |
| 586 | read_options=read_options, |
| 587 | parse_options=parse_options) |
| 588 | assert reader.schema == expected_schema |
| 589 | assert reader.read_next_batch().to_pydict() == expected_data |
| 590 | with pytest.raises(pa.ArrowInvalid, |
| 591 | match="JSON parse error: unexpected field"): |
| 592 | reader.read_next_batch() |
| 593 | |
| 594 | expected_schema = pa.schema([ |
| 595 | ('a', pa.int64()), |
| 596 | ('b', pa.utf8()), |
| 597 | ('c', pa.bool_()), |
| 598 | ]) |
| 599 | expected_data = {'a': [0, 1], 'b': ["foo", None], 'c': [None, True]} |
| 600 | read_options = ReadOptions(block_size=64) |
| 601 | reader = self.open_bytes(rows, read_options=read_options, |
| 602 | parse_options=parse_options) |
| 603 | assert reader.schema == expected_schema |
| 604 | assert reader.read_next_batch().to_pydict() == expected_data |
| 605 | with pytest.raises(pa.ArrowInvalid, |
| 606 | match="JSON parse error: unexpected field"): |
| 607 | reader.read_next_batch() |
| 608 | |
| 609 | expected_schema = pa.schema([ |
| 610 | ('a', pa.int64()), |
| 611 | ('b', pa.utf8()), |
| 612 | ('c', pa.bool_()), |
| 613 | ('d', pa.float64()), |
| 614 | ]) |
| 615 | expected_data = {'a': [0, 1, 2], 'b': ["foo", None, None], |
| 616 | 'c': [None, True, None], 'd': [None, None, 4.0]} |
| 617 | read_options = ReadOptions(block_size=96) |
| 618 | reader = self.open_bytes(rows, read_options=read_options, |
| 619 | parse_options=parse_options) |
| 620 | assert reader.schema == expected_schema |
| 621 | assert reader.read_next_batch().to_pydict() == expected_data |
| 622 | |
| 623 | |
| 624 | class TestSerialJSONRead(BaseTestJSONRead, unittest.TestCase): |
nothing calls this directly
no test coverage detected