(self)
| 555 | self.check_reader(reader, explicit_schema, [expected_data]) |
| 556 | |
| 557 | def test_inference(self): |
| 558 | rows = b'{"a": 0, "b": "foo" }\n\ |
| 559 | {"a": 1, "c": true }\n{"a": 2, "d": 4.0}' |
| 560 | expected_schema = pa.schema([ |
| 561 | ('a', pa.int64()), |
| 562 | ('b', pa.utf8()) |
| 563 | ]) |
| 564 | expected_data = {'a': [0], 'b': ["foo"]} |
| 565 | |
| 566 | read_options = ReadOptions(block_size=32) |
| 567 | parse_options = ParseOptions(unexpected_field_behavior="infer") |
| 568 | reader = self.open_bytes( |
| 569 | rows, |
| 570 | read_options=read_options, |
| 571 | parse_options=parse_options) |
| 572 | assert reader.schema == expected_schema |
| 573 | assert reader.read_next_batch().to_pydict() == expected_data |
| 574 | with pytest.raises(pa.ArrowInvalid, |
| 575 | match="JSON parse error: unexpected field"): |
| 576 | reader.read_next_batch() |
| 577 | |
| 578 | expected_schema = pa.schema([ |
| 579 | ('a', pa.int64()), |
| 580 | ('b', pa.utf8()), |
| 581 | ('c', pa.bool_()), |
| 582 | ]) |
| 583 | expected_data = {'a': [0, 1], 'b': ["foo", None], 'c': [None, True]} |
| 584 | read_options = ReadOptions(block_size=64) |
| 585 | reader = self.open_bytes(rows, read_options=read_options, |
| 586 | parse_options=parse_options) |
| 587 | assert reader.schema == expected_schema |
| 588 | assert reader.read_next_batch().to_pydict() == expected_data |
| 589 | with pytest.raises(pa.ArrowInvalid, |
| 590 | match="JSON parse error: unexpected field"): |
| 591 | reader.read_next_batch() |
| 592 | |
| 593 | expected_schema = pa.schema([ |
| 594 | ('a', pa.int64()), |
| 595 | ('b', pa.utf8()), |
| 596 | ('c', pa.bool_()), |
| 597 | ('d', pa.float64()), |
| 598 | ]) |
| 599 | expected_data = {'a': [0, 1, 2], 'b': ["foo", None, None], |
| 600 | 'c': [None, True, None], 'd': [None, None, 4.0]} |
| 601 | read_options = ReadOptions(block_size=96) |
| 602 | reader = self.open_bytes(rows, read_options=read_options, |
| 603 | parse_options=parse_options) |
| 604 | assert reader.schema == expected_schema |
| 605 | assert reader.read_next_batch().to_pydict() == expected_data |
| 606 | |
| 607 | |
| 608 | class TestSerialJSONRead(BaseTestJSONRead, unittest.TestCase): |
nothing calls this directly
no test coverage detected