MCPcopy Create free account
hub / github.com/apache/arrow / test_inference

Method test_inference

python/pyarrow/tests/test_json.py:573–621  ·  view source on GitHub ↗
(self)

Source from the content-addressed store, hash-verified

571 self.check_reader(reader, explicit_schema, [expected_data])
572
573 def test_inference(self):
574 rows = b'{"a": 0, "b": "foo" }\n\
575 {"a": 1, "c": true }\n{"a": 2, "d": 4.0}'
576 expected_schema = pa.schema([
577 ('a', pa.int64()),
578 ('b', pa.utf8())
579 ])
580 expected_data = {'a': [0], 'b': ["foo"]}
581
582 read_options = ReadOptions(block_size=32)
583 parse_options = ParseOptions(unexpected_field_behavior="infer")
584 reader = self.open_bytes(
585 rows,
586 read_options=read_options,
587 parse_options=parse_options)
588 assert reader.schema == expected_schema
589 assert reader.read_next_batch().to_pydict() == expected_data
590 with pytest.raises(pa.ArrowInvalid,
591 match="JSON parse error: unexpected field"):
592 reader.read_next_batch()
593
594 expected_schema = pa.schema([
595 ('a', pa.int64()),
596 ('b', pa.utf8()),
597 ('c', pa.bool_()),
598 ])
599 expected_data = {'a': [0, 1], 'b': ["foo", None], 'c': [None, True]}
600 read_options = ReadOptions(block_size=64)
601 reader = self.open_bytes(rows, read_options=read_options,
602 parse_options=parse_options)
603 assert reader.schema == expected_schema
604 assert reader.read_next_batch().to_pydict() == expected_data
605 with pytest.raises(pa.ArrowInvalid,
606 match="JSON parse error: unexpected field"):
607 reader.read_next_batch()
608
609 expected_schema = pa.schema([
610 ('a', pa.int64()),
611 ('b', pa.utf8()),
612 ('c', pa.bool_()),
613 ('d', pa.float64()),
614 ])
615 expected_data = {'a': [0, 1, 2], 'b': ["foo", None, None],
616 'c': [None, True, None], 'd': [None, None, 4.0]}
617 read_options = ReadOptions(block_size=96)
618 reader = self.open_bytes(rows, read_options=read_options,
619 parse_options=parse_options)
620 assert reader.schema == expected_schema
621 assert reader.read_next_batch().to_pydict() == expected_data
622
623
624class TestSerialJSONRead(BaseTestJSONRead, unittest.TestCase):

Callers

nothing calls this directly

Calls 4

open_bytesMethod · 0.95
ReadOptionsClass · 0.90
ParseOptionsClass · 0.90
schemaMethod · 0.45

Tested by

no test coverage detected