MCPcopy Create free account
hub / github.com/apache/arrow / test_specified_schema

Function test_specified_schema

python/pyarrow/tests/test_dataset.py:3299–3353  ·  view source on GitHub ↗
(tempdir, dataset_reader)

Source from the content-addressed store, hash-verified

3297
3298@pytest.mark.parquet
3299def test_specified_schema(tempdir, dataset_reader):
3300 table = pa.table({'a': [1, 2, 3], 'b': [.1, .2, .3]})
3301 pq.write_table(table, tempdir / "data.parquet")
3302
3303 def _check_dataset(schema, expected, expected_schema=None):
3304 dataset = ds.dataset(str(tempdir / "data.parquet"), schema=schema)
3305 if expected_schema is not None:
3306 assert dataset.schema.equals(expected_schema)
3307 else:
3308 assert dataset.schema.equals(schema)
3309 result = dataset_reader.to_table(dataset)
3310 assert result.equals(expected)
3311
3312 # no schema specified
3313 schema = None
3314 expected = table
3315 _check_dataset(schema, expected, expected_schema=table.schema)
3316
3317 # identical schema specified
3318 schema = table.schema
3319 expected = table
3320 _check_dataset(schema, expected)
3321
3322 # Specifying schema with change column order
3323 schema = pa.schema([('b', 'float64'), ('a', 'int64')])
3324 expected = pa.table([[.1, .2, .3], [1, 2, 3]], names=['b', 'a'])
3325 _check_dataset(schema, expected)
3326
3327 # Specifying schema with missing column
3328 schema = pa.schema([('a', 'int64')])
3329 expected = pa.table([[1, 2, 3]], names=['a'])
3330 _check_dataset(schema, expected)
3331
3332 # Specifying schema with additional column
3333 schema = pa.schema([('a', 'int64'), ('c', 'int32')])
3334 expected = pa.table([[1, 2, 3],
3335 pa.array([None, None, None], type='int32')],
3336 names=['a', 'c'])
3337 _check_dataset(schema, expected)
3338
3339 # Specifying with differing field types
3340 schema = pa.schema([('a', 'int32'), ('b', 'float64')])
3341 dataset = ds.dataset(str(tempdir / "data.parquet"), schema=schema)
3342 expected = pa.table([table['a'].cast('int32'),
3343 table['b']],
3344 names=['a', 'b'])
3345 _check_dataset(schema, expected)
3346
3347 # Specifying with incompatible schema
3348 schema = pa.schema([('a', pa.list_(pa.int32())), ('b', 'float64')])
3349 dataset = ds.dataset(str(tempdir / "data.parquet"), schema=schema)
3350 assert dataset.schema.equals(schema)
3351 with pytest.raises(NotImplementedError,
3352 match='Unsupported cast from int64 to list'):
3353 dataset_reader.to_table(dataset)
3354
3355
3356@pytest.mark.parquet

Callers

nothing calls this directly

Calls 7

_check_datasetFunction · 0.85
equalsMethod · 0.80
write_tableMethod · 0.45
schemaMethod · 0.45
arrayMethod · 0.45
castMethod · 0.45
to_tableMethod · 0.45

Tested by

no test coverage detected