MCPcopy Create free account
hub / github.com/apache/arrow / test_write_metadata

Function test_write_metadata

python/pyarrow/tests/parquet/test_metadata.py:542–582  ·  view source on GitHub ↗
(tempdir)

Source from the content-addressed store, hash-verified

540
541@pytest.mark.filterwarnings("ignore:Parquet format:FutureWarning")
542def test_write_metadata(tempdir):
543 path = str(tempdir / "metadata")
544 schema = pa.schema([("a", "int64"), ("b", "float64")])
545
546 # write a pyarrow schema
547 pq.write_metadata(schema, path)
548 parquet_meta = pq.read_metadata(path)
549 schema_as_arrow = parquet_meta.schema.to_arrow_schema()
550 assert schema_as_arrow.equals(schema)
551
552 # ARROW-8980: Check that the ARROW:schema metadata key was removed
553 if schema_as_arrow.metadata:
554 assert b'ARROW:schema' not in schema_as_arrow.metadata
555
556 # pass through writer keyword arguments
557 for version in ["1.0", "2.4", "2.6"]:
558 pq.write_metadata(schema, path, version=version)
559 parquet_meta = pq.read_metadata(path)
560 # The version is stored as a single integer in the Parquet metadata,
561 # so it cannot correctly express dotted format versions
562 expected_version = "1.0" if version == "1.0" else "2.6"
563 assert parquet_meta.format_version == expected_version
564
565 # metadata_collector: list of FileMetaData objects
566 table = pa.table({'a': [1, 2], 'b': [.1, .2]}, schema=schema)
567 pq.write_table(table, tempdir / "data.parquet")
568 parquet_meta = pq.read_metadata(str(tempdir / "data.parquet"))
569 pq.write_metadata(
570 schema, path, metadata_collector=[parquet_meta, parquet_meta]
571 )
572 parquet_meta_mult = pq.read_metadata(path)
573 assert parquet_meta_mult.num_row_groups == 2
574
575 # append metadata with different schema raises an error
576 msg = ("AppendRowGroups requires equal schemas.\n"
577 "The two columns with index 0 differ.")
578 with pytest.raises(RuntimeError, match=msg):
579 pq.write_metadata(
580 pa.schema([("a", "int32"), ("b", "null")]),
581 path, metadata_collector=[parquet_meta, parquet_meta]
582 )
583
584
585def test_table_large_metadata():

Callers

nothing calls this directly

Calls 4

read_metadataMethod · 0.80
equalsMethod · 0.80
schemaMethod · 0.45
write_tableMethod · 0.45

Tested by

no test coverage detected