MCPcopy Create free account
hub / github.com/apache/arrow / _add_any_metadata

Function _add_any_metadata

python/pyarrow/pandas_compat.py:1214–1272  ·  view source on GitHub ↗
(table, pandas_metadata)

Source from the content-addressed store, hash-verified

1212
1213
1214def _add_any_metadata(table, pandas_metadata):
1215 modified_columns = {}
1216 modified_fields = {}
1217
1218 schema = table.schema
1219
1220 index_columns = pandas_metadata['index_columns']
1221 # only take index columns into account if they are an actual table column
1222 index_columns = [idx_col for idx_col in index_columns
1223 if isinstance(idx_col, str)]
1224 n_index_levels = len(index_columns)
1225 n_columns = len(pandas_metadata['columns']) - n_index_levels
1226
1227 # Add time zones
1228 for i, col_meta in enumerate(pandas_metadata['columns']):
1229
1230 raw_name = col_meta.get('field_name')
1231 if not raw_name:
1232 # deal with metadata written with arrow < 0.8 or fastparquet
1233 raw_name = col_meta['name']
1234 if i >= n_columns:
1235 # index columns
1236 raw_name = index_columns[i - n_columns]
1237 if raw_name is None:
1238 raw_name = 'None'
1239
1240 idx = schema.get_field_index(raw_name)
1241 if idx != -1:
1242 if col_meta['pandas_type'] == 'datetimetz':
1243 col = table[idx]
1244 if not isinstance(col.type, pa.lib.TimestampType):
1245 continue
1246 metadata = col_meta['metadata']
1247 if not metadata:
1248 continue
1249 metadata_tz = metadata.get('timezone')
1250 if metadata_tz and metadata_tz != col.type.tz:
1251 converted = col.to_pandas()
1252 tz_aware_type = pa.timestamp('ns', tz=metadata_tz)
1253 with_metadata = pa.Array.from_pandas(converted,
1254 type=tz_aware_type)
1255
1256 modified_fields[idx] = pa.field(schema[idx].name,
1257 tz_aware_type)
1258 modified_columns[idx] = with_metadata
1259
1260 if len(modified_columns) > 0:
1261 columns = []
1262 fields = []
1263 for i in range(len(table.schema)):
1264 if i in modified_columns:
1265 columns.append(modified_columns[i])
1266 fields.append(modified_fields[i])
1267 else:
1268 columns.append(table[i])
1269 fields.append(table.schema[i])
1270 return pa.Table.from_arrays(columns, schema=pa.schema(fields))
1271 else:

Callers 1

table_to_dataframeFunction · 0.85

Calls 5

lenFunction · 0.85
getMethod · 0.45
fieldMethod · 0.45
appendMethod · 0.45
schemaMethod · 0.45

Tested by

no test coverage detected