Returns a dictionary containing enough metadata to reconstruct a pandas DataFrame as an Arrow Table, including index columns. Parameters ---------- columns_to_convert : list[pd.Series] df : pandas.DataFrame column_names : list[str | None] column_field_names: list[str]
(columns_to_convert, df, column_names, index_levels,
index_descriptors, preserve_index, types,
column_field_names=None)
| 195 | |
| 196 | |
| 197 | def construct_metadata(columns_to_convert, df, column_names, index_levels, |
| 198 | index_descriptors, preserve_index, types, |
| 199 | column_field_names=None): |
| 200 | """Returns a dictionary containing enough metadata to reconstruct a pandas |
| 201 | DataFrame as an Arrow Table, including index columns. |
| 202 | |
| 203 | Parameters |
| 204 | ---------- |
| 205 | columns_to_convert : list[pd.Series] |
| 206 | df : pandas.DataFrame |
| 207 | column_names : list[str | None] |
| 208 | column_field_names: list[str] |
| 209 | index_levels : List[pd.Index] |
| 210 | index_descriptors : List[Dict] |
| 211 | preserve_index : bool |
| 212 | types : List[pyarrow.DataType] |
| 213 | |
| 214 | Returns |
| 215 | ------- |
| 216 | dict |
| 217 | """ |
| 218 | if column_field_names is None: |
| 219 | # backwards compatibility for external projects that are using |
| 220 | # `construct_metadata` such as cudf |
| 221 | # see https://github.com/apache/arrow/pull/44963#discussion_r1875771953 |
| 222 | column_field_names = [str(name) for name in column_names] |
| 223 | |
| 224 | serialized_index_levels = [ |
| 225 | (level, descriptor) |
| 226 | for level, descriptor in zip(index_levels, index_descriptors) |
| 227 | if not isinstance(descriptor, dict) |
| 228 | ] |
| 229 | |
| 230 | num_serialized_index_levels = len(serialized_index_levels) |
| 231 | |
| 232 | # Use ntypes instead of Python shorthand notation [:-len(x)] as [:-0] |
| 233 | # behaves differently to what we want. |
| 234 | ntypes = len(types) |
| 235 | df_types = types[:ntypes - num_serialized_index_levels] |
| 236 | index_types = types[ntypes - num_serialized_index_levels:] |
| 237 | |
| 238 | column_metadata = [] |
| 239 | for col, name, field_name, arrow_type in zip(columns_to_convert, column_names, |
| 240 | column_field_names, df_types): |
| 241 | metadata = get_column_metadata(col, name=name, |
| 242 | arrow_type=arrow_type, |
| 243 | field_name=field_name) |
| 244 | column_metadata.append(metadata) |
| 245 | |
| 246 | index_column_metadata = [] |
| 247 | if preserve_index is not False: |
| 248 | non_str_index_names = [] |
| 249 | for (level, descriptor), arrow_type in zip( |
| 250 | serialized_index_levels, index_types |
| 251 | ): |
| 252 | if level.name is not None and not isinstance(level.name, str): |
| 253 | non_str_index_names.append(level.name) |
| 254 |
no test coverage detected