Dataframe with numpy arrays columns of every possible primitive type. Returns ------- df: pandas.DataFrame schema: pyarrow.Schema Arrow schema definition that is in line with the constructed df.
(include_index=False)
| 24 | |
| 25 | |
| 26 | def dataframe_with_arrays(include_index=False): |
| 27 | """ |
| 28 | Dataframe with numpy arrays columns of every possible primitive type. |
| 29 | |
| 30 | Returns |
| 31 | ------- |
| 32 | df: pandas.DataFrame |
| 33 | schema: pyarrow.Schema |
| 34 | Arrow schema definition that is in line with the constructed df. |
| 35 | """ |
| 36 | dtypes = [('i1', pa.int8()), ('i2', pa.int16()), |
| 37 | ('i4', pa.int32()), ('i8', pa.int64()), |
| 38 | ('u1', pa.uint8()), ('u2', pa.uint16()), |
| 39 | ('u4', pa.uint32()), ('u8', pa.uint64()), |
| 40 | ('f4', pa.float32()), ('f8', pa.float64())] |
| 41 | |
| 42 | arrays = OrderedDict() |
| 43 | fields = [] |
| 44 | for dtype, arrow_dtype in dtypes: |
| 45 | fields.append(pa.field(dtype, pa.list_(arrow_dtype))) |
| 46 | arrays[dtype] = [ |
| 47 | np.arange(10, dtype=dtype), |
| 48 | np.arange(5, dtype=dtype), |
| 49 | None, |
| 50 | np.arange(1, dtype=dtype) |
| 51 | ] |
| 52 | |
| 53 | fields.append(pa.field('str', pa.list_(pa.string()))) |
| 54 | arrays['str'] = [ |
| 55 | np.array(["1", "ä"], dtype="object"), |
| 56 | None, |
| 57 | np.array(["1"], dtype="object"), |
| 58 | np.array(["1", "2", "3"], dtype="object") |
| 59 | ] |
| 60 | |
| 61 | fields.append(pa.field('datetime64', pa.list_(pa.timestamp('ms')))) |
| 62 | arrays['datetime64'] = [ |
| 63 | np.array(['2007-07-13T01:23:34.123456789', |
| 64 | None, |
| 65 | '2010-08-13T05:46:57.437699912'], |
| 66 | dtype='datetime64[ms]'), |
| 67 | None, |
| 68 | None, |
| 69 | np.array(['2007-07-13T02', |
| 70 | None, |
| 71 | '2010-08-13T05:46:57.437699912'], |
| 72 | dtype='datetime64[ms]'), |
| 73 | ] |
| 74 | |
| 75 | if include_index: |
| 76 | fields.append(pa.field('__index_level_0__', pa.int64())) |
| 77 | df = pd.DataFrame(arrays) |
| 78 | schema = pa.schema(fields) |
| 79 | |
| 80 | return df, schema |
| 81 | |
| 82 | |
| 83 | def dataframe_with_lists(include_index=False, parquet_compatible=False): |