MCPcopy Create free account
hub / github.com/apache/arrow / test_auto_dict_encode

Method test_auto_dict_encode

python/pyarrow/tests/test_csv.py:1195–1242  ·  view source on GitHub ↗
(self)

Source from the content-addressed store, hash-verified

1193 }
1194
1195 def test_auto_dict_encode(self):
1196 opts = ConvertOptions(auto_dict_encode=True)
1197 rows = "a,b\nab,1\ncdé,2\ncdé,3\nab,4".encode()
1198 table = self.read_bytes(rows, convert_options=opts)
1199 schema = pa.schema([('a', pa.dictionary(pa.int32(), pa.string())),
1200 ('b', pa.int64())])
1201 expected = {
1202 'a': ["ab", "cdé", "cdé", "ab"],
1203 'b': [1, 2, 3, 4],
1204 }
1205 assert table.schema == schema
1206 assert table.to_pydict() == expected
1207
1208 opts.auto_dict_max_cardinality = 2
1209 table = self.read_bytes(rows, convert_options=opts)
1210 assert table.schema == schema
1211 assert table.to_pydict() == expected
1212
1213 # Cardinality above max => plain-encoded
1214 opts.auto_dict_max_cardinality = 1
1215 table = self.read_bytes(rows, convert_options=opts)
1216 assert table.schema == pa.schema([('a', pa.string()),
1217 ('b', pa.int64())])
1218 assert table.to_pydict() == expected
1219
1220 # With invalid UTF8, not checked
1221 opts.auto_dict_max_cardinality = 50
1222 opts.check_utf8 = False
1223 rows = b"a,b\nab,1\ncd\xff,2\nab,3"
1224 table = self.read_bytes(rows, convert_options=opts,
1225 validate_full=False)
1226 assert table.schema == schema
1227 dict_values = table['a'].chunk(0).dictionary
1228 assert len(dict_values) == 2
1229 assert dict_values[0].as_py() == "ab"
1230 assert dict_values[1].as_buffer() == b"cd\xff"
1231
1232 # With invalid UTF8, checked
1233 opts.check_utf8 = True
1234 table = self.read_bytes(rows, convert_options=opts)
1235 schema = pa.schema([('a', pa.dictionary(pa.int32(), pa.binary())),
1236 ('b', pa.int64())])
1237 expected = {
1238 'a': [b"ab", b"cd\xff", b"ab"],
1239 'b': [1, 2, 3],
1240 }
1241 assert table.schema == schema
1242 assert table.to_pydict() == expected
1243
1244 def test_custom_nulls(self):
1245 # Infer nulls with custom values

Callers

nothing calls this directly

Calls 8

read_bytesMethod · 0.95
ConvertOptionsClass · 0.90
lenFunction · 0.85
as_pyMethod · 0.80
encodeMethod · 0.45
schemaMethod · 0.45
dictionaryMethod · 0.45
stringMethod · 0.45

Tested by

no test coverage detected