MCPcopy Create free account
hub / github.com/apache/arrow / test_batch_lifetime

Method test_batch_lifetime

python/pyarrow/tests/test_csv.py:1915–1941  ·  view source on GitHub ↗
(self)

Source from the content-addressed store, hash-verified

1913 assert table.to_pydict() == expected.to_pydict()
1914
1915 def test_batch_lifetime(self):
1916 gc.collect()
1917 old_allocated = pa.total_allocated_bytes()
1918
1919 # Memory occupation should not grow with CSV file size
1920 def check_one_batch(reader, expected):
1921 batch = reader.read_next_batch()
1922 assert batch.to_pydict() == expected
1923
1924 rows = b"10,11\n12,13\n14,15\n16,17\n"
1925 read_options = ReadOptions()
1926 read_options.column_names = ['a', 'b']
1927 read_options.block_size = 6
1928 reader = self.open_bytes(rows, read_options=read_options)
1929 check_one_batch(reader, {'a': [10], 'b': [11]})
1930 allocated_after_first_batch = pa.total_allocated_bytes()
1931 check_one_batch(reader, {'a': [12], 'b': [13]})
1932 assert pa.total_allocated_bytes() <= allocated_after_first_batch
1933 check_one_batch(reader, {'a': [14], 'b': [15]})
1934 assert pa.total_allocated_bytes() <= allocated_after_first_batch
1935 check_one_batch(reader, {'a': [16], 'b': [17]})
1936 assert pa.total_allocated_bytes() <= allocated_after_first_batch
1937 with pytest.raises(StopIteration):
1938 reader.read_next_batch()
1939 assert pa.total_allocated_bytes() == old_allocated
1940 reader = None
1941 assert pa.total_allocated_bytes() == old_allocated
1942
1943 def test_header_skip_rows(self):
1944 super().test_header_skip_rows()

Callers

nothing calls this directly

Calls 2

open_bytesMethod · 0.95
ReadOptionsClass · 0.90

Tested by

no test coverage detected