MCPcopy Create free account
hub / github.com/apache/arrow / test_chunker_out_of_sync

Method test_chunker_out_of_sync

python/pyarrow/tests/test_csv.py:769–792  ·  view source on GitHub ↗
(self)

Source from the content-addressed store, hash-verified

767 }
768
769 def test_chunker_out_of_sync(self):
770 # GH-39892: if there are newlines in values, the parser may become
771 # out of sync with the chunker. In this case, we try to produce an
772 # informative error message.
773 rows = b"""a,b,c\nd,e,"f\n"\ng,h,i\n"""
774 expected = {
775 'a': ["d", "g"],
776 'b': ["e", "h"],
777 'c': ["f\n", "i"],
778 }
779 for block_size in range(8, 15):
780 # Sanity check: parsing works with newlines_in_values=True
781 d = self.read_bytes(
782 rows, parse_options=ParseOptions(newlines_in_values=True),
783 read_options=ReadOptions(block_size=block_size)).to_pydict()
784 assert d == expected
785 # With these block sizes, a block would end on the physical newline
786 # inside the quoted cell value, leading to a mismatch between
787 # CSV chunker and parser.
788 for block_size in range(8, 11):
789 with pytest.raises(ValueError,
790 match="cell values spanning multiple lines"):
791 self.read_bytes(
792 rows, read_options=ReadOptions(block_size=block_size))
793
794
795class BaseCSVTableRead(BaseTestCSV):

Callers

nothing calls this directly

Calls 3

read_bytesMethod · 0.95
ParseOptionsClass · 0.90
ReadOptionsClass · 0.90

Tested by

no test coverage detected