| 214 | } |
| 215 | |
| 216 | void TestInvalidRowsSkipped(TableReaderFactory reader_factory, bool async) { |
| 217 | const int NROWS = 1000; |
| 218 | const int INVALID_EVERY = 20; |
| 219 | const int NINVALID = 50; |
| 220 | |
| 221 | auto opts = ParseOptions::Defaults(); |
| 222 | std::atomic<int> num_invalid_rows(0); |
| 223 | opts.invalid_row_handler = [&](const InvalidRow& row) { |
| 224 | auto cur_invalid_rows = ++num_invalid_rows; |
| 225 | if (async) { |
| 226 | // Row numbers are not counted in batches during async processing |
| 227 | EXPECT_EQ(-1, row.number); |
| 228 | } else { |
| 229 | // actual data starts at row #2 in the CSV "file" |
| 230 | EXPECT_EQ(cur_invalid_rows * INVALID_EVERY + 1, row.number); |
| 231 | } |
| 232 | return InvalidRowResult::Skip; |
| 233 | }; |
| 234 | |
| 235 | ASSERT_OK_AND_ASSIGN(auto table_buffer, MakeSampleCsvBuffer(NROWS, [=](size_t row_num) { |
| 236 | // row_num is 0-based |
| 237 | return (row_num + 1) % static_cast<size_t>(INVALID_EVERY) != 0; |
| 238 | })); |
| 239 | auto input = std::make_shared<io::BufferReader>(table_buffer); |
| 240 | ASSERT_OK_AND_ASSIGN(auto reader, |
| 241 | reader_factory(input, std::move(opts), /*block_size=*/{})); |
| 242 | ASSERT_OK_AND_ASSIGN(auto table, reader->Read()); |
| 243 | ASSERT_EQ(NROWS - NINVALID, table->num_rows()); |
| 244 | ASSERT_EQ(NINVALID, num_invalid_rows); |
| 245 | } |
| 246 | |
| 247 | TableReaderFactory MakeSerialFactory() { |
| 248 | return [](std::shared_ptr<io::InputStream> input_stream, ParseOptions parse_options, |