| 550 | template <typename SpecializedOptions, bool IgnoreExtraColumns, |
| 551 | typename ValueDescWriter, typename DataWriter, typename BulkFilter> |
| 552 | Status ParseChunk(ValueDescWriter* values_writer, DataWriter* parsed_writer, |
| 553 | const char* data, const char* data_end, bool is_final, |
| 554 | int32_t rows_in_chunk, const char** out_data, bool* finished_parsing, |
| 555 | const BulkFilter& bulk_filter) { |
| 556 | const int32_t start_num_rows = batch_.num_rows_; |
| 557 | const int32_t num_rows_deadline = batch_.num_rows_ + rows_in_chunk; |
| 558 | |
| 559 | if (use_bulk_filter_) { |
| 560 | while (data < data_end && batch_.num_rows_ < num_rows_deadline) { |
| 561 | const char* line_end = data; |
| 562 | RETURN_NOT_OK((ParseLine<SpecializedOptions, true, IgnoreExtraColumns>( |
| 563 | values_writer, parsed_writer, data, data_end, is_final, &line_end, |
| 564 | bulk_filter))); |
| 565 | RETURN_NOT_OK(values_writer->status()); |
| 566 | if (line_end == data) { |
| 567 | // Cannot parse any further |
| 568 | *finished_parsing = true; |
| 569 | break; |
| 570 | } |
| 571 | data = line_end; |
| 572 | } |
| 573 | } else { |
| 574 | while (data < data_end && batch_.num_rows_ < num_rows_deadline) { |
| 575 | const char* line_end = data; |
| 576 | RETURN_NOT_OK((ParseLine<SpecializedOptions, false, IgnoreExtraColumns>( |
| 577 | values_writer, parsed_writer, data, data_end, is_final, &line_end, |
| 578 | bulk_filter))); |
| 579 | RETURN_NOT_OK(values_writer->status()); |
| 580 | if (line_end == data) { |
| 581 | // Cannot parse any further |
| 582 | *finished_parsing = true; |
| 583 | break; |
| 584 | } |
| 585 | data = line_end; |
| 586 | } |
| 587 | } |
| 588 | |
| 589 | if (batch_.num_rows_ > start_num_rows && batch_.num_cols_ > 0) { |
| 590 | // Use bulk filter only if average value length is >= 10 bytes |
| 591 | // (its fixed cost isn't compensated for short values), and the block |
| 592 | // has no embedded NUL bytes (see block_has_nul_). |
| 593 | const int64_t bulk_filter_threshold = static_cast<int64_t>(batch_.num_cols_) * |
| 594 | (batch_.num_rows_ - start_num_rows) * 10; |
| 595 | use_bulk_filter_ = !block_has_nul_ && (data - *out_data) > bulk_filter_threshold; |
| 596 | } |
| 597 | |
| 598 | // Append new buffers and update size |
| 599 | ARROW_ASSIGN_OR_RAISE(auto values_buffer, values_writer->Finish()); |
| 600 | if (values_buffer->size() > 0) { |
| 601 | values_size_ += |
| 602 | static_cast<int32_t>(values_buffer->size() / sizeof(ParsedValueDesc) - 1); |
| 603 | batch_.values_buffers_.push_back(std::move(values_buffer)); |
| 604 | } |
| 605 | *out_data = data; |
| 606 | return Status::OK(); |
| 607 | } |
| 608 | |
| 609 | template <typename SpecializedOptions, bool IgnoreExtraColumns> |