MCPcopy Create free account
hub / github.com/apache/arrow / ParseChunk

Method ParseChunk

cpp/src/arrow/csv/parser.cc:552–607  ·  view source on GitHub ↗

Source from the content-addressed store, hash-verified

550 template <typename SpecializedOptions, bool IgnoreExtraColumns,
551 typename ValueDescWriter, typename DataWriter, typename BulkFilter>
552 Status ParseChunk(ValueDescWriter* values_writer, DataWriter* parsed_writer,
553 const char* data, const char* data_end, bool is_final,
554 int32_t rows_in_chunk, const char** out_data, bool* finished_parsing,
555 const BulkFilter& bulk_filter) {
556 const int32_t start_num_rows = batch_.num_rows_;
557 const int32_t num_rows_deadline = batch_.num_rows_ + rows_in_chunk;
558
559 if (use_bulk_filter_) {
560 while (data < data_end && batch_.num_rows_ < num_rows_deadline) {
561 const char* line_end = data;
562 RETURN_NOT_OK((ParseLine<SpecializedOptions, true, IgnoreExtraColumns>(
563 values_writer, parsed_writer, data, data_end, is_final, &line_end,
564 bulk_filter)));
565 RETURN_NOT_OK(values_writer->status());
566 if (line_end == data) {
567 // Cannot parse any further
568 *finished_parsing = true;
569 break;
570 }
571 data = line_end;
572 }
573 } else {
574 while (data < data_end && batch_.num_rows_ < num_rows_deadline) {
575 const char* line_end = data;
576 RETURN_NOT_OK((ParseLine<SpecializedOptions, false, IgnoreExtraColumns>(
577 values_writer, parsed_writer, data, data_end, is_final, &line_end,
578 bulk_filter)));
579 RETURN_NOT_OK(values_writer->status());
580 if (line_end == data) {
581 // Cannot parse any further
582 *finished_parsing = true;
583 break;
584 }
585 data = line_end;
586 }
587 }
588
589 if (batch_.num_rows_ > start_num_rows && batch_.num_cols_ > 0) {
590 // Use bulk filter only if average value length is >= 10 bytes
591 // (its fixed cost isn't compensated for short values), and the block
592 // has no embedded NUL bytes (see block_has_nul_).
593 const int64_t bulk_filter_threshold = static_cast<int64_t>(batch_.num_cols_) *
594 (batch_.num_rows_ - start_num_rows) * 10;
595 use_bulk_filter_ = !block_has_nul_ && (data - *out_data) > bulk_filter_threshold;
596 }
597
598 // Append new buffers and update size
599 ARROW_ASSIGN_OR_RAISE(auto values_buffer, values_writer->Finish());
600 if (values_buffer->size() > 0) {
601 values_size_ +=
602 static_cast<int32_t>(values_buffer->size() / sizeof(ParsedValueDesc) - 1);
603 batch_.values_buffers_.push_back(std::move(values_buffer));
604 }
605 *out_data = data;
606 return Status::OK();
607 }
608
609 template <typename SpecializedOptions, bool IgnoreExtraColumns>

Callers

nothing calls this directly

Calls 4

push_backMethod · 0.80
OKFunction · 0.50
statusMethod · 0.45
sizeMethod · 0.45

Tested by

no test coverage detected