MCPcopy Create free account
hub / github.com/apache/arrow / InitFromBlock

Method InitFromBlock

cpp/src/arrow/csv/reader.cc:914–960  ·  view source on GitHub ↗

Source from the content-addressed store, hash-verified

912 }
913
914 Future<> InitFromBlock(const DecodedBlock& block,
915 AsyncGenerator<DecodedBlock> batch_gen, int max_readahead,
916 int64_t prev_bytes_processed) {
917 if (!block.record_batch) {
918 // End of file just return null batches
919 record_batch_gen_ = MakeEmptyGenerator<std::shared_ptr<RecordBatch>>();
920 return Status::OK();
921 }
922
923 schema_ = block.record_batch->schema();
924
925 if (block.record_batch->num_rows() == 0) {
926 // Keep consuming blocks until the first non empty block is found
927 auto self = shared_from_this();
928 prev_bytes_processed += block.bytes_processed;
929 return batch_gen().Then([self, batch_gen, max_readahead,
930 prev_bytes_processed](const DecodedBlock& next_block) {
931 return self->InitFromBlock(next_block, std::move(batch_gen), max_readahead,
932 prev_bytes_processed);
933 });
934 }
935
936 AsyncGenerator<DecodedBlock> readahead_gen;
937 if (read_options_.use_threads) {
938 readahead_gen = MakeReadaheadGenerator(std::move(batch_gen), max_readahead);
939 } else {
940 readahead_gen = std::move(batch_gen);
941 }
942
943 AsyncGenerator<DecodedBlock> restarted_gen =
944 MakeGeneratorStartsWith({block}, std::move(readahead_gen));
945
946 auto bytes_decoded = bytes_decoded_;
947 auto unwrap_and_record_bytes =
948 [bytes_decoded, prev_bytes_processed](
949 const DecodedBlock& block) mutable -> Result<std::shared_ptr<RecordBatch>> {
950 bytes_decoded->fetch_add(block.bytes_processed + prev_bytes_processed);
951 prev_bytes_processed = 0;
952 return block.record_batch;
953 };
954
955 auto unwrapped =
956 MakeMappedGenerator(std::move(restarted_gen), std::move(unwrap_and_record_bytes));
957
958 record_batch_gen_ = MakeCancellable(std::move(unwrapped), io_context_.stop_token());
959 return Status::OK();
960 }
961
962 std::shared_ptr<Schema> schema_;
963 AsyncGenerator<std::shared_ptr<RecordBatch>> record_batch_gen_;

Callers 1

InitAfterFirstBufferMethod · 0.95

Calls 9

MakeReadaheadGeneratorFunction · 0.85
MakeGeneratorStartsWithFunction · 0.85
MakeMappedGeneratorFunction · 0.85
MakeCancellableFunction · 0.85
ThenMethod · 0.80
OKFunction · 0.50
batch_genFunction · 0.50
schemaMethod · 0.45
num_rowsMethod · 0.45

Tested by

no test coverage detected