Method GetNextBuffer

be/src/exec/json/hdfs-json-scanner.cc:304–342

}

void HdfsJsonScanner::GetNextBuffer(const char** begin, const char** end) {
  DCHECK(*begin == *end);
  SCOPED_TIMER(get_buffer_timer_);

  // The eosr indicates that we have scanned all data within the scan range. If the
  // scanner state is OPENED, it means that we encountered eosr in FindFirstTuple(),
  // indicating that there is no start of tuple in this scan range, this will be handle by
  // the previous scan range in the same file. If the scanner state is SCANNING, it means
  // that we have completed scanning the data within the range, and need to read the next
  // range of data to complete the scan.
  if (stream_->eosr()) {
    if (scanner_state_ == OPENED) return;
    if (scanner_state_ == SCANNING) scanner_state_ = PAST_SCANNING;
  }

  if (stream_->eof() || scanner_state_ == FINISHED) return;

  uint8_t* next_buffer_begin = nullptr;
  int64_t next_buffer_size = 0;
  if (decompressor_ == nullptr) {
    buffer_status_ = FillBytesToBuffer(&next_buffer_begin, &next_buffer_size);
  } else if (decompressor_->supports_streaming()) {
    bool eosr = false;
    // The JsonParser always copies values instead of referencing them, so it doesn't
    // reference the data in the data_buffer_pool_. Therefore, we don't need current_pool_
    // to acquire data from the data_buffer_pool_, so we pass nullptr to
    // DecompressStreamToBuffer().
    buffer_status_ = DecompressStreamToBuffer(&next_buffer_begin, &next_buffer_size,
        nullptr, &eosr);
  } else {
    buffer_status_ = DecompressFileToBuffer(&next_buffer_begin, &next_buffer_size);
    if (next_buffer_size == 0) scanner_state_ = FINISHED;
  }
  RETURN_VOID_IF_ERROR(buffer_status_);
  if (UNLIKELY(next_buffer_size == 0)) return;

  *begin = reinterpret_cast<char*>(next_buffer_begin);
  *end = *begin + next_buffer_size;
}

void HdfsJsonScanner::InitRow() {
  InitTuple(template_tuple_, tuple_);