Method FillBytesToBuffer

be/src/exec/json/hdfs-json-scanner.cc:277–302

}

Status HdfsJsonScanner::FillBytesToBuffer(uint8_t** buffer, int64_t* bytes_read) {
  if (scanner_state_ == PAST_SCANNING) {
    // In the PAST_SCANNING state, we only read a small block data at a time for scanning.
    // If the parser completes the parsing of the last json object, it will exit the loop
    // due to BreakParse().
    Status status;
    if (UNLIKELY(!stream_->GetBytes(NEXT_BLOCK_READ_SIZE, buffer, bytes_read, &status))) {
      DCHECK(!status.ok());
      return status;
    }

    // A special case is when the first character of the next scan range is a newline
    // character (perhaps with other whitespace characters before it). Our scan should
    // stop at the first newline character in the next range, while the parser skips
    // whitespace characters. If we don't handle this case, the first line of the next
    // range will be scanned twice. Therefore, we need to return directly here to inform
    // the parser that eos has been reached.
    if (UNLIKELY(AllWhitespaceBeforeNewline(*buffer, *bytes_read))) {
      scanner_state_ = FINISHED;
      *bytes_read = 0;
    }
  } else {
    RETURN_IF_ERROR(stream_->GetBuffer(false, buffer, bytes_read));
  }
  return Status::OK();
}

void HdfsJsonScanner::GetNextBuffer(const char** begin, const char** end) {
  DCHECK(*begin == *end);