Function processFile

tools/src/FileMemory.cc:59–93

TestMemoryPool::~TestMemoryPool() {}

void processFile(const char* filename, const orc::RowReaderOptions& rowReaderOpts,
                 uint64_t batchSize) {
  orc::ReaderOptions readerOpts;
  std::unique_ptr<orc::MemoryPool> pool(new TestMemoryPool());
  readerOpts.setMemoryPool(*(pool.get()));

  std::unique_ptr<orc::Reader> reader = orc::createReader(
      orc::readFile(std::string(filename), readerOpts.getReaderMetrics()), readerOpts);
  std::unique_ptr<orc::RowReader> rowReader = reader->createRowReader(rowReaderOpts);

  std::unique_ptr<orc::ColumnVectorBatch> batch = rowReader->createRowBatch(batchSize);
  uint64_t readerMemory;
  if (rowReaderOpts.getIndexesSet()) {
    readerMemory = reader->getMemoryUseByFieldId(rowReaderOpts.getInclude());
  } else if (rowReaderOpts.getNamesSet()) {
    readerMemory = reader->getMemoryUseByName(rowReaderOpts.getIncludeNames());
  } else if (rowReaderOpts.getTypeIdsSet()) {
    readerMemory = reader->getMemoryUseByTypeId(rowReaderOpts.getInclude());
  } else {
    // default is to select all columns
    readerMemory = reader->getMemoryUseByName({});
  }

  uint64_t batchMemory = batch->getMemoryUsage();
  while (rowReader->next(*batch)) {
  }
  uint64_t actualMemory = static_cast<TestMemoryPool*>(pool.get())->getMaxMemory();
  std::cout << "Reader memory estimate: " << readerMemory << "\nBatch memory estimate:  ";
  if (batch->hasVariableLength()) {
    std::cout << "Cannot estimate because reading ARRAY or MAP columns";
  } else {
    std::cout << batchMemory << "\nTotal memory estimate:  " << readerMemory + batchMemory;
  }
  std::cout << "\nActual max memory used: " << actualMemory << "\n";
}

int main(int argc, char* argv[]) {
  uint64_t batchSize = 1000;