Class BatchData

examples/BatchData.h:41–175

#include "BatchDataCPU.h"

class BatchData
{
public:
  BatchData(
      const std::vector<std::vector<char>>& host_data,
      const size_t chunk_size) :
      m_ptrs(),
      m_sizes(),
      m_data(),
      m_size(0)
  {
    m_size = compute_batch_size(host_data, chunk_size);

    m_data = nvcomp::thrust::device_vector<uint8_t>(chunk_size * size());

    std::vector<void*> uncompressed_ptrs(size());
    for (size_t i = 0; i < size(); ++i) {
      uncompressed_ptrs[i] = static_cast<void*>(data() + chunk_size * i);
    }

    m_ptrs = nvcomp::thrust::device_vector<void*>(uncompressed_ptrs);
    std::vector<size_t> sizes
        = compute_chunk_sizes(host_data, size(), chunk_size);
    m_sizes = nvcomp::thrust::device_vector<size_t>(sizes);

    // copy data to GPU
    size_t offset = 0;
    for (size_t i = 0; i < host_data.size(); ++i) {
      CUDA_CHECK(cudaMemcpy(
          uncompressed_ptrs[offset],
          host_data[i].data(),
          host_data[i].size(),
          cudaMemcpyHostToDevice));

      const size_t num_chunks
          = (host_data[i].size() + chunk_size - 1) / chunk_size;
      offset += num_chunks;
    }
  }

  BatchData(const BatchDataCPU& batch_data, bool copy_data = false) :
      m_ptrs(),
      m_sizes(),
      m_data(),
      m_size()
  {
    m_size = batch_data.size();
    m_sizes = nvcomp::thrust::device_vector<size_t>(
        batch_data.sizes(), batch_data.sizes() + size());

    size_t data_size = std::accumulate(
        batch_data.sizes(),
        batch_data.sizes() + size(),
        static_cast<size_t>(0));
    m_data = nvcomp::thrust::device_vector<uint8_t>(data_size);

    size_t offset = 0;
    std::vector<void*> ptrs(size());