Method ReadAndSend

IO/ParallelNetCDF/vtkPNetCDFPOPReader.cxx:529–621

------------------------------------------------------------------------------ Helper function for RequestData. Reads one or more depth arrays from the netCDF file and sends sub-arrays out to all ranks that need that data

// Helper function for RequestData.  Reads one or more depth arrays from the
// netCDF file and sends sub-arrays out to all ranks that need that data
int vtkPNetCDFPOPReader::ReadAndSend(vtkInformation* outInfo, int varID)
{
  int wholeExtent[6];
  outInfo->Get(vtkStreamingDemandDrivenPipeline::WHOLE_EXTENT(), wholeExtent);
  // Z & X dimensions have been swapped (see the comments at the top of the header
  // file).  All arrays in this function, however, should reflect the on-disk
  // layout, so we'll 'un-swap' the X & Z extents.
  swap(wholeExtent[0], wholeExtent[4]);
  swap(wholeExtent[1], wholeExtent[5]);

  // this->Stride is also in the in-memory layout
  ptrdiff_t rStride[3] = { this->Stride[2], this->Stride[1], this->Stride[0] };

  int rank = this->Controller->GetLocalProcessId();

  // We read one depth at a time, skipping over the depths that other reader processes will read
  for (int curDepth = wholeExtent[0]; curDepth <= wholeExtent[1]; curDepth++)
  {
    if (ReaderForDepth(curDepth) == rank)
    {
      size_t start[3] = { static_cast<size_t>(curDepth * rStride[0]),
        static_cast<size_t>(wholeExtent[2]), static_cast<size_t>(wholeExtent[4]) };
      size_t count[3] = { 1, static_cast<size_t>(wholeExtent[3] - wholeExtent[2] + 1),
        static_cast<size_t>(wholeExtent[5] - wholeExtent[4] + 1) };

      float* buffer = new float[count[1] * count[2]];
      this->Internals->SendBufs.push_back(buffer);

      int ncErr = nc_get_vars_float(this->NCDFFD, varID, start, count, rStride, buffer);
      if (ncErr != NC_NOERR)
      {
        std::cerr << "!!!nc_get_vars_float() returned error code " << ncErr << endl;
      }

      // Create sub arrays and send to all processes
      for (int destRank = 0; destRank < this->Controller->GetNumberOfProcesses(); destRank++)
      {
        int destExtent[6];
        memcpy(destExtent, &this->Internals->AllExtents[destRank * 6], 6 * sizeof(int));
        // Note that AllExtents is also in in-memory layout order, so we need to swap
        // the X & Z values
        swap(destExtent[0], destExtent[4]);
        swap(destExtent[1], destExtent[5]);

        // Verify that destRank does, in fact, receive an extent at this depth
        // TODO: Don't use MPI to send data to ourself.....
        if (curDepth >= destExtent[0] && curDepth <= destExtent[1])
        {
          int subarray_sizes[2] = {
            (wholeExtent[3] - wholeExtent[2] + 1),
            (wholeExtent[5] - wholeExtent[4] + 1),
          };
          int subarray_subsizes[2] = { (destExtent[3] - destExtent[2] + 1),
            (destExtent[5] - destExtent[4] + 1) };
          int subarray_starts[2] = { destExtent[2], destExtent[4] };
          MPI_Datatype subArrayType;
          vtkMPICommunicator::Request sendReq;
          MPI_Type_create_subarray(2, subarray_sizes, subarray_subsizes, subarray_starts,