Method load

src/data_readers/data_reader_csv.cpp:109–279

}

void csv_reader::load()
{
  bool master = m_comm->am_world_master();
  setup_ifstreams();
  std::ifstream& ifs = *m_ifstreams[0];
  const El::mpi::Comm& world_comm = m_comm->get_world_comm();
  // Parse the header to determine how many columns there are.
  // Skip rows if needed.
  if (master) {
    skip_rows(ifs, m_skip_rows);
  }
  m_comm->broadcast<int>(0, m_skip_rows, world_comm);

  // This will be broadcast from root to other procs, and will
  // then be converted to std::vector<int> m_labels; this is because
  // El::mpi::Broadcast<std::streampos> doesn't work
  std::vector<long long> index;

  if (master) {
    std::string line;
    std::streampos header_start = ifs.tellg();
    // TODO: Skip comment lines.
    if (std::getline(ifs, line)) {
      m_num_cols = std::count(line.begin(), line.end(), m_separator) + 1;
      if (m_skip_cols >= m_num_cols) {
        throw lbann_exception(
          "csv_reader: asked to skip more columns than are present");
      }

      if (!m_disable_labels) {
        if (m_label_col < 0) {
          // Last column becomes the label column.
          m_label_col = m_num_cols - 1;
        }
        if (m_label_col >= m_num_cols) {
          throw lbann_exception("csv_reader: label column" +
                                std::to_string(m_label_col) +
                                " is not present");
        }
      }

      if (!m_disable_responses) {
        if (m_response_col < 0) {
          // Last column becomes the response column.
          m_response_col = m_num_cols - 1;
        }
        if (m_response_col >= m_num_cols) {
          throw lbann_exception("csv_reader: response column" +
                                std::to_string(m_response_col) +
                                " is not present");
        }
      }
    }
    else {
      throw lbann_exception("csv_reader: failed to read header in " +
                            get_data_filename());
    }
    if (ifs.eof()) {