Method do_preload_data_store

src/data_readers/data_reader_image.cpp:263–320

}

void image_data_reader::do_preload_data_store()
{
  auto& arg_parser = global_argument_parser();

  int rank = m_comm->get_rank_in_trainer();

  bool threaded = !arg_parser.get<bool>(LBANN_OPTION_DATA_STORE_NO_THREAD);
  if (threaded) {
    if (get_comm()->am_world_master()) {
      std::cout << "mode: data_store_thread\n";
    }
    std::shared_ptr<thread_pool> io_thread_pool =
      construct_io_thread_pool(m_comm, false);
    int num_threads = static_cast<int>(io_thread_pool->get_num_threads());

    std::vector<std::unordered_set<int>> data_ids(num_threads);
    int j = 0;
    for (size_t data_id = 0; data_id < m_shuffled_indices.size(); data_id++) {
      int index = m_shuffled_indices[data_id];
      if (m_data_store->get_index_owner(index) != rank) {
        continue;
      }
      data_ids[j++].insert(index);
      if (j == num_threads) {
        j = 0;
      }
    }

    for (int t = 0; t < num_threads; t++) {
      if (t == io_thread_pool->get_local_thread_id()) {
        continue;
      }
      else {
        io_thread_pool->submit_job_to_work_group(
          std::bind(&image_data_reader::load_conduit_nodes_from_file,
                    this,
                    data_ids[t]));
      }
    }
    load_conduit_nodes_from_file(
      data_ids[io_thread_pool->get_local_thread_id()]);
    io_thread_pool->finish_work_group();
  }
  else {
    if (get_comm()->am_world_master()) {
      std::cout << "mode: NOT data_store_thread\n";
    }
    for (size_t data_id = 0; data_id < m_shuffled_indices.size(); data_id++) {
      int index = m_shuffled_indices[data_id];
      if (m_data_store->get_index_owner(index) != rank) {
        continue;
      }
      conduit::Node& node = m_data_store->get_empty_node(index);
      load_conduit_node_from_file(index, node);
      m_data_store->set_preloaded_conduit_node(index, node);
    }
  }
}