Method load

src/data_readers/data_reader_smiles.cpp:98–151

}

void smiles_data_reader::load()
{
  if (get_comm()->am_world_master()) {
    std::cout << "starting load for role: " << get_role() << std::endl;
  }

  double tm1 = get_time();
  auto& arg_parser = global_argument_parser();

  // for now, only implemented for data store with preloading
  set_use_data_store(true);

  if (m_sequence_length == 0) {
    if (arg_parser.get<int>(LBANN_OPTION_SEQUENCE_LENGTH) == -1) {
      LBANN_ERROR("you must pass --sequence_length=<int> on the cmd line or "
                  "call set_sequence_length()");
    }
    m_sequence_length = arg_parser.get<int>(LBANN_OPTION_SEQUENCE_LENGTH);
  }
  m_linearized_data_size = m_sequence_length + 2;

  // load the vocabulary; this is a map: string -> short
  if (m_vocab.size() == 0) {
    if (arg_parser.get<std::string>(LBANN_OPTION_VOCAB) == "") {
      LBANN_ERROR("you must either pass --vocab=<string> on the command line "
                  "or call load_vocab(...)");
    }
    const std::string fn = arg_parser.get<std::string>(LBANN_OPTION_VOCAB);
    load_vocab(fn);
  }
  else {
    LBANN_ERROR("you passed --vocab=<string>, but it looks like load_vocab() "
                "was previously called. You must use one or the other.");
  }

  // Load the sample list(s)
  data_reader_sample_list::load();
  if (get_comm()->am_world_master()) {
    std::cout << "time to load sample list: " << get_time() - tm1 << std::endl;
  }

  // do what we almost always do (TODO: this should be refactored as a method
  m_shuffled_indices.clear();
  m_shuffled_indices.resize(m_sample_list.size());
  std::iota(m_shuffled_indices.begin(), m_shuffled_indices.end(), 0);
  resize_shuffled_indices();
  instantiate_data_store();
  select_subset_of_data();

  // load various metadata
  build_some_maps();
  load_offsets_and_lengths();
  print_statistics();
}

void smiles_data_reader::do_preload_data_store()
{