Method readOneFile

src/data/slot_reader.cc:60–155

}

bool SlotReader::readOneFile(const DataConfig& data, int ith_file) {
  if (FLAGS_verbose) {
    Lock l(mu_);
    LI << "loading data file [" << data.file(0) << "]; loaded [" <<
      loaded_file_count_ << "/" << data_.file_size() << "]";
  }
  // check if hit cache
  string info_name = cache_ + getFilename(data.file(0)) + ".info";
  ExampleInfo info;
  if (readFileToProto(info_name, &info)) {
    // the data is already in cache_dir
    Lock l(mu_);
    info_ = mergeExampleInfo(info_, info);
    num_ex_[ith_file] = info.num_ex();
    return true;
  }

  InfoParser info_parser;
  struct VSlot {
    SArray<float> val;
    SArray<uint64> col_idx;
    SArray<uint16> row_siz;
    bool writeToFile(const string& name) {
      return val.compressTo().writeToFile(name+".value")
          && col_idx.compressTo().writeToFile(name+".colidx")
          && row_siz.compressTo().writeToFile(name+".rowsiz");
    }
  };
  VSlot vslots[kSlotIDmax];
  uint32 num_ex = 0;
  Example ex;

  // store ex in slots and also extract the info
  auto store = [&]() {
    if (!info_parser.add(ex)) return;
    for (int i = 0; i < ex.slot_size(); ++i) {
      const auto& slot = ex.slot(i);
      CHECK_LT(slot.id(), kSlotIDmax);
      auto& vslot = vslots[slot.id()];
      int key_size = slot.key_size();
      for (int j = 0; j < key_size; ++j) vslot.col_idx.pushBack(slot.key(j));
      int val_size = slot.val_size();
      for (int j = 0; j < val_size; ++j) vslot.val.pushBack(slot.val(j));
      while (vslot.row_siz.size() < num_ex) vslot.row_siz.pushBack(0);
      vslot.row_siz.pushBack(std::max(key_size, val_size));
    }
    ++ num_ex;
  };

  // read examples one by one
  if (data_.format() == DataConfig::TEXT) {
    ExampleParser text_parser;
    text_parser.init(data.text(), data.ignore_feature_group());
    std::function<void(char*)> handle = [&] (char *line) {
      if (!text_parser.toProto(line, &ex)) return;
      store();
    };
    FileLineReader reader(data);