Method LoadIntoMemoryByLine

paddle/fluid/framework/data_feed.cc:2316–2413

}

void SlotRecordInMemoryDataFeed::LoadIntoMemoryByLine() {
#ifdef _LINUX
  paddle::framework::CustomParser* parser =
      global_dlmanager_pool().Load(so_parser_name_, all_slots_info_);
  std::string filename;
  BufferedLineFileReader line_reader;
  line_reader.set_sample_rate(sample_rate_);
  BufferedLineFileReader::LineFunc line_func = nullptr;

  while (this->PickOneFile(&filename)) {
    VLOG(3) << "PickOneFile, filename=" << filename
            << ", thread_id=" << thread_id_;
    std::vector<SlotRecord> record_vec;
    platform::Timer timeline;
    timeline.Start();
    int offset = 0;
    int old_offset = 0;

    SlotRecordPool().get(&record_vec, OBJPOOL_BLOCK_SIZE);
    // get slotrecord object function
    auto record_func = [this, &offset, &record_vec, &old_offset](
                           std::vector<SlotRecord>& vec, int num) {
      vec.resize(num);
      if (offset + num > OBJPOOL_BLOCK_SIZE) {
        input_channel_->WriteMove(offset, &record_vec[0]);
        SlotRecordPool().get(&record_vec[0], offset);
        record_vec.resize(OBJPOOL_BLOCK_SIZE);
        offset = 0;
        old_offset = 0;
      }
      for (int i = 0; i < num; ++i) {
        auto& ins = record_vec[offset + i];
        ins->reset();
        vec[i] = ins;
      }
      offset = offset + num;
    };

    line_func = [this,
                 &parser,
                 &record_vec,
                 &offset,
                 &filename,
                 &record_func,
                 &old_offset](const std::string& line) {
      old_offset = offset;
      if (!parser->ParseOneInstance(line, record_func)) {
        offset = old_offset;
        LOG(WARNING) << "read file:[" << filename << "] item error, line:["
                     << line << "]";
        return false;
      }
      if (offset >= OBJPOOL_BLOCK_SIZE) {
        input_channel_->Write(std::move(record_vec));
        record_vec.clear();
        SlotRecordPool().get(&record_vec, OBJPOOL_BLOCK_SIZE);
        offset = 0;
      }