| 91 | } |
| 92 | |
| 93 | std::vector<std::string> ReadKLineFromFile(const char* filename, bool header, int k) { |
| 94 | auto reader = VirtualFileReader::Make(filename); |
| 95 | if (!reader->Init()) { |
| 96 | Log::Fatal("Data file %s doesn't exist.", filename); |
| 97 | } |
| 98 | std::vector<std::string> ret; |
| 99 | std::string cur_line; |
| 100 | const size_t buffer_size = 1024 * 1024; |
| 101 | auto buffer = std::vector<char>(buffer_size); |
| 102 | size_t read_len = reader->Read(buffer.data(), buffer_size); |
| 103 | if (read_len <= 0) { |
| 104 | Log::Fatal("Data file %s couldn't be read.", filename); |
| 105 | } |
| 106 | std::string read_str = std::string(buffer.data(), read_len); |
| 107 | std::stringstream tmp_file(read_str); |
| 108 | if (header) { |
| 109 | if (!tmp_file.eof()) { |
| 110 | GetLine(&tmp_file, &cur_line, reader.get(), &buffer, buffer_size); |
| 111 | } |
| 112 | } |
| 113 | for (int i = 0; i < k; ++i) { |
| 114 | if (!tmp_file.eof()) { |
| 115 | GetLine(&tmp_file, &cur_line, reader.get(), &buffer, buffer_size); |
| 116 | cur_line = Common::Trim(cur_line); |
| 117 | if (!cur_line.empty()) { |
| 118 | ret.push_back(cur_line); |
| 119 | } |
| 120 | } else { |
| 121 | break; |
| 122 | } |
| 123 | } |
| 124 | if (ret.empty()) { |
| 125 | Log::Fatal("Data file %s should have at least one line.", filename); |
| 126 | } else if (ret.size() == 1) { |
| 127 | Log::Warning("Data file %s only has one line.", filename); |
| 128 | } |
| 129 | return ret; |
| 130 | } |
| 131 | |
| 132 | DataType GetDataType(const std::vector<std::string>& lines, int* num_col) { |
| 133 | DataType type = DataType::INVALID; |