| 24 | } |
| 25 | |
| 26 | CacheDataLocalFile::CacheDataLocalFile(std::unique_ptr<ral::frame::BlazingTable> table, std::string orc_files_path, std::string ctx_token) |
| 27 | : CacheData(CacheDataType::LOCAL_FILE, table->names(), table->get_schema(), table->num_rows()) |
| 28 | { |
| 29 | this->size_in_bytes = table->sizeInBytes(); |
| 30 | this->filePath_ = orc_files_path + "/.blazing-temp-" + ctx_token + "-" + randomString(64) + ".orc"; |
| 31 | |
| 32 | // filling this->col_names |
| 33 | for(auto name : table->names()) { |
| 34 | this->col_names.push_back(name); |
| 35 | } |
| 36 | |
| 37 | int attempts = 0; |
| 38 | int attempts_limit = 10; |
| 39 | while(attempts <= attempts_limit){ |
| 40 | try { |
| 41 | cudf::io::table_metadata metadata; |
| 42 | for(auto name : table->names()) { |
| 43 | metadata.column_names.emplace_back(name); |
| 44 | } |
| 45 | |
| 46 | cudf::io::orc_writer_options out_opts = cudf::io::orc_writer_options::builder(cudf::io::sink_info{this->filePath_}, table->view()) |
| 47 | .metadata(&metadata); |
| 48 | |
| 49 | cudf::io::write_orc(out_opts); |
| 50 | } catch (cudf::logic_error & err){ |
| 51 | std::shared_ptr<spdlog::logger> logger = spdlog::get("batch_logger"); |
| 52 | if(logger) { |
| 53 | logger->error("|||{info}||||rows|{rows}", |
| 54 | "info"_a="Failed to create CacheDataLocalFile in path: " + this->filePath_ + " attempt " + std::to_string(attempts), |
| 55 | "rows"_a=table->num_rows()); |
| 56 | } |
| 57 | attempts++; |
| 58 | if (attempts == attempts_limit){ |
| 59 | throw; |
| 60 | } |
| 61 | std::this_thread::sleep_for (std::chrono::milliseconds(5 * attempts)); |
| 62 | } |
| 63 | } |
| 64 | } |
| 65 | |
| 66 | size_t CacheDataLocalFile::fileSizeInBytes() const { |
| 67 | struct stat st; |
nothing calls this directly
no test coverage detected