| 338 | } |
| 339 | |
| 340 | void ZenFileSystem::finalize() { |
| 341 | // 1. Write to index.json |
| 342 | if (options_.record) { |
| 343 | nlohmann::json blobs_info = nlohmann::json::array(); |
| 344 | for (const auto& [blob_id, blob] : blob_) { |
| 345 | nlohmann::json blob_info; |
| 346 | blob_info["id"] = blob_id; |
| 347 | blob_info["type"] = static_cast<int>(blob.type); |
| 348 | blob_info["data_shape"] = blob.data.shape(); |
| 349 | blob_info["data_dtype"] = blob.data.dtype(); |
| 350 | |
| 351 | // Record if blob has mmap file |
| 352 | if (blob.mmap_file.has_value()) { |
| 353 | blob_info["has_mmap_file"] = true; |
| 354 | blob_info["mmap_file_path"] = options_.working_dir + "/blob_" + std::to_string(blob_id) + ".kv"; |
| 355 | } else { |
| 356 | blob_info["has_mmap_file"] = false; |
| 357 | } |
| 358 | |
| 359 | blobs_info.push_back(blob_info); |
| 360 | } |
| 361 | index_["blobs"] = blobs_info; |
| 362 | |
| 363 | std::string index_file_path = options_.working_dir + "/index.json"; |
| 364 | std::ofstream index_file(index_file_path); |
| 365 | if (index_file.is_open()) { |
| 366 | index_file << index_ << "\n"; |
| 367 | index_file.close(); |
| 368 | } |
| 369 | } |
| 370 | |
| 371 | // 2. Write first Tensor Blob into disk. |
| 372 | // The first blob (blob id 0) should be written to disk |
| 373 | auto it = blob_.find(0); |
| 374 | if (it != blob_.end() && options_.record) { |
| 375 | std::string blob_file_path = options_.working_dir + "/blob_0.kv"; |
| 376 | // Write the tensor data to disk |
| 377 | auto& tensor = it->second.data; |
| 378 | auto* data_ptr = tensor.ptr<char>(); |
| 379 | auto data_size = tensor.bytes(); |
| 380 | std::ofstream blob_file(blob_file_path, std::ios::binary); |
| 381 | if (blob_file.is_open()) { |
| 382 | blob_file.write(data_ptr, data_size); |
| 383 | blob_file.close(); |
| 384 | } |
| 385 | } |
| 386 | |
| 387 | // 3. Write back mmap file if some page is dirty |
| 388 | for (auto& [blob_id, blob] : blob_) { |
| 389 | if (blob.type == ZenFSBlobType::kOnDiskMMAP && blob.mmap_file.has_value()) { |
| 390 | // Sync the mmap file to ensure all changes are written to disk |
| 391 | blob.mmap_file.value()->sync(); |
| 392 | } |
| 393 | } |
| 394 | } |
| 395 | |
| 396 | void ZenFileSystem::recover(const std::string& working_dir) { |
| 397 | // 1. Check if working directory exists |