MCPcopy Create free account
hub / github.com/Oneflow-Inc/oneflow / LoadSnapshotImpl

Method LoadSnapshotImpl

oneflow/core/embedding/persistent_table.cpp:583–608  ·  view source on GitHub ↗

Source from the content-addressed store, hash-verified

581
582template<typename Key, typename Engine>
583void PersistentTableImpl<Key, Engine>::LoadSnapshotImpl(const std::string& name) {
584 std::lock_guard<std::recursive_mutex> lock(mutex_);
585 const std::string snapshot_base = SnapshotDirPath(name);
586 const std::string snapshot_list = SnapshotListFilePath(name);
587 row_id_mapping_.clear();
588 std::ifstream list_if(snapshot_list);
589 std::string index_filename;
590 while (std::getline(list_if, index_filename)) {
591 const uint64_t chunk_id = GetChunkId(index_filename, kIndexFileNamePrefix);
592 PosixFile index_file(PosixFile::JoinPath(snapshot_base, index_filename), O_RDONLY, 0644);
593 const size_t index_file_size = index_file.Size();
594 CHECK_EQ(index_file_size % sizeof(uint64_t), 0);
595 if (index_file_size == 0) { return; }
596 const size_t n_entries = index_file_size / sizeof(uint64_t);
597 PosixMappedFile mapped_index(std::move(index_file), index_file_size, PROT_READ);
598 PosixFile key_file(KeyFilePath(chunk_id), O_RDONLY, 0644);
599 PosixMappedFile mapped_key(std::move(key_file), key_file.Size(), PROT_READ);
600 const uint64_t* indices = static_cast<const uint64_t*>(mapped_index.ptr());
601 const Key* keys = static_cast<const Key*>(mapped_key.ptr());
602 const uint64_t chunk_start_index = chunk_id * num_values_per_chunk_;
603 row_id_mapping_.reserve(row_id_mapping_.size() + n_entries);
604 for (size_t i = 0; i < n_entries; ++i) {
605 CHECK(row_id_mapping_.emplace(keys[indices[i] - chunk_start_index], indices[i]).second);
606 }
607 }
608}
609
610template<typename Key, typename Engine>
611void PersistentTableImpl<Key, Engine>::SaveSnapshotImpl(const std::string& name) {

Callers

nothing calls this directly

Calls 6

GetChunkIdFunction · 0.85
JoinPathFunction · 0.85
clearMethod · 0.45
SizeMethod · 0.45
ptrMethod · 0.45
sizeMethod · 0.45

Tested by

no test coverage detected