MCPcopy Create free account
hub / github.com/Snapchat/KeyDB / bulkInsert

Method bulkInsert

src/storage/rocksdb.cpp:63–116  ·  view source on GitHub ↗

Source from the content-addressed store, hash-verified

61}
62
63void RocksDBStorageProvider::bulkInsert(char **rgkeys, size_t *rgcbkeys, char **rgvals, size_t *rgcbvals, size_t celem)
64{
65 if (celem >= 16384) {
66 rocksdb::Options options = DefaultRocksDBOptions();
67 rocksdb::SstFileWriter sst_file_writer(rocksdb::EnvOptions(), options, options.comparator);
68 std::string file_path = m_pfactory->getTempFolder() + "/tmpIngest.";
69 file_path += std::to_string(gettid());
70 file_path += ".sst";
71
72 rocksdb::Status s = sst_file_writer.Open(file_path);
73 if (!s.ok())
74 goto LFallback;
75
76 // Insert rows into the SST file, note that inserted keys must be
77 // strictly increasing (based on options.comparator)
78 for (size_t ielem = 0; ielem < celem; ++ielem) {
79 std::string prefixed_key = prefixKey(rgkeys[ielem], rgcbkeys[ielem]);
80 s = sst_file_writer.Put(rocksdb::Slice(prefixed_key), rocksdb::Slice(rgvals[ielem], rgcbvals[ielem]));
81 if (!s.ok()) {
82 unlink(file_path.c_str());
83 goto LFallback;
84 }
85 }
86
87 s = sst_file_writer.Finish();
88 if (!s.ok()) {
89 unlink(file_path.c_str());
90 goto LFallback;
91 }
92
93 auto ingestOptions = rocksdb::IngestExternalFileOptions();
94 ingestOptions.move_files = true;
95 ingestOptions.write_global_seqno = false;
96 ingestOptions.failed_move_fall_back_to_copy = false;
97
98 // Ingest the external SST file into the DB
99 s = m_spdb->IngestExternalFile(m_spcolfamily.get(), {file_path}, ingestOptions);
100 if (!s.ok()) {
101 unlink(file_path.c_str());
102 goto LFallback;
103 }
104 } else {
105 LFallback:
106 auto spbatch = std::make_unique<rocksdb::WriteBatch>();
107 for (size_t ielem = 0; ielem < celem; ++ielem) {
108 std::string prefixed_key = prefixKey(rgkeys[ielem], rgcbkeys[ielem]);
109 spbatch->Put(m_spcolfamily.get(), rocksdb::Slice(prefixed_key), rocksdb::Slice(rgvals[ielem], rgcbvals[ielem]));
110 }
111 m_spdb->Write(WriteOptions(), spbatch.get());
112 }
113
114 std::unique_lock<fastlock> l(m_lock);
115 m_count += celem;
116}
117
118bool RocksDBStorageProvider::erase(const char *key, size_t cchKey)
119{

Callers

nothing calls this directly

Calls 6

DefaultRocksDBOptionsFunction · 0.85
gettidFunction · 0.85
prefixKeyFunction · 0.85
getTempFolderMethod · 0.80
okMethod · 0.80
getMethod · 0.45

Tested by

no test coverage detected