MCPcopy Create free account
hub / github.com/apache/arrow / CreateBlock

Method CreateBlock

cpp/src/arrow/filesystem/azurefs.cc:1292–1324  ·  view source on GitHub ↗

Source from the content-addressed store, hash-verified

1290 }
1291
1292 std::string CreateBlock() {
1293 std::unique_lock<std::mutex> lock(upload_state_->mutex);
1294 const auto n_block_ids = upload_state_->block_ids.size();
1295
1296 // New block ID must always be distinct from the existing block IDs. Otherwise we
1297 // will accidentally replace the content of existing blocks, causing corruption.
1298 // We will use monotonically increasing integers.
1299 auto new_block_id = std::to_string(n_block_ids);
1300
1301 // Pad to 5 digits, because Azure allows a maximum of 50,000 blocks.
1302 const size_t target_number_of_digits = 5;
1303 const auto required_padding_digits =
1304 target_number_of_digits - std::min(target_number_of_digits, new_block_id.size());
1305 new_block_id.insert(0, required_padding_digits, '0');
1306 // There is a small risk when appending to a blob created by another client that
1307 // `new_block_id` may overlapping with an existing block id. Adding the `-arrow`
1308 // suffix significantly reduces the risk, but does not 100% eliminate it. For
1309 // example if the blob was previously created with one block, with id `00001-arrow`
1310 // then the next block we append will conflict with that, and cause corruption.
1311 new_block_id += "-arrow";
1312 new_block_id = Core::Convert::Base64Encode(
1313 std::vector<uint8_t>(new_block_id.begin(), new_block_id.end()));
1314
1315 upload_state_->block_ids.push_back(new_block_id);
1316
1317 // We only use the future if we have background writes enabled. Without background
1318 // writes the future is initialized as finished and not mutated any more.
1319 if (background_writes_ && upload_state_->blocks_in_progress++ == 0) {
1320 upload_state_->pending_blocks_completed = Future<>::Make();
1321 }
1322
1323 return new_block_id;
1324 }
1325
1326 Status AppendBlock(const void* data, int64_t nbytes,
1327 std::shared_ptr<Buffer> owned_buffer = nullptr) {

Callers

nothing calls this directly

Calls 7

to_stringFunction · 0.85
push_backMethod · 0.80
MakeFunction · 0.50
sizeMethod · 0.45
insertMethod · 0.45
beginMethod · 0.45
endMethod · 0.45

Tested by

no test coverage detected