FIXME: Could maybe reduce duplicated code between this and chunkedSnapshot for chunking
| 959 | |
| 960 | // FIXME: Could maybe reduce duplicated code between this and chunkedSnapshot for chunking |
| 961 | Value serializeChunkedDeltaFile(const Standalone<StringRef>& fileNameRef, |
| 962 | const Standalone<GranuleDeltas>& deltas, |
| 963 | const KeyRangeRef& fileRange, |
| 964 | int chunkSize, |
| 965 | Optional<CompressionFilter> compressFilter, |
| 966 | Optional<BlobGranuleCipherKeysCtx> cipherKeysCtx) { |
| 967 | if (BG_ENCRYPT_COMPRESS_DEBUG) { |
| 968 | TraceEvent(SevDebug, "SerializeChunkedDelta") |
| 969 | .detail("Filename", fileNameRef.toString()) |
| 970 | .detail("RangeBegin", fileRange.begin.printable()) |
| 971 | .detail("RangeEnd", fileRange.end.printable()) |
| 972 | .detail("Encrypted", cipherKeysCtx.present()) |
| 973 | .detail("Compressed", compressFilter.present()); |
| 974 | } |
| 975 | |
| 976 | CODE_PROBE(compressFilter.present(), "serializing compressed delta file"); |
| 977 | CODE_PROBE(cipherKeysCtx.present(), "serializing encrypted delta file"); |
| 978 | Standalone<IndexedBlobGranuleFile> file; |
| 979 | |
| 980 | file.init(DELTA_FILE_TYPE, cipherKeysCtx); |
| 981 | |
| 982 | // build in-memory version of boundaries - TODO separate functions |
| 983 | SortedDeltasT boundaries; |
| 984 | sortDeltasByKey(deltas, fileRange, boundaries); |
| 985 | |
| 986 | std::vector<Value> chunks; |
| 987 | chunks.push_back(Value()); // dummy value for index block |
| 988 | |
| 989 | Standalone<GranuleSortedDeltas> currentChunk; |
| 990 | size_t currentChunkBytesEstimate = 0; |
| 991 | size_t previousChunkBytes = 0; |
| 992 | |
| 993 | // TODO REMOVE - for validation |
| 994 | KeyRef lastKey; |
| 995 | int i = 0; |
| 996 | for (auto& it : boundaries) { |
| 997 | // TODO REMOVE sanity check |
| 998 | if (i > 0) { |
| 999 | ASSERT(lastKey < it.first); |
| 1000 | } |
| 1001 | lastKey = it.first; |
| 1002 | it.second.key = it.first; |
| 1003 | |
| 1004 | currentChunk.boundaries.push_back_deep(currentChunk.arena(), it.second); |
| 1005 | currentChunkBytesEstimate += it.second.totalSize(); |
| 1006 | |
| 1007 | if (currentChunkBytesEstimate >= chunkSize || i == boundaries.size() - 1) { |
| 1008 | Value serialized = |
| 1009 | BinaryWriter::toValue(currentChunk, IncludeVersion(ProtocolVersion::withBlobGranuleFile())); |
| 1010 | Value chunkBytes = |
| 1011 | IndexBlobGranuleFileChunkRef::toBytes(cipherKeysCtx, compressFilter, serialized, file.arena()); |
| 1012 | chunks.push_back(chunkBytes); |
| 1013 | |
| 1014 | // TODO remove validation |
| 1015 | if (!file.indexBlockRef.block.children.empty()) { |
| 1016 | ASSERT(file.indexBlockRef.block.children.back().key < currentChunk.boundaries.begin()->key); |
| 1017 | } |
| 1018 | file.indexBlockRef.block.children.emplace_back_deep( |
no test coverage detected