Method sampleFiles

cpp/dbtools/ShardDBTools.cpp:519–694

static constexpr uint64_t  SAMPLE_RATE_SIZE_LIMIT = 10ull << 30;

void ShardDBTools::sampleFiles(const std::string& dbPath, const std::string& outputFilePath) {
    Logger logger(LogLevel::LOG_INFO, STDERR_FILENO, false, false);
    std::ofstream outputStream(outputFilePath);
    ALWAYS_ASSERT(outputStream.is_open(), "Failed to open output file for sampling results");
    std::shared_ptr<XmonAgent> xmon;
    Env env(logger, xmon, "ShardDBTools");
    SharedRocksDB sharedDb(logger, xmon, dbPath, "");
    sharedDb.registerCFDescriptors(ShardDB::getColumnFamilyDescriptors());
    sharedDb.registerCFDescriptors(LogsDB::getColumnFamilyDescriptors());
    rocksdb::Options rocksDBOptions;
    rocksDBOptions.compression = rocksdb::kLZ4Compression;
    rocksDBOptions.bottommost_compression = rocksdb::kZSTD;
    sharedDb.openForReadOnly(rocksDBOptions);
    auto db = sharedDb.db();
    rocksdb::ReadOptions options;
    auto edgesCf = sharedDb.getCF("edges");
    auto filesCf = sharedDb.getCF("files");
    auto spansCf = sharedDb.getCF("spans");
    std::unordered_map<InodeId, FileInfo> files;
    std::random_device rd;
    std::mt19937 gen(rd());
    std::uniform_real_distribution<double> realDis(0.0, 1.0);
    {
        std::unique_ptr<rocksdb::Iterator> it(db->NewIterator(options, filesCf));
        for (it->SeekToFirst(); it->Valid(); it->Next()) {
            auto fileK = ExternalValue<InodeIdKey>::FromSlice(it->key());
            InodeId fileId = fileK().id();
            if (fileId.type() != InodeType::FILE) {
                continue;
            }
            auto fileV = ExternalValue<FileBody>::FromSlice(it->value());
            auto logicalSize = fileV().fileSize();
            if ( logicalSize == 0) { continue; } // nothing in this
            auto probability = (double)logicalSize / SAMPLE_RATE_SIZE_LIMIT;
            if( probability > realDis(gen)) {
                files.insert(std::make_pair(fileId, FileInfo{fileV().mtime(), fileV().atime(), SizePerStorageClass{logicalSize,{StorageClassSize{0,0},StorageClassSize{0,0}},0}, std::max(logicalSize, SAMPLE_RATE_SIZE_LIMIT)}));
            }
        }
        ROCKS_DB_CHECKED(it->status());
    }
    {
        std::unique_ptr<rocksdb::Iterator> it(db->NewIterator(options, spansCf));
        InodeId thisFile = NULL_INODE_ID;
        LocationSize thisFileSize{StorageClassSize{0,0},StorageClassSize{0,0}, StorageClassSize{0,0}};
        uint64_t thisFileInlineSize{0};
        for (it->SeekToFirst(); it->Valid(); it->Next()) {
            auto spanK = ExternalValue<SpanKey>::FromSlice(it->key());
            if (spanK().fileId().type() != InodeType::FILE) {
                continue;
            }
            if (spanK().fileId() != thisFile) {
                auto file_it = files.find(thisFile);
                if (file_it != files.end()) {
                    file_it->second.size.size = thisFileSize;
                    file_it->second.size.inMetadata = thisFileInlineSize;
                }
                thisFile = spanK().fileId();
                thisFileSize = LocationSize{StorageClassSize{0,0},StorageClassSize{0,0}, StorageClassSize{0,0}};