Method Load

tools/eval/eval_dataset.cpp:105–327

}

EvalDatasetPtr
EvalDataset::Load(const std::string& filename) {
    H5::H5File file(filename, H5F_ACC_RDONLY);

    // check datasets exist
    bool has_labels = false;
    bool has_valid_ratio = false;
    {
        auto datasets = get_datasets(file);
        assert(datasets.count("train"));
        assert(datasets.count("test"));
        assert(datasets.count("neighbors"));
        assert(datasets.count("distances"));
        has_labels = datasets.count("train_labels") && datasets.count("test_labels");
        has_valid_ratio = datasets.count("valid_ratios") > 0;
    }

    // get and (should check shape)
    auto train_shape = get_shape(file, "train");
    spdlog::debug("train.shape: " + to_string(train_shape));
    auto test_shape = get_shape(file, "test");
    spdlog::debug("test.shape: " + to_string(test_shape));
    auto neighbors_shape = get_shape(file, "neighbors");
    spdlog::debug("neighbors.shape: " + to_string(neighbors_shape));
    assert(train_shape.second == test_shape.second);

    auto obj = std::make_shared<EvalDataset>();
    obj->file_path_ = filename;
    obj->train_shape_ = train_shape;
    obj->test_shape_ = test_shape;
    obj->neighbors_shape_ = neighbors_shape;
    obj->dim_ = train_shape.second;
    obj->number_of_base_ = train_shape.first;
    obj->number_of_query_ = test_shape.first;

    // compatible with the hdf5 files from internet
    if (not file.attrExists("type")) {
        obj->vector_type_ = DENSE_VECTORS;
    } else {
        try {
            H5::Attribute attr = file.openAttribute("type");
            H5::StrType str_type = attr.getStrType();
            std::string type;
            attr.read(str_type, type);
            if (type == "dense") {
                obj->vector_type_ = DENSE_VECTORS;
            } else if (type == "sparse") {
                obj->vector_type_ = SPARSE_VECTORS;
            }
        } catch (H5::Exception& err) {
            throw std::runtime_error("fail to read metric: there is no 'type' in the dataset");
        }
    }

    if (obj->vector_type_ == DENSE_VECTORS) {
        // read from file
        {
            H5::DataSet dataset = file.openDataSet("/train");