Class MinHashLSHEnsemble

join/LSH/datasketch/lshensemble.py:47–254

`E. Zhu et al. `_. Args: threshold (float): Containment阈值 取值范围[0.0, 1.0]。 num_perm (int, optional): Minhash中所使用的排列函数的个数。 num_part (int, optional): LSH Ensemble分区的个数。 m (int, optional): 特殊超参数: LSH Ensemble使用大约比相同

class MinHashLSHEnsemble(object):
    '''
    `E. Zhu et al. <http://www.vldb.org/pvldb/vol9/p1185-zhu.pdf>`_.

    Args:
        threshold (float): Containment阈值 取值范围[0.0, 1.0]。
        num_perm (int, optional): Minhash中所使用的排列函数的个数。 
        num_part (int, optional): LSH Ensemble分区的个数。
        m (int, optional): 特殊超参数: LSH Ensemble使用大约比相同数量的MinHash LSH多出m倍的内存空间。
            另外，m越大，精确度越高。
        weights (tuple, optional): 在优化参数设定时，对fp和fn重要性的权衡考量。
        storage_config (dict, optional): 存储相关参数，不重要
        prepickle (bool, optional): 默认值由`storage_config`确定，不重要.

    Note:
        更多的分区(`num_part`)可以取得更好的准确性。
    ''&#x27;

    def __init__(self, threshold=0.9, num_perm=128, num_part=16, m=8,
            weights=(0.5,0.5), storage_config=None, prepickle=None):
        if threshold > 1.0 or threshold < 0.0:
            raise ValueError("threshold must be in [0.0, 1.0]")
        if num_perm < 2:
            raise ValueError("Too few permutation functions")
        if num_part < 1:
            raise ValueError("num_part must be at least 1")
        if m < 2 or m > num_perm:
            raise ValueError("m must be in the range of [2, num_perm]")
        if any(w < 0.0 or w > 1.0 for w in weights):
            raise ValueError("Weight must be in [0.0, 1.0]")
        if sum(weights) != 1.0:
            raise ValueError("Weights must sum to 1.0")
        self.threshold = threshold
        self.h = num_perm
        self.m = m
        rs = self._init_optimal_params(weights)
        # 对于r的每个可能取值，对LSHEnsemble中每个分区初始化一个LSH，索引初始化完成
        storage_config = {'type': 'dict'} if not storage_config else storage_config
        basename = storage_config.get('basename', _random_name(11))
        self.indexes = [
            dict((r, MinHashLSH(
                num_perm=self.h,
                params=(int(self.h/r), r),
                # 不同的分区存储方式可能不同
                storage_config=self._get_storage_config(
                    basename, storage_config, partition, r),
                prepickle=prepickle)) for r in rs)
            for partition in range(0, num_part)]
        self.lowers = [None for _ in self.indexes]
        self.uppers = [None for _ in self.indexes]

    # 给出一系列可能存在的xq，预处理得到使fp和fn带权和最优的参数b与r
    def _init_optimal_params(self, weights):
        false_positive_weight, false_negative_weight = weights
        self.xqs = np.exp(np.linspace(-5, 5, 10))
        self.params = np.array([_optimal_param(self.threshold, self.h, self.m,
                                               xq,
                                               false_positive_weight,