Perform table-level top-k nearest neighbour search over the configured LSH backends. Note that this functions assumes that the table name is part of the canonical indexed item ids. In other words, it considers the first part of the item id separated by a dot to be the table
(
self,
table: pd.DataFrame,
aggregator: Optional[callable] = None,
k: Optional[int] = None,
verbose: bool = False,
)
| 167 | return results[:k] |
| 168 | |
| 169 | def table_query( |
| 170 | self, |
| 171 | table: pd.DataFrame, |
| 172 | aggregator: Optional[callable] = None, |
| 173 | k: Optional[int] = None, |
| 174 | verbose: bool = False, |
| 175 | ) -> Union[Iterable[Tuple], Tuple[Iterable[Tuple], Iterable[Tuple]]]: |
| 176 | """ |
| 177 | Perform table-level top-k nearest neighbour search over the configured LSH backends. |
| 178 | Note that this functions assumes that the table name is part of the canonical indexed item ids. |
| 179 | In other words, it considers the first part of the item id separated by a dot to be the table name. |
| 180 | Parameters |
| 181 | ---------- |
| 182 | table : pd.DataFrame |
| 183 | The table query as a Pandas DataFrame. |
| 184 | Each column will be the subject of a column-based query. |
| 185 | aggregator: callable |
| 186 | An aggregating function used to merge the results of all configured backends at table-level. |
| 187 | k : Optional[int] |
| 188 | Only the top-k neighbours will be retrieved from each backend. |
| 189 | Then, these results are aggregated using the aggregator function and the results re-ranked to retrieve |
| 190 | the top-k aggregated neighbours. |
| 191 | If this is None all results are retrieved. |
| 192 | verbose: bool |
| 193 | Whether or not to also return the detailed scores for each similar column to some query column. |
| 194 | |
| 195 | Returns |
| 196 | ------- |
| 197 | Union[Iterable[Tuple], Tuple[Iterable[Tuple], Iterable[Tuple]]] |
| 198 | Pairs of the form (candidate table name, aggregated similarity score). |
| 199 | If verbosity is required, also return pairs with column-level similarity details. |
| 200 | """ |
| 201 | |
| 202 | extended_table_results = None |
| 203 | score_distributions = {} |
| 204 | for column in table.columns: |
| 205 | """Column scores are not aggregated when performing table queries.""" |
| 206 | column_results = self.column_query( |
| 207 | column=table[column], aggregator=None, k=None |
| 208 | ) |
| 209 | |
| 210 | score_distributions[column] = np.sort( |
| 211 | np.array([scores for _, scores in column_results]), axis=0 |
| 212 | ) |
| 213 | extended_table_results = self.group_results_by_table( |
| 214 | target_id=column, |
| 215 | results=column_results, |
| 216 | table_groups=extended_table_results, |
| 217 | ) |
| 218 | |
| 219 | table_results = {} |
| 220 | for candidate in extended_table_results.keys(): |
| 221 | candidate_scores = np.array( |
| 222 | [details[1] for details in extended_table_results[candidate]] |
| 223 | ) |
| 224 | distributions = [ |
| 225 | score_distributions[details[0][0]] |
| 226 | for details in extended_table_results[candidate] |
no test coverage detected