| 231 | } |
| 232 | |
| 233 | public Neighbor[] distances(int[] unknown, int k, double distanceThreshold) { |
| 234 | int curTokenRuleIndex = unknown[Trainer.INDEX_PREV_EARLIEST_RIGHT_ANCESTOR]; |
| 235 | int prevTokenRuleIndex = unknown[Trainer.INDEX_EARLIEST_LEFT_ANCESTOR]; |
| 236 | int pr = Trainer.unrulealt(prevTokenRuleIndex)[0]; |
| 237 | int cr = Trainer.unrulealt(curTokenRuleIndex)[0]; |
| 238 | |
| 239 | List<Integer> vectorIndexesMatchingContext = null; |
| 240 | |
| 241 | // look for exact match and take result even if < k results. If we have exact matches they always win let's say |
| 242 | if ( FEATURES==FEATURES_INJECT_WS ) { |
| 243 | vectorIndexesMatchingContext = corpus.wsFeaturesToExemplarIndexes.get(new FeatureVectorAsObject(unknown, FEATURES)); |
| 244 | } |
| 245 | else if ( FEATURES==FEATURES_HPOS ) { |
| 246 | vectorIndexesMatchingContext = corpus.hposFeaturesToExemplarIndexes.get(new FeatureVectorAsObject(unknown, FEATURES)); |
| 247 | } |
| 248 | // else might be specialized feature set for testing so ignore these caches in that case |
| 249 | |
| 250 | if ( FEATURES==FEATURES_INJECT_WS && // can't use this cache if we are testing out different feature sets |
| 251 | (vectorIndexesMatchingContext==null || vectorIndexesMatchingContext.size()<=3) ) // must have at 4 or more dist=0.0 for WS else we search wider |
| 252 | { |
| 253 | // ok, not exact. look for match with prev and current rule index |
| 254 | Pair<Integer, Integer> key = new Pair<>(pr, cr); |
| 255 | vectorIndexesMatchingContext = corpus.curAndPrevTokenRuleIndexToExemplarIndexes.get(key); |
| 256 | } |
| 257 | if ( FEATURES==FEATURES_HPOS && |
| 258 | (vectorIndexesMatchingContext==null || vectorIndexesMatchingContext.size()<k) ) |
| 259 | { |
| 260 | // ok, not exact. look for match with prev and current rule index |
| 261 | Pair<Integer, Integer> key = new Pair<>(pr, cr); |
| 262 | vectorIndexesMatchingContext = corpus.curAndPrevTokenRuleIndexToExemplarIndexes.get(key); |
| 263 | } |
| 264 | |
| 265 | if ( distanceThreshold==MAX_CONTEXT_DIFF_THRESHOLD2 ) { // couldn't find anything, open it all up. |
| 266 | vectorIndexesMatchingContext = null; |
| 267 | } |
| 268 | List<Neighbor> distances = new ArrayList<>(); |
| 269 | if ( vectorIndexesMatchingContext==null ) { |
| 270 | // no matching contexts for this feature, must rely on full training set |
| 271 | int n = corpus.featureVectors.size(); // num training samples |
| 272 | int num0 = 0; // how many 0-distance elements have we seen? If k we can stop! |
| 273 | for (int i = 0; i<n; i++) { |
| 274 | int[] x = corpus.featureVectors.get(i); |
| 275 | double d = distance(x, unknown); |
| 276 | if ( d<=distanceThreshold ) { |
| 277 | Neighbor neighbor = new Neighbor(corpus, d, i); |
| 278 | distances.add(neighbor); |
| 279 | if ( d==0.0 ) { |
| 280 | num0++; |
| 281 | if ( num0==k ) break; |
| 282 | } |
| 283 | } |
| 284 | } |
| 285 | } |
| 286 | else { |
| 287 | int num0 = 0; // how many 0-distance elements have we seen? If k we can stop! |
| 288 | for (Integer vectorIndex : vectorIndexesMatchingContext) { |
| 289 | int[] x = corpus.featureVectors.get(vectorIndex); |
| 290 | double d = distance(x, unknown); |