Attempts to avoid spending too much effort on dictionary entries that were rarely seen, since these are likely just noise.
(
entries: &HashMap<u64, DictionaryItem>,
)
| 81 | /// Attempts to avoid spending too much effort on dictionary entries that were rarely seen, since |
| 82 | /// these are likely just noise. |
| 83 | fn weight_dictionary_by_frequent_items( |
| 84 | entries: &HashMap<u64, DictionaryItem>, |
| 85 | ) -> Option<rand_distr::WeightedAliasIndex<f32>> { |
| 86 | let mut counts: Vec<_> = entries.iter().map(|(_, entry)| entry.count).collect(); |
| 87 | counts.sort_unstable(); |
| 88 | let threshold = if counts.len() > 10 { |
| 89 | let median = counts[counts.len() / 2]; |
| 90 | median / 2 |
| 91 | } |
| 92 | else { |
| 93 | 0 |
| 94 | }; |
| 95 | rand_distr::WeightedAliasIndex::new( |
| 96 | counts.into_iter().map(|x| if x >= threshold { 10.0 } else { 1.0 }).collect(), |
| 97 | ) |
| 98 | .ok() |
| 99 | } |
| 100 | |
| 101 | pub type MultiStreamDict = HashMap<StreamKey, Dictionary>; |
| 102 |