Index a sparse vector for a document. For each (token_id, weight) pair, appends a posting to the token's posting list. Also stores the document's L2 norm.
(
&self,
collection: &str,
doc_id: &str,
vector: &SparseVector,
)
| 75 | /// For each (token_id, weight) pair, appends a posting to the |
| 76 | /// token's posting list. Also stores the document's L2 norm. |
| 77 | pub fn index_document( |
| 78 | &self, |
| 79 | collection: &str, |
| 80 | doc_id: &str, |
| 81 | vector: &SparseVector, |
| 82 | ) -> crate::Result<()> { |
| 83 | if vector.is_empty() { |
| 84 | return Ok(()); |
| 85 | } |
| 86 | |
| 87 | let write_txn = self.db.begin_write().map_err(|e| crate::Error::Storage { |
| 88 | engine: "sparse_vector".into(), |
| 89 | detail: format!("write txn: {e}"), |
| 90 | })?; |
| 91 | { |
| 92 | let mut postings_table = |
| 93 | write_txn |
| 94 | .open_table(SPARSE_POSTINGS) |
| 95 | .map_err(|e| crate::Error::Storage { |
| 96 | engine: "sparse_vector".into(), |
| 97 | detail: format!("open postings: {e}"), |
| 98 | })?; |
| 99 | let mut norms_table = |
| 100 | write_txn |
| 101 | .open_table(SPARSE_NORMS) |
| 102 | .map_err(|e| crate::Error::Storage { |
| 103 | engine: "sparse_vector".into(), |
| 104 | detail: format!("open norms: {e}"), |
| 105 | })?; |
| 106 | |
| 107 | // Append to each token's posting list. |
| 108 | for &(token_id, weight) in vector { |
| 109 | if weight.abs() < f32::EPSILON { |
| 110 | continue; |
| 111 | } |
| 112 | let key = format!("{collection}:{token_id}"); |
| 113 | let mut postings: Vec<SparsePosting> = postings_table |
| 114 | .get(key.as_str()) |
| 115 | .ok() |
| 116 | .flatten() |
| 117 | .and_then(|g| zerompk::from_msgpack(g.value()).ok()) |
| 118 | .unwrap_or_default(); |
| 119 | |
| 120 | // Remove existing posting for this doc (update case). |
| 121 | postings.retain(|p| p.doc_id != doc_id); |
| 122 | postings.push(SparsePosting { |
| 123 | doc_id: doc_id.to_string(), |
| 124 | weight, |
| 125 | }); |
| 126 | |
| 127 | let bytes = |
| 128 | zerompk::to_msgpack_vec(&postings).map_err(|e| crate::Error::Storage { |
| 129 | engine: "sparse_vector".into(), |
| 130 | detail: format!("serialize postings: {e}"), |
| 131 | })?; |
| 132 | postings_table |
| 133 | .insert(key.as_str(), bytes.as_slice()) |
| 134 | .map_err(|e| crate::Error::Storage { |