()
| 12230 | batch_token_count = 0 |
| 12231 | |
| 12232 | def decode_embedding_batch() -> None: |
| 12233 | nonlocal batch_token_count |
| 12234 | if not batch_sizes: |
| 12235 | return |
| 12236 | self.clear_memory() |
| 12237 | self.decode() |
| 12238 | self.clear_batch() |
| 12239 | for seq_id in range(len(batch_sizes)): |
| 12240 | ptr = llama_cpp.llama_get_embeddings_seq( |
| 12241 | self.ctx, |
| 12242 | llama_cpp.llama_seq_id(seq_id), |
| 12243 | ) |
| 12244 | if not ptr: |
| 12245 | raise RuntimeError(f"missing embedding output for input {seq_id}") |
| 12246 | embeddings.append( |
| 12247 | np.ctypeslib.as_array(ptr, shape=(self.n_embd_out,)).astype( |
| 12248 | float |
| 12249 | ).tolist() |
| 12250 | ) |
| 12251 | batch_sizes.clear() |
| 12252 | batch_token_count = 0 |
| 12253 | |
| 12254 | try: |
| 12255 | self.clear_batch() |
nothing calls this directly
no test coverage detected