(
self,
inputs: Sequence[Union[str, List[int]]],
)
| 12202 | return np.ctypeslib.as_array(ptr, shape=(self.n_vocab,)).copy() |
| 12203 | |
| 12204 | def embed( |
| 12205 | self, |
| 12206 | inputs: Sequence[Union[str, List[int]]], |
| 12207 | ) -> Tuple[List[List[float]], int]: |
| 12208 | if not self.embedding: |
| 12209 | raise CompletionRequestValidationError( |
| 12210 | "model.embedding must be true to use /v1/embeddings" |
| 12211 | ) |
| 12212 | pooling_type = int(llama_cpp.llama_pooling_type(self.ctx)) |
| 12213 | if pooling_type == llama_cpp.LLAMA_POOLING_TYPE_NONE: |
| 12214 | raise CompletionRequestValidationError( |
| 12215 | "/v1/embeddings requires a pooled embedding model; " |
| 12216 | "set model.pooling_type to MEAN, CLS, or LAST" |
| 12217 | ) |
| 12218 | if pooling_type == llama_cpp.LLAMA_POOLING_TYPE_RANK: |
| 12219 | raise CompletionRequestValidationError( |
| 12220 | "/v1/embeddings does not support reranking pooling" |
| 12221 | ) |
| 12222 | if len(inputs) > 2048: |
| 12223 | raise CompletionRequestValidationError( |
| 12224 | "embedding input batch size exceeds 2048" |
| 12225 | ) |
| 12226 | |
| 12227 | embeddings: List[List[float]] = [] |
| 12228 | total_tokens = 0 |
| 12229 | batch_sizes: List[int] = [] |
| 12230 | batch_token_count = 0 |
| 12231 | |
| 12232 | def decode_embedding_batch() -> None: |
| 12233 | nonlocal batch_token_count |
| 12234 | if not batch_sizes: |
| 12235 | return |
| 12236 | self.clear_memory() |
| 12237 | self.decode() |
| 12238 | self.clear_batch() |
| 12239 | for seq_id in range(len(batch_sizes)): |
| 12240 | ptr = llama_cpp.llama_get_embeddings_seq( |
| 12241 | self.ctx, |
| 12242 | llama_cpp.llama_seq_id(seq_id), |
| 12243 | ) |
| 12244 | if not ptr: |
| 12245 | raise RuntimeError(f"missing embedding output for input {seq_id}") |
| 12246 | embeddings.append( |
| 12247 | np.ctypeslib.as_array(ptr, shape=(self.n_embd_out,)).astype( |
| 12248 | float |
| 12249 | ).tolist() |
| 12250 | ) |
| 12251 | batch_sizes.clear() |
| 12252 | batch_token_count = 0 |
| 12253 | |
| 12254 | try: |
| 12255 | self.clear_batch() |
| 12256 | self.clear_memory() |
| 12257 | for input_item in inputs: |
| 12258 | tokens = ( |
| 12259 | self.tokenize(input_item) |
| 12260 | if isinstance(input_item, str) |
| 12261 | else list(input_item) |
no test coverage detected