(
self,
provider: Optional[str] = None,
model: Optional[str] = None,
dimensions: Optional[int] = None,
cache_dir: Optional[Path] = None,
)
| 235 | row_count += 1 |
| 236 | if row_count == 0: |
| 237 | return [] |
| 238 | return [value / row_count for value in pooled] |
| 239 | |
| 240 | return [float(value) for value in payload] |
| 241 | |
| 242 | |
| 243 | class EmbeddingService: |
| 244 | """Configurable embedding backend with caching and safe fallbacks.""" |
| 245 | |
| 246 | def __init__( |
| 247 | self, |
| 248 | provider: Optional[str] = None, |
| 249 | model: Optional[str] = None, |
| 250 | dimensions: Optional[int] = None, |
| 251 | cache_dir: Optional[Path] = None, |
| 252 | ): |
| 253 | requested_provider = ( |
| 254 | provider |
| 255 | or _get_first_env_value("PAPERFLOW_EMBED_PROVIDER", "EMBEDDING_PROVIDER") |
| 256 | or "hash" |
| 257 | ).strip().lower() |
| 258 | self.dimensions = int( |
| 259 | dimensions |
| 260 | or _get_first_env_value("PAPERFLOW_EMBED_DIMENSIONS", "EMBEDDING_DIMENSIONS") |
| 261 | or "768" |
| 262 | ) |
| 263 | self.cache_dir = Path(cache_dir or DEFAULT_CACHE_DIR) |
| 264 | self.cache_dir.mkdir(parents=True, exist_ok=True) |
| 265 | |
| 266 | self.client = None |
| 267 | self.local_model = None |
| 268 | self.model_source = None |
| 269 | |
| 270 | if requested_provider in {"openai", "dashscope", "aliyun", "bailian"}: |
| 271 | self.provider = "openai" |
| 272 | provider_hint = requested_provider |
| 273 | self.model = model or _get_openai_embedding_model(provider_hint) |
| 274 | api_key = _get_openai_api_key(provider_hint) |
| 275 | if OPENAI_AVAILABLE and not _is_placeholder_openai_key(api_key): |
| 276 | self.client = OpenAI( |
| 277 | api_key=api_key, |
| 278 | base_url=_get_openai_base_url(provider_hint), |
| 279 | timeout=_get_openai_timeout(provider_hint), |
| 280 | ) |
| 281 | else: |
| 282 | self.provider = "hash" |
| 283 | self.model = "hash" |
| 284 | elif requested_provider in {"nscale_api", "nscale"}: |
| 285 | self.provider = "nscale_api" |
| 286 | self.model = model or os.environ.get("NSCALE_EMBEDDING_MODEL") or os.environ.get("HF_EMBEDDING_MODEL") or "Qwen3-Embedding-8B" |
| 287 | api_key = ( |
| 288 | os.environ.get("NSCALE_API_KEY") |
| 289 | or os.environ.get("NSCALE_SERVICE_TOKEN") |
| 290 | or "" |
| 291 | ) |
| 292 | if _is_placeholder_nscale_key(api_key): |
| 293 | api_key = "" |
| 294 | base_url = ( |
nothing calls this directly
no test coverage detected