(apiKey: string, baseUrl: string)
| 12 | * @throws Will throw an error if the request fails or the response is not as expected. |
| 13 | */ |
| 14 | export async function getLiteLLMModels(apiKey: string, baseUrl: string): Promise<ModelRecord> { |
| 15 | try { |
| 16 | const headers: Record<string, string> = { |
| 17 | "Content-Type": "application/json", |
| 18 | ...DEFAULT_HEADERS, |
| 19 | } |
| 20 | |
| 21 | if (apiKey) { |
| 22 | headers["Authorization"] = `Bearer ${apiKey}` |
| 23 | } |
| 24 | // Use URL constructor to properly join base URL and path |
| 25 | // This approach handles all edge cases including paths, query params, and fragments |
| 26 | const urlObj = new URL(baseUrl) |
| 27 | // Normalize the pathname by removing trailing slashes and multiple slashes |
| 28 | urlObj.pathname = urlObj.pathname.replace(/\/+$/, "").replace(/\/+/g, "/") + "/v1/model/info" |
| 29 | const url = urlObj.href |
| 30 | // Added timeout to prevent indefinite hanging |
| 31 | const response = await axios.get(url, { headers, timeout: 5000 }) |
| 32 | const models: ModelRecord = {} |
| 33 | |
| 34 | // Process the model info from the response |
| 35 | if (response.data && response.data.data && Array.isArray(response.data.data)) { |
| 36 | for (const model of response.data.data) { |
| 37 | const modelName = model.model_name |
| 38 | const modelInfo = model.model_info |
| 39 | const litellmModelName = model?.litellm_params?.model as string | undefined |
| 40 | |
| 41 | if (!modelName || !modelInfo || !litellmModelName) continue |
| 42 | |
| 43 | models[modelName] = { |
| 44 | maxTokens: modelInfo.max_output_tokens || modelInfo.max_tokens || 8192, |
| 45 | contextWindow: modelInfo.max_input_tokens || 200000, |
| 46 | supportsImages: Boolean(modelInfo.supports_vision), |
| 47 | supportsPromptCache: Boolean(modelInfo.supports_prompt_caching), |
| 48 | inputPrice: modelInfo.input_cost_per_token ? modelInfo.input_cost_per_token * 1000000 : undefined, |
| 49 | outputPrice: modelInfo.output_cost_per_token |
| 50 | ? modelInfo.output_cost_per_token * 1000000 |
| 51 | : undefined, |
| 52 | cacheWritesPrice: modelInfo.cache_creation_input_token_cost |
| 53 | ? modelInfo.cache_creation_input_token_cost * 1000000 |
| 54 | : undefined, |
| 55 | cacheReadsPrice: modelInfo.cache_read_input_token_cost |
| 56 | ? modelInfo.cache_read_input_token_cost * 1000000 |
| 57 | : undefined, |
| 58 | description: `${modelName} via LiteLLM proxy`, |
| 59 | } |
| 60 | } |
| 61 | } else { |
| 62 | // If response.data.data is not in the expected format, consider it an error. |
| 63 | console.error("Error fetching LiteLLM models: Unexpected response format", response.data) |
| 64 | throw new Error("Failed to fetch LiteLLM models: Unexpected response format.") |
| 65 | } |
| 66 | |
| 67 | return models |
| 68 | } catch (error: any) { |
| 69 | console.error("Error fetching LiteLLM models:", error.message ? error.message : error) |
| 70 | if (axios.isAxiosError(error) && error.response) { |
| 71 | throw new Error( |
no test coverage detected