| 5 | import { getModelConfigByModelName, MODEL_TYPE } from '../src/modelLoader' |
| 6 | |
| 7 | export class EvaluationRunner { |
| 8 | static metrics = new Map<string, string[]>() |
| 9 | |
| 10 | static getCostMetrics = async (selectedProvider: string, selectedModel: string) => { |
| 11 | let modelConfig = await getModelConfigByModelName(MODEL_TYPE.CHAT, selectedProvider, selectedModel) |
| 12 | if (modelConfig) { |
| 13 | if (modelConfig['cost_values']) { |
| 14 | return modelConfig.cost_values |
| 15 | } |
| 16 | return { cost_values: modelConfig } |
| 17 | } else { |
| 18 | modelConfig = await getModelConfigByModelName(MODEL_TYPE.LLM, selectedProvider, selectedModel) |
| 19 | if (modelConfig) { |
| 20 | if (modelConfig['cost_values']) { |
| 21 | return modelConfig.cost_values |
| 22 | } |
| 23 | return { cost_values: modelConfig } |
| 24 | } |
| 25 | } |
| 26 | return undefined |
| 27 | } |
| 28 | |
| 29 | static async getAndDeleteMetrics(id: string) { |
| 30 | const val = EvaluationRunner.metrics.get(id) |
| 31 | if (val) { |
| 32 | try { |
| 33 | //first lets get the provider and model |
| 34 | let selectedModel = undefined |
| 35 | let selectedProvider = undefined |
| 36 | if (val && val.length > 0) { |
| 37 | let modelName = '' |
| 38 | let providerName = '' |
| 39 | for (let i = 0; i < val.length; i++) { |
| 40 | const metric = val[i] |
| 41 | if (typeof metric === 'object') { |
| 42 | modelName = metric['model'] |
| 43 | providerName = metric['provider'] |
| 44 | } else { |
| 45 | modelName = JSON.parse(metric)['model'] |
| 46 | providerName = JSON.parse(metric)['provider'] |
| 47 | } |
| 48 | |
| 49 | if (modelName) { |
| 50 | selectedModel = modelName |
| 51 | } |
| 52 | if (providerName) { |
| 53 | selectedProvider = providerName |
| 54 | } |
| 55 | } |
| 56 | } |
| 57 | if (selectedProvider && selectedModel) { |
| 58 | const modelConfig = await EvaluationRunner.getCostMetrics(selectedProvider, selectedModel) |
| 59 | if (modelConfig) { |
| 60 | val.push(JSON.stringify({ cost_values: modelConfig })) |
| 61 | } |
| 62 | } |
| 63 | } catch (error) { |
| 64 | //stay silent |
nothing calls this directly
no test coverage detected