( config: ConfigYaml, )
| 19 | } |
| 20 | |
| 21 | export function validateConfigYaml( |
| 22 | config: ConfigYaml, |
| 23 | ): ConfigValidationError[] { |
| 24 | const errors: ConfigValidationError[] = []; |
| 25 | |
| 26 | try { |
| 27 | configYamlSchema.parse(config); |
| 28 | } catch (e: any) { |
| 29 | return [ |
| 30 | { |
| 31 | fatal: true, |
| 32 | message: e.message, |
| 33 | }, |
| 34 | ]; |
| 35 | } |
| 36 | |
| 37 | config.models?.forEach((model) => { |
| 38 | if ("uses" in model) { |
| 39 | return; |
| 40 | } |
| 41 | |
| 42 | // request headers to the llm api should not contain unicode characters |
| 43 | if (model.apiKey && containsUnicode(model.apiKey)) { |
| 44 | errors.push({ |
| 45 | fatal: true, |
| 46 | message: `Model "${model.name}" has an API key containing unicode characters. API keys should only contain ASCII characters.`, |
| 47 | }); |
| 48 | } |
| 49 | |
| 50 | if (model.requestOptions?.headers) { |
| 51 | for (const [key, value] of Object.entries(model.requestOptions.headers)) { |
| 52 | if (containsUnicode(key) || containsUnicode(value)) { |
| 53 | errors.push({ |
| 54 | fatal: true, |
| 55 | message: `Model "${model.name}" has a request header "${key}" containing unicode characters. Request headers should only contain ASCII characters.`, |
| 56 | }); |
| 57 | } |
| 58 | } |
| 59 | } |
| 60 | // Max tokens not too close to context length |
| 61 | const effectiveContextLength = |
| 62 | model.contextLength ?? model.defaultCompletionOptions?.contextLength; |
| 63 | const effectiveMaxTokens = model.defaultCompletionOptions?.maxTokens; |
| 64 | if (effectiveContextLength && effectiveMaxTokens) { |
| 65 | const difference = effectiveContextLength - effectiveMaxTokens; |
| 66 | |
| 67 | if (difference < 1000) { |
| 68 | errors.push({ |
| 69 | fatal: false, |
| 70 | message: `Model "${model.name}" has a contextLength of ${effectiveContextLength} and a maxTokens of ${effectiveMaxTokens}. This leaves only ${difference} tokens for input context and will likely result in your inputs being truncated.`, |
| 71 | }); |
| 72 | } |
| 73 | } |
| 74 | |
| 75 | if (model.roles?.includes("autocomplete")) { |
| 76 | const modelName = model.model.toLocaleLowerCase(); |
| 77 | const nonAutocompleteModels = [ |
| 78 | // "gpt", |
no test coverage detected