RouteModel runs after SetModelAndConfig and the schema-specific SetXRequest, looks at the resolved model's Router config, and (when present) reclassifies the request to one of the candidates. The middleware: 1. Loads MODEL_CONFIG from the echo context. If nil or HasRouter() is false, passes throug
(loader *config.ModelConfigLoader, appConfig *config.ApplicationConfig, store router.DecisionStore, fallbackUser *auth.User, extractor ProbeExtractor, source string, deps ClassifierDeps)
| 140 | // router.SourceChat for the OpenAI chat endpoint, router.SourceAnthropic |
| 141 | // for the Anthropic messages endpoint. |
| 142 | func RouteModel(loader *config.ModelConfigLoader, appConfig *config.ApplicationConfig, store router.DecisionStore, fallbackUser *auth.User, extractor ProbeExtractor, source string, deps ClassifierDeps) echo.MiddlewareFunc { |
| 143 | registry := deps.Registry |
| 144 | if registry == nil { |
| 145 | registry = router.NewRegistry() |
| 146 | } |
| 147 | candidateLoader := func(name string) (*config.ModelConfig, error) { |
| 148 | return loader.LoadModelConfigFileByNameDefaultOptions(name, appConfig) |
| 149 | } |
| 150 | return func(next echo.HandlerFunc) echo.HandlerFunc { |
| 151 | return func(c echo.Context) error { |
| 152 | cfg, ok := c.Get(CONTEXT_LOCALS_KEY_MODEL_CONFIG).(*config.ModelConfig) |
| 153 | if !ok || cfg == nil || !cfg.HasRouter() { |
| 154 | return next(c) |
| 155 | } |
| 156 | |
| 157 | parsed := c.Get(CONTEXT_LOCALS_KEY_LOCALAI_REQUEST) |
| 158 | if parsed == nil { |
| 159 | return next(c) |
| 160 | } |
| 161 | |
| 162 | probe, probeOK := extractor(parsed) |
| 163 | if !probeOK { |
| 164 | return next(c) |
| 165 | } |
| 166 | |
| 167 | classifier, err := GetOrBuildClassifier(registry, cfg, deps) |
| 168 | if err != nil { |
| 169 | // Build-time failures are config bugs (missing |
| 170 | // classifier_model, undeclared usecase, policy |
| 171 | // validation, ...). Silently falling back would hide |
| 172 | // them and make the router look "working" while the |
| 173 | // classifier model is never invoked — surface as 503 |
| 174 | // with the underlying reason so operators see it. |
| 175 | xlog.Warn("router: classifier build failed", |
| 176 | "router_model", cfg.Name, "classifier", cfg.Router.Classifier, "error", err) |
| 177 | return echo.NewHTTPError(503, "router classifier unavailable: "+err.Error()) |
| 178 | } |
| 179 | |
| 180 | result, err := router.Resolve(c.Request().Context(), cfg, classifier, candidateLoader, probe) |
| 181 | if err != nil { |
| 182 | xlog.Warn("router: resolve failed", "router_model", cfg.Name, "error", err) |
| 183 | return echo.NewHTTPError(500, err.Error()) |
| 184 | } |
| 185 | |
| 186 | if req, ok := parsed.(schema.LocalAIRequest); ok { |
| 187 | chosen := result.ChosenModel |
| 188 | req.ModelName(&chosen) |
| 189 | } |
| 190 | |
| 191 | c.Set(CONTEXT_LOCALS_KEY_MODEL_CONFIG, result.ChosenConfig) |
| 192 | // Preserve an upstream requested model (e.g. an alias that points |
| 193 | // at this router model) so accounting keeps the name the client |
| 194 | // actually sent. Served always reflects the final candidate. |
| 195 | if c.Get(ContextKeyRequestedModel) == nil { |
| 196 | c.Set(ContextKeyRequestedModel, result.RouterModel) |
| 197 | } |
| 198 | c.Set(ContextKeyServedModel, result.ChosenModel) |
| 199 |