ModelScore loads the backend for modelConfig and returns a closure that scores `candidates` against `prompt`. The closure is bound to the loaded model so callers can keep it around for repeat scoring within the same request without re-resolving the backend.
(prompt string, candidates []string, opts ScoreOptions, loader *model.ModelLoader, modelConfig config.ModelConfig, appConfig *config.ApplicationConfig)
| 74 | // the loaded model so callers can keep it around for repeat scoring |
| 75 | // within the same request without re-resolving the backend. |
| 76 | func ModelScore(prompt string, candidates []string, opts ScoreOptions, loader *model.ModelLoader, modelConfig config.ModelConfig, appConfig *config.ApplicationConfig) (func(ctx context.Context) ([]CandidateScore, error), error) { |
| 77 | modelOpts := ModelOptions(modelConfig, appConfig) |
| 78 | inferenceModel, err := loader.Load(modelOpts...) |
| 79 | if err != nil { |
| 80 | recordModelLoadFailure(appConfig, modelConfig.Name, modelConfig.Backend, err, nil) |
| 81 | return nil, err |
| 82 | } |
| 83 | b, ok := inferenceModel.(grpc.Backend) |
| 84 | if !ok { |
| 85 | return nil, fmt.Errorf("scoring not supported by backend %q", modelConfig.Backend) |
| 86 | } |
| 87 | if len(candidates) == 0 { |
| 88 | return nil, fmt.Errorf("Score: candidates must be non-empty") |
| 89 | } |
| 90 | return func(ctx context.Context) ([]CandidateScore, error) { |
| 91 | // Surface score calls in the Traces UI alongside the LLM calls |
| 92 | // they typically gate (router classifier, eval scoring). Without |
| 93 | // this, a router-classified request shows only the downstream LLM |
| 94 | // trace with no record of the classification that picked it. |
| 95 | var startTime time.Time |
| 96 | if appConfig.EnableTracing { |
| 97 | trace.InitBackendTracingIfEnabled(appConfig.TracingMaxItems, appConfig.TracingMaxBodyBytes) |
| 98 | startTime = time.Now() |
| 99 | } |
| 100 | resp, err := b.Score(ctx, &pb.ScoreRequest{ |
| 101 | Prompt: prompt, |
| 102 | Candidates: candidates, |
| 103 | IncludeTokenLogprobs: opts.IncludeTokenLogprobs, |
| 104 | LengthNormalize: opts.LengthNormalize, |
| 105 | }) |
| 106 | results := scoreResponseToCandidates(resp, opts.IncludeTokenLogprobs) |
| 107 | if appConfig.EnableTracing { |
| 108 | errStr := "" |
| 109 | if err != nil { |
| 110 | errStr = err.Error() |
| 111 | } |
| 112 | trace.RecordBackendTrace(trace.BackendTrace{ |
| 113 | Timestamp: startTime, |
| 114 | Duration: time.Since(startTime), |
| 115 | Type: trace.BackendTraceScore, |
| 116 | ModelName: modelConfig.Name, |
| 117 | Backend: modelConfig.Backend, |
| 118 | Summary: trace.TruncateString(prompt, 200), |
| 119 | Error: errStr, |
| 120 | Data: map[string]any{ |
| 121 | // Copy candidates so the trace buffer doesn't pin a |
| 122 | // caller-owned slice for the lifetime of the ring. |
| 123 | "candidates": append([]string(nil), candidates...), |
| 124 | "results": results, |
| 125 | }, |
| 126 | }) |
| 127 | } |
| 128 | if err != nil { |
| 129 | return nil, err |
| 130 | } |
| 131 | return results, nil |
| 132 | }, nil |
| 133 | } |
no test coverage detected