| 29 | } |
| 30 | |
| 31 | func (s *Server) inferOrchestrator() *inferexec.Orchestrator { |
| 32 | var ledger inferexec.Ledger |
| 33 | if s.ledger != nil { |
| 34 | ledger = serverLedger{s} |
| 35 | } |
| 36 | return &inferexec.Orchestrator{ |
| 37 | Policy: s.policy, |
| 38 | Router: s.router, |
| 39 | Reliability: s.reliability, |
| 40 | SLO: s.sloEngine, |
| 41 | Ledger: ledger, |
| 42 | CachedHealth: s.cachedBackendHealth, |
| 43 | InferInflight: func() int32 { return s.inferInflight.Load() }, |
| 44 | BeginInferLoad: s.beginInferLoad, |
| 45 | RunRAG: func(ctx context.Context, sw *execution.StepWriter, req *types.AIRequest) error { |
| 46 | return s.runRAGPipeline(ctx, sw, req) |
| 47 | }, |
| 48 | BuildFallback: s.buildFallback, |
| 49 | NoteTimeout: s.noteTimeoutForScaling, |
| 50 | ParseRAGError: func(err error) (trace string, httpStatus int, errCode, msg string, ok bool) { |
| 51 | var pe *ragPipelineError |
| 52 | if errors.As(err, &pe) { |
| 53 | return pe.trace, pe.httpStatus, pe.errCode, pe.msg, true |
| 54 | } |
| 55 | return "", 0, "", "", false |
| 56 | }, |
| 57 | } |
| 58 | } |