ModelTokenClassify loads the backend for modelConfig and returns a closure that classifies `text`. Mirrors ModelScore: the closure is bound to the loaded model so a caller can reuse it within a request without re-resolving the backend. When tracing is enabled it records a BackendTraceTokenClassify
(text string, opts TokenClassifyOptions, loader *model.ModelLoader, modelConfig config.ModelConfig, appConfig *config.ApplicationConfig)
| 74 | // This is the technical view for debugging false positives (e.g. a phone |
| 75 | // number scored as SSN); the persisted PIIEvent keeps only a hash. |
| 76 | func ModelTokenClassify(text string, opts TokenClassifyOptions, loader *model.ModelLoader, modelConfig config.ModelConfig, appConfig *config.ApplicationConfig) (func(ctx context.Context) ([]TokenEntity, error), error) { |
| 77 | modelOpts := ModelOptions(modelConfig, appConfig) |
| 78 | inferenceModel, err := loader.Load(modelOpts...) |
| 79 | if err != nil { |
| 80 | recordModelLoadFailure(appConfig, modelConfig.Name, modelConfig.Backend, err, nil) |
| 81 | return nil, err |
| 82 | } |
| 83 | return func(ctx context.Context) ([]TokenEntity, error) { |
| 84 | var startTime time.Time |
| 85 | if appConfig.EnableTracing { |
| 86 | trace.InitBackendTracingIfEnabled(appConfig.TracingMaxItems, appConfig.TracingMaxBodyBytes) |
| 87 | startTime = time.Now() |
| 88 | } |
| 89 | resp, err := inferenceModel.TokenClassify(ctx, &pb.TokenClassifyRequest{ |
| 90 | Text: text, |
| 91 | Threshold: opts.Threshold, |
| 92 | }) |
| 93 | entities := tokenClassifyResponseToEntities(resp) |
| 94 | if appConfig.EnableTracing { |
| 95 | trace.RecordBackendTrace(tokenClassifyTrace(modelConfig, text, opts.Threshold, entities, startTime, err)) |
| 96 | } |
| 97 | if err != nil { |
| 98 | return nil, err |
| 99 | } |
| 100 | return entities, nil |
| 101 | }, nil |
| 102 | } |
| 103 | |
| 104 | // tokenClassifyTrace assembles the Traces-UI row for one NER call: the input |
| 105 | // preview, the threshold, and every detected entity (group, byte range, |