spawnGRPCModel starts the backend process (or attaches to a remote address), waits for it to come up, and issues the LoadModel RPC. Reached only for actual loads: LoadModel resolves cache hits and coalesces concurrent loads before invoking the grpcModel closure. uri is the resolved external-backend
(backend, uri string, o *Options, modelID, modelName, modelFile string)
| 88 | // resolved external-backend runtime (empty when the backend isn't |
| 89 | // registered). |
| 90 | func (ml *ModelLoader) spawnGRPCModel(backend, uri string, o *Options, modelID, modelName, modelFile string) (*Model, error) { |
| 91 | var client *Model |
| 92 | |
| 93 | getFreeAddress := func() (string, error) { |
| 94 | port, err := freeport.GetFreePort() |
| 95 | if err != nil { |
| 96 | return "", fmt.Errorf("failed allocating free ports: %s", err.Error()) |
| 97 | } |
| 98 | return fmt.Sprintf("127.0.0.1:%d", port), nil |
| 99 | } |
| 100 | |
| 101 | // If no specific model path is set for transformers/HF, set it to the model path |
| 102 | for _, env := range []string{"HF_HOME", "TRANSFORMERS_CACHE", "HUGGINGFACE_HUB_CACHE"} { |
| 103 | if os.Getenv(env) == "" { |
| 104 | err := os.Setenv(env, ml.ModelPath) |
| 105 | if err != nil { |
| 106 | xlog.Error("unable to set environment variable to modelPath", "error", err, "name", env, "modelPath", ml.ModelPath) |
| 107 | } |
| 108 | } |
| 109 | } |
| 110 | |
| 111 | // Check if the backend is provided as external |
| 112 | if uri != "" { |
| 113 | xlog.Debug("Loading external backend", "uri", uri) |
| 114 | // check if uri is a file or an address |
| 115 | if fi, err := os.Stat(uri); err == nil { |
| 116 | xlog.Debug("external backend is file", "file", fi) |
| 117 | serverAddress, err := getFreeAddress() |
| 118 | if err != nil { |
| 119 | return nil, fmt.Errorf("failed allocating free ports: %s", err.Error()) |
| 120 | } |
| 121 | // Make sure the process is executable |
| 122 | process, err := ml.startProcess(uri, modelID, serverAddress) |
| 123 | if err != nil { |
| 124 | xlog.Error("failed to launch", "error", err, "path", uri) |
| 125 | return nil, err |
| 126 | } |
| 127 | |
| 128 | xlog.Debug("GRPC Service Started") |
| 129 | |
| 130 | client = NewModel(modelID, serverAddress, process) |
| 131 | } else { |
| 132 | xlog.Debug("external backend is a uri") |
| 133 | // address |
| 134 | client = NewModel(modelID, uri, nil) |
| 135 | } |
| 136 | } else { |
| 137 | xlog.Error("Backend not found", "backend", backend) |
| 138 | return nil, fmt.Errorf("backend not found: %s", backend) |
| 139 | } |
| 140 | |
| 141 | xlog.Debug("Wait for the service to start up") |
| 142 | xlog.Debug("Options", "options", o.gRPCOptions) |
| 143 | |
| 144 | // Wait for the service to start up |
| 145 | ready := false |
| 146 | for i := range o.grpcAttempts { |
| 147 | alive, err := client.GRPC(o.parallelRequests, ml.wd).HealthCheck(context.Background()) |
no test coverage detected