| 215 | }); |
| 216 | |
| 217 | const createAppConfig = ( |
| 218 | env: "stage" | "prod", |
| 219 | a100Models: LoraConfig[], |
| 220 | a10Models: LoraConfig[], |
| 221 | ): Config => ({ |
| 222 | name: `inference-${env}`, |
| 223 | compute_config: "openpipe5", |
| 224 | cluster_env: "llm-env:7", |
| 225 | |
| 226 | ray_serve_config: { |
| 227 | applications: [ |
| 228 | { |
| 229 | name: "a100", |
| 230 | route_prefix: "/", |
| 231 | import_path: "aviary_private_endpoints.backend.server.run:router_application", |
| 232 | args: { |
| 233 | models: [], |
| 234 | multiplex_models: a100Models, |
| 235 | dynamic_lora_loading_path: `s3://${buckets[env]}/models/`, |
| 236 | }, |
| 237 | }, |
| 238 | { |
| 239 | name: "a10", |
| 240 | route_prefix: "/a10-v1", |
| 241 | import_path: "aviary_private_endpoints.backend.server.run:router_application", |
| 242 | args: { |
| 243 | models: [], |
| 244 | multiplex_models: a10Models, |
| 245 | dynamic_lora_loading_path: `s3://${buckets[env]}/models/`, |
| 246 | }, |
| 247 | }, |
| 248 | ], |
| 249 | }, |
| 250 | }); |
| 251 | |
| 252 | const prodConfig = createAppConfig( |
| 253 | "prod", |