Function a10Model

inference/deploy.ts:166–186

(modelId: string, keepHot: boolean, contextSize: number)

  .parse();

const a10Model = (modelId: string, keepHot: boolean, contextSize: number) =>
  createLoraConfig(
    {
      deployment_config: {
        autoscaling_config: {
          min_replicas: keepHot ? 1 : 0,
          initial_replicas: keepHot ? 1 : 0,
          max_replicas: 24,
          target_num_ongoing_requests_per_replica: 8,
          downscale_smoothing_factor: 0.5,
        },
        max_concurrent_queries: 30,
      },
      engine_config: {
        model_id: modelId,
        max_total_tokens: contextSize,
        engine_kwargs: { max_num_seqs: 12 },
      },
    },
    { gpuType: "a10" },
  );

// Otherwise we OOM on A10s
const fixLlama13b = (config: LoraConfig) =>