Function createLoraConfig

inference/deploy.ts:106–149

(
  baseConfig: PartialDeep<LoraConfig>,
  options: { gpuType: "a10" | "a100_40g" },
)

};

const createLoraConfig = (
  baseConfig: PartialDeep<LoraConfig>,
  options: { gpuType: "a10" | "a100_40g" },
): LoraConfig => {
  const maxRequests = baseConfig.engine_config?.engine_kwargs?.max_num_seqs;

  if (maxRequests === undefined) {
    throw new Error("max_num_seqs is required");
  }

  const withDefaults = cloneDeep(baseConfig);

  withDefaults.multiplex_config ??= { max_num_models_per_replica: maxRequests };

  withDefaults.engine_config = {
    ...withDefaults.engine_config,
    hf_model_id: withDefaults.engine_config?.hf_model_id ?? withDefaults.engine_config?.model_id,
  };

  const acceleratorConfig = {
    [`accelerator_type_${options.gpuType}`]: 0.01,
  };
  withDefaults.scaling_config = withDefaults.scaling_config ??= {
    resources_per_worker: acceleratorConfig,
    ...(withDefaults?.scaling_config ?? {}),
  };

  withDefaults.deployment_config = {
    ...withDefaults.deployment_config,

    max_concurrent_queries: withDefaults.deployment_config?.max_concurrent_queries ?? maxRequests,
    ray_actor_options: { resources: acceleratorConfig },
  };

  withDefaults.engine_config = {
    ...withDefaults.engine_config,
    engine_kwargs: {
      ...withDefaults.engine_config?.engine_kwargs,
      max_loras: maxRequests,
    },
  };

  return LoraConfigSchema.parse(withDefaults);
};

const argv = await yargs(hideBin(process.argv))
  .options({