Function generateResponse

src/controllers/generation.controller.ts:44–200

(
  request: FastifyRequestTypebox<typeof GenerateResponseSchema>,
  reply: FastifyReplyTypebox<typeof GenerateResponseSchema>,
)

 * @returns {Promise<void>} - A promise that resolves when the response is generated
 */
export const generateResponse = async (
  request: FastifyRequestTypebox<typeof GenerateResponseSchema>,
  reply: FastifyReplyTypebox<typeof GenerateResponseSchema>,
) => {
  try {
    const { data, stream, timeout = DEFAULT_TIMEOUT_IN_MS, maxRetries = DEFAULT_MAX_RETRIES } = request.body;
    data.sort((a, b) => a.order - b.order);
    const orm = request.server.orm;
    for (const params of data) {
      let provider;
      let model;
      if (params.providerId) {
        provider = await orm.getRepository(Provider).findOne({ where: { id: params.providerId } });
      } else if (params.providerName) {
        provider = await orm.getRepository(Provider).findOne({ where: { name: params.providerName } });
      } else {
        throw new ApiError(400, `Neither provider ID nor name was provided`);
      }
      if (!!!provider) {
        throw new ApiError(
          400,
          `Provider with ${
            params.providerId ? `id ${params.providerId}` : `name ${params.providerName}`
          } does not exist`,
        );
      }

      if (params.modelId) {
        model = await orm.getRepository(Model).findOne({ where: { id: params.modelId }, relations: ["cost"] });
      } else if (params.modelName) {
        model = await orm.getRepository(Model).findOne({ where: { name: params.modelName }, relations: ["cost"] });
      } else {
        throw new ApiError(400, `Neither model ID nor name was provided`);
      }
      if (!!!model) {
        throw new ApiError(
          400,
          `Model with ${params.modelId ? `id ${params.modelId}` : `name ${params.modelName}`} does not exist`,
        );
      }

      const obj = await orm
        .getRepository(ModelHealthCheck)
        .findOne({ where: { modelId: model.id, providerId: provider.id }, order: { lastChecked: "DESC" } });

      if (!obj || obj.isAvailable === false || obj.latency > MODEL_LATENCY_LIMIT_FOR_GENERATION) {
        continue;
      }

      if (stream) {
        const response = streamHandler({
          modelName: model.name,
          providerName: provider.name,
          promptParams: params?.promptParams,
          timeout,
          maxRetries,
        });
        reply.sse(