Function runInference

app/src/server/modal-rpc/clients.ts:54–99

(args: ModalInput)

}.modal.run`;

export const runInference = async (args: ModalInput) => {
  const resp = await fetch(inferenceBase, {
    method: "POST",
    body: JSON.stringify(args),
    headers: {
      "Content-Type": "application/json",
    },
  });

  const respText = await resp.text();
  if (!resp.ok) {
    captureException("Failed to run inference.", {
      extra: {
        response: respText,
        ...pick(args, ["model", "n", "max_tokens", "temperature"]),
      },
    });
    throw new Error("Failed to run inference");
  }

  let json;
  try {
    json = JSON.parse(respText);
  } catch (e) {
    // captureException
    captureException("Failed to parse response from modal.", {
      extra: {
        response: respText,
        ...pick(args, ["model", "n", "max_tokens", "temperature"]),
      },
    });
    throw new Error("Failed to parse LLM response");
  }
  const output = outputSchema.safeParse(json);
  if (output.success) {
    return output.data;
  } else {
    captureException("Failed to validate output from modal.", {
      extra: {
        response: respText,
        ...pick(args, ["model", "n", "max_tokens", "temperature"]),
      },
    });
    throw new Error("Failed to validate LLM response");
  }
};