Interface Inference

sdk/javascript/src/interface/inference.ts:1–138

export default interface Inference {
  /** Send back pure media or JSON. Default: `true` */
  json?: boolean;
  /** Stream results back. Default: `false` */
  stream?: boolean;

  /** Maximum length of generated tokens, defaults to 20. Overrides `max_new_tokens`. */
  max_length?: number;

  /** Maximum number of tokens to generate, ignoring prompt tokens. */
  max_new_tokens?: number;

  /** Minimum sequence length, defaults to 0. Overrides `min_new_tokens`. */
  min_length?: number;

  /** Minimum number of tokens to generate, ignoring prompt tokens. */
  min_new_tokens?: number;

  /** Controls stopping condition for beam search. `true` stops at `num_beams` complete candidates; `never` continues indefinitely. */
  early_stopping?: boolean | "never";

  /** Maximum computation time in seconds. Generation completes current pass after time expires. */
  max_time?: number;

  /** Whether to use sampling instead of greedy decoding, defaults to false. */
  do_sample?: boolean;

  /** Number of beams for beam search, defaults to 1. */
  num_beams?: number;

  /** Number of groups to divide beams into for diversity, defaults to 1. */
  num_beam_groups?: number;

  /** Balances model confidence and degeneration penalty in contrastive search decoding. */
  penalty_alpha?: number;

  /** Whether to use past attentions to speed up decoding, defaults to true. */
  use_cache?: boolean;

  /** Modulates next token probabilities, defaults to 1.0. */
  temperature?: number;

  /** Keeps highest probability vocabulary tokens for top-k filtering, defaults to 50. */
  top_k?: number;

  /** Keeps smallest set of tokens summing to at least `top_p`, defaults to 1.0. */
  top_p?: number;

  /** Local typicality, keeps smallest set of most typical tokens summing to `typical_p`, defaults to 1.0. */
  typical_p?: number;

  /** Only samples tokens with conditional probability above `epsilon_cutoff`. */
  epsilon_cutoff?: number;

  /** Hybrid sampling method using `eta_cutoff` and expected token probability. */
  eta_cutoff?: number;

  /** Subtracts from a beam's score if it generates a token same as any other beam's token at that time. */
  diversity_penalty?: number;