Class SequentialFunctionRewardManager

verl/workers/reward/function.py:82–154

class SequentialFunctionRewardManager(FunctionRewardManager):
    reward_fn: SequentialRewardFunction

    def compute_reward(self, data: DataProto) -> Tuple[torch.Tensor, Dict[str, List[float]]]:
        reward_tensor = torch.zeros_like(data.batch["responses"], dtype=torch.float32)
        reward_metrics = defaultdict(list)
        response_ids = data.batch["responses"]
        response_length = data.batch["response_mask"].sum(dim=-1)

        # pull arrays once
        gt_arr = data.non_tensor_batch["ground_truth"]
        qn_arr = data.non_tensor_batch.get("question", None)
        desc_arr = data.non_tensor_batch.get("description_answers", None)
        # support either `images` or `image` in upstream payload
        img_arr = (
            data.non_tensor_batch.get("images", None)
            if "images" in data.non_tensor_batch
            else data.non_tensor_batch.get("image", None)
        )

        # inspect parameters for flexible calling
        target_fn = self.reward_fn.func if isinstance(self.reward_fn, partial) else self.reward_fn
        param_names = list(inspect.signature(target_fn).parameters.keys())

        for i in range(len(data)):
            valid_response_ids = response_ids[i][: response_length[i]]
            response_str = self.tokenizer.decode(
                valid_response_ids, skip_special_tokens=self.config.skip_special_tokens
            )
            ground_truth = gt_arr[i]
            question = None if qn_arr is None else qn_arr[i]
            description_answer = "" if desc_arr is None else desc_arr[i]
            image_val = None if img_arr is None else img_arr[i]

            base_args = [response_str, ground_truth]
            call_kwargs = {}
            # prefer kwargs when the function advertises them
            if question is not None and "question" in param_names:
                call_kwargs["question"] = question
            if "description_answers" in param_names:
                call_kwargs["description_answers"] = description_answer
            if "images" in param_names:
                call_kwargs["images"] = image_val
            elif "image" in param_names:
                call_kwargs["image"] = image_val

            try:
                score = self.reward_fn(*base_args, **call_kwargs)
            except TypeError:
                # fallback 1: include question as positional third
                try:
                    if question is None:
                        raise TypeError
                    score = self.reward_fn(response_str, ground_truth, question)
                except TypeError:
                    # fallback 2: include description and image positionally if accepted
                    try:
                        score = self.reward_fn(