Method generate_inner

vlmeval/vlm/parrot.py:138–195

(self, message, dataset=None)

        return answer

    def generate_inner(self, message, dataset=None):
        query, image_paths = self.prepare_inputs(message)
        images_list = [Image.open(image_path).convert('RGB') for image_path in image_paths]
        args = abstractproperty()
        args.image_aspect_ratio = 'pad'
        image_tensors = self.process_images(images_list, self.image_processor, args).cuda()
        prompt, input_ids = self.conversation_formatter.format_query(query)
        input_ids = input_ids.unsqueeze(0).cuda()

        with torch.inference_mode():
            kwargs = dict(
                images=image_tensors,
            )
            kwargs.update(self.kwargs)
            output_ids = self.model.generate(input_ids, **kwargs)

        input_token_len = input_ids.shape[1]
        n_diff_input_output = (input_ids != output_ids[:, :input_token_len]).sum().item()
        if n_diff_input_output > 0:
            print(f'[Warning] {n_diff_input_output} output_ids are not the same as the input_ids')
        response = self.tokenizer.batch_decode(output_ids[:, input_token_len:],
                                               skip_special_tokens=True)[0].strip(string.whitespace)
        answer = response

        if query.endswith("Answer with the option's letter from the given choices directly.") or query.endswith(
                '请直接回答选项字母。'):
            qtype = 'multiple-choice'
            while True:
                answer = answer.strip(string.punctuation + string.whitespace)
                if len(answer) > 1:
                    if answer[0] in string.ascii_uppercase and answer[1] in string.whitespace + string.punctuation:
                        answer = answer[0]
                        break
                    elif answer[-1] in string.ascii_uppercase and answer[-2] in string.whitespace + string.punctuation:
                        answer = answer[-1]
                        break
                    elif listinstr(['answer is', 'answer:'], answer.lower()):
                        answer = self.process_answer_prefix(answer, ['answer is', 'answer:'])
                        answer = self.process_answer_prefix(answer, ['option'])
                    else:
                        break
                else:
                    break
        else:
            qtype = 'open'

        if self.count % 50 == 0 and int(os.environ.get('LOCAL_RANK', '0')) == 0:
            print(f'\n{self.BEGIN_LINE}')
            print(f'image_paths: {image_paths}\n')
            print(f'prompt: {prompt}\n')
            print(f'qtype: {qtype}\n')
            print(f'output: {response}\n')
            print(f'answer: {answer}\n')
            print(f'{self.END_LINE}\n', flush=True)

        self.count += 1

        return answer