Method get_video_embedding

demo/test.py:310–317

(self, video_file)

    @torch.no_grad()
    def get_video_embedding(self, video_file):
        pixel_values = load_video(video_file)
        TC, H, W = pixel_values.shape
        pixel_values = pixel_values.reshape(TC // 3, 3, H, W).transpose(0, 1)  # [C, T, H, W]
        pixel_values = pixel_values.unsqueeze(0).to(self.device, dtype=self.dtype)
        assert len(pixel_values.shape) == 5
        language_model_inputs = self.model.extract_feature(pixel_values)
        return language_model_inputs

    @torch.no_grad()
    def answer(self, conversations, language_model_inputs, modal_type="image"):