Video preprocessor, converts videos into crops and tokens
| 13 | |
| 14 | @dataclass |
| 15 | class VideoPreprocessorConfig(VideoLoaderConfig): |
| 16 | """Video preprocessor, converts videos into crops and tokens""" |
| 17 | |
| 18 | time_mode: str = "per-frame-compact" |
| 19 | |
| 20 | subtitle_mode: str = "frame_1" # "frame_N", "all", "truncate_N", "ignore" |
| 21 | |
| 22 | pooling_w: int = 3 |
| 23 | """pooling w stride""" |
| 24 | |
| 25 | pooling_h: int = 3 |
| 26 | """pooling h stride""" |
| 27 | |
| 28 | use_frame_special_tokens: bool = True |
| 29 | """Whether to use frame special tokens in the video preprocessor""" |
| 30 | |
| 31 | per_frame_special_token: bool = False |
| 32 | """Put special tokens around each frame instead of around all the frames""" |
| 33 | |
| 34 | max_subtitle_tokens: Optional[int] = None |
| 35 | """Max number of subtitle tokens to use""" |
| 36 | |
| 37 | @classmethod |
| 38 | def build_from_legacy_config(cls, config: D) -> D: |
| 39 | assert config.max_crops == 1 |
| 40 | assert config.periodic_high_res_frame is None |
| 41 | assert not config.image_padding_mask |
| 42 | assert not config.query_based_resolution_selection |
| 43 | return VideoPreprocessorConfig( |
| 44 | max_frames=config.max_frames, |
| 45 | frame_sample_mode=config.frame_sample_mode, |
| 46 | candidate_sampling_fps=config.candidate_sampling_fps, |
| 47 | cache_videos=config.cache_videos, |
| 48 | loading_method=config.loading_method, |
| 49 | max_fps=config.max_fps, |
| 50 | time_sampling=config.time_sampling, |
| 51 | time_mode=config.time_mode, |
| 52 | subtitle_mode=config.subtitle_mode, |
| 53 | pooling_w=config.pooling_w, |
| 54 | pooling_h=config.pooling_h, |
| 55 | use_frame_special_tokens=config.use_frame_special_tokens, |
| 56 | per_frame_special_token=True, |
| 57 | max_subtitle_tokens=config.max_subtitle_tokens, |
| 58 | ) |
| 59 | |
| 60 | def build_video_preprocessor(self, tokenizer, image_preprocessor, add_end_of_mm_token=False) -> 'TokenIndexingVideoPreprocessor': |
| 61 | assert self.pooling_w == self.pooling_h |
| 62 | return TokenIndexingVideoPreprocessor( |
| 63 | tokenizer, |
| 64 | image_preprocessor, |
| 65 | video_loader=self.build_video_loader(), |
| 66 | pooling=self.pooling_w, |
| 67 | time_mode=self.time_mode, |
| 68 | subtitle_mode=self.subtitle_mode, |
| 69 | use_frame_special_tokens=self.use_frame_special_tokens, |
| 70 | max_subtitle_tokens=self.max_subtitle_tokens, |
| 71 | per_frame_special_token=self.per_frame_special_token, |
| 72 | ) |
no outgoing calls
no test coverage detected