Produce a reference speech signal by sampling only a few short segments. Instead of extracting and running VAD over the entire reference, this samples ``segment_count`` short windows spread across it, runs VAD on each, and places the results into a full-length array that is zero everywh
| 758 | |
| 759 | |
| 760 | class MultiSegmentVideoSpeechTransformer(TransformerMixin): |
| 761 | """Produce a reference speech signal by sampling only a few short segments. |
| 762 | |
| 763 | Instead of extracting and running VAD over the entire reference, this samples |
| 764 | ``segment_count`` short windows spread across it, runs VAD on each, and places |
| 765 | the results into a full-length array that is zero everywhere else. That sparse |
| 766 | signal can be fed straight into the normal alignment path: because it preserves |
| 767 | each segment's true position on the reference timeline, the existing |
| 768 | framerate-ratio search and offset cross-correlation recover the same answer |
| 769 | they would from the full signal -- but only the sampled audio has to be |
| 770 | extracted (and, for remote URLs, downloaded). Useful for long / remote |
| 771 | references; for typical local files the plain VideoSpeechTransformer is fine. |
| 772 | """ |
| 773 | |
| 774 | # margins skipped at the very start/end when skip_intro_outro is set, since |
| 775 | # intros/credits often lack dialogue |
| 776 | START_MARGIN_SECONDS: int = 30 |
| 777 | END_MARGIN_SECONDS: int = 60 |
| 778 | |
| 779 | def __init__( |
| 780 | self, |
| 781 | vad: str, |
| 782 | sample_rate: int, |
| 783 | frame_rate: int, |
| 784 | non_speech_label: float, |
| 785 | segment_count: int = 8, |
| 786 | segment_duration: int = 60, |
| 787 | skip_intro_outro: bool = False, |
| 788 | parallel_workers: int = 4, |
| 789 | ffmpeg_path: Optional[str] = None, |
| 790 | ref_stream: Optional[str] = None, |
| 791 | vlc_mode: bool = False, |
| 792 | gui_mode: bool = False, |
| 793 | ) -> None: |
| 794 | super(MultiSegmentVideoSpeechTransformer, self).__init__() |
| 795 | # sampling is audio-only, so drop any "subs_then_" prefix (embedded-subtitle |
| 796 | # extraction ignores the per-segment time window) |
| 797 | self.vad: str = vad.split("subs_then_")[-1] |
| 798 | self.sample_rate: int = sample_rate |
| 799 | self.frame_rate: int = frame_rate |
| 800 | self._non_speech_label: float = non_speech_label |
| 801 | self.segment_count: int = segment_count |
| 802 | self.segment_duration: int = segment_duration |
| 803 | self.skip_intro_outro: bool = skip_intro_outro |
| 804 | self.parallel_workers: int = parallel_workers |
| 805 | self.ffmpeg_path: Optional[str] = ffmpeg_path |
| 806 | self.ref_stream: Optional[str] = ref_stream |
| 807 | self.vlc_mode: bool = vlc_mode |
| 808 | self.gui_mode: bool = gui_mode |
| 809 | self.video_speech_results_: Optional[np.ndarray] = None |
| 810 | |
| 811 | def _segment_starts(self, total_duration: float) -> List[int]: |
| 812 | """Evenly-spaced segment start times (seconds) across the reference.""" |
| 813 | duration = self.segment_duration |
| 814 | if total_duration <= duration: |
| 815 | return [0] |
| 816 | start_margin = self.START_MARGIN_SECONDS if self.skip_intro_outro else 0 |
| 817 | end_margin = self.END_MARGIN_SECONDS if self.skip_intro_outro else 0 |
no outgoing calls