| 37 | mel_first: bool = True, |
| 38 | ): |
| 39 | class AudioProcessor: |
| 40 | def __init__( |
| 41 | self, |
| 42 | clip_duration: float = 2, |
| 43 | clips_per_audio: int = 1, |
| 44 | num_mel_bins: int = 128, |
| 45 | max_frames: int = 204, |
| 46 | mel_first: bool = True, |
| 47 | mean=-4.268, |
| 48 | std=9.138, |
| 49 | ): |
| 50 | self.clip_sampler = ConstantClipsPerVideoSampler( |
| 51 | clip_duration=clip_duration, clips_per_video=clips_per_audio |
| 52 | ) |
| 53 | self.normalize = Normalize(mean=mean, std=std) |
| 54 | self.num_mel_bins = num_mel_bins |
| 55 | self.max_frames = max_frames |
| 56 | self.mel_first = mel_first |
| 57 | |
| 58 | def waveform2melspec(self, waveform, sample_rate, num_mel_bins, max_frames): |
| 59 | # Based on https://github.com/YuanGongND/ast/blob/d7d8b4b8e06cdaeb6c843cdb38794c1c7692234c/src/dataloader.py#L102 |
| 60 | waveform -= waveform.mean() |
| 61 | fbank = torchaudio.compliance.kaldi.fbank( |
| 62 | waveform, |
| 63 | htk_compat=True, |
| 64 | sample_frequency=sample_rate, |
| 65 | use_energy=False, |
| 66 | window_type='hanning', |
| 67 | num_mel_bins=num_mel_bins, |
| 68 | dither=0.0, |
| 69 | frame_length=25, |
| 70 | frame_shift=DEFAULT_AUDIO_FRAME_SHIFT_MS, |
| 71 | ) |
| 72 | # Convert to [mel_bins, num_frames] shape |
| 73 | fbank = fbank.transpose(0, 1) |
| 74 | # Pad to target_length |
| 75 | n_frames = fbank.size(1) |
| 76 | p = max_frames - n_frames |
| 77 | # cut and pad |
| 78 | if p > 0: |
| 79 | fbank = torch.nn.functional.pad(fbank, (0, p), mode='constant', value=0) |
| 80 | elif p < 0: |
| 81 | fbank = fbank[:, 0:max_frames] |
| 82 | # Convert to [1, mel_bins, num_frames] shape, essentially like a 1 |
| 83 | # channel image |
| 84 | fbank = fbank.unsqueeze(0) |
| 85 | return fbank |
| 86 | |
| 87 | def get_clip_timepoints(self, duration): |
| 88 | # Read out all clips |
| 89 | all_clips_timepoints = [] |
| 90 | is_last_clip = False |
| 91 | end = 0.0 |
| 92 | while not is_last_clip: |
| 93 | start, end, _, _, is_last_clip = self.clip_sampler(end, duration, annotation=None) |
| 94 | all_clips_timepoints.append((start, end)) |
| 95 | return all_clips_timepoints |
| 96 |
no outgoing calls
no test coverage detected