(self, vid, clip)
| 67 | self.dst_lmdb_env.close() |
| 68 | |
| 69 | def _sample_from_clip(self, vid, clip): |
| 70 | clip_skeleton = clip['skeletons_3d'] |
| 71 | clip_audio = clip['audio_feat'] |
| 72 | clip_audio_raw = clip['audio_raw'] |
| 73 | clip_word_list = clip['words'] |
| 74 | clip_s_f, clip_e_f = clip['start_frame_no'], clip['end_frame_no'] |
| 75 | clip_s_t, clip_e_t = clip['start_time'], clip['end_time'] |
| 76 | |
| 77 | n_filtered_out = defaultdict(int) |
| 78 | |
| 79 | # skeleton resampling |
| 80 | clip_skeleton = utils.data_utils.resample_pose_seq(clip_skeleton, clip_e_t - clip_s_t, self.skeleton_resampling_fps) |
| 81 | |
| 82 | # divide |
| 83 | aux_info = [] |
| 84 | sample_skeletons_list = [] |
| 85 | sample_words_list = [] |
| 86 | sample_audio_list = [] |
| 87 | sample_spectrogram_list = [] |
| 88 | |
| 89 | num_subdivision = math.floor( |
| 90 | (len(clip_skeleton) - self.n_poses) |
| 91 | / self.subdivision_stride) + 1 # floor((K - (N+M)) / S) + 1 |
| 92 | expected_audio_length = utils.data_utils.calc_spectrogram_length_from_motion_length(len(clip_skeleton), self.skeleton_resampling_fps) |
| 93 | assert abs(expected_audio_length - clip_audio.shape[1]) <= 5, 'audio and skeleton lengths are different' |
| 94 | |
| 95 | for i in range(num_subdivision): |
| 96 | start_idx = i * self.subdivision_stride |
| 97 | fin_idx = start_idx + self.n_poses |
| 98 | |
| 99 | sample_skeletons = clip_skeleton[start_idx:fin_idx] |
| 100 | subdivision_start_time = clip_s_t + start_idx / self.skeleton_resampling_fps |
| 101 | subdivision_end_time = clip_s_t + fin_idx / self.skeleton_resampling_fps |
| 102 | sample_words = self.get_words_in_time_range(word_list=clip_word_list, |
| 103 | start_time=subdivision_start_time, |
| 104 | end_time=subdivision_end_time) |
| 105 | |
| 106 | # spectrogram |
| 107 | audio_start = math.floor(start_idx / len(clip_skeleton) * clip_audio.shape[1]) |
| 108 | audio_end = audio_start + self.spectrogram_sample_length |
| 109 | if audio_end > clip_audio.shape[1]: # correct size mismatch between poses and audio |
| 110 | # logging.info('expanding audio array, audio start={}, end={}, clip_length={}'.format( |
| 111 | # audio_start, audio_end, clip_audio.shape[1])) |
| 112 | n_padding = audio_end - clip_audio.shape[1] |
| 113 | padded_data = np.pad(clip_audio, ((0, 0), (0, n_padding)), mode='symmetric') |
| 114 | sample_spectrogram = padded_data[:, audio_start:audio_end] |
| 115 | else: |
| 116 | sample_spectrogram = clip_audio[:, audio_start:audio_end] |
| 117 | |
| 118 | # raw audio |
| 119 | audio_start = math.floor(start_idx / len(clip_skeleton) * len(clip_audio_raw)) |
| 120 | audio_end = audio_start + self.audio_sample_length |
| 121 | if audio_end > len(clip_audio_raw): # correct size mismatch between poses and audio |
| 122 | # logging.info('expanding audio array, audio start={}, end={}, clip_length={}'.format( |
| 123 | # audio_start, audio_end, len(clip_audio_raw))) |
| 124 | n_padding = audio_end - len(clip_audio_raw) |
| 125 | padded_data = np.pad(clip_audio_raw, (0, n_padding), mode='symmetric') |
| 126 | sample_audio = padded_data[audio_start:audio_end] |
no test coverage detected