(
self, uid: str, data: Dict[str, Union[str, np.ndarray]]
)
| 1277 | |
| 1278 | @typechecked |
| 1279 | def _speech_process( |
| 1280 | self, uid: str, data: Dict[str, Union[str, np.ndarray]] |
| 1281 | ) -> Dict[str, Union[str, np.ndarray]]: |
| 1282 | |
| 1283 | if self.speech_name not in data: |
| 1284 | return data |
| 1285 | |
| 1286 | num_spk = self.num_spk |
| 1287 | |
| 1288 | # Add the category information (an integer) to `data` |
| 1289 | if not self.categories and "category" in data: |
| 1290 | raise ValueError( |
| 1291 | "categories must be set in the config file when utt2category files " |
| 1292 | "exist in the data directory (e.g., dump/raw/*/utt2category)" |
| 1293 | ) |
| 1294 | |
| 1295 | # Add the sampling rate information (an integer) to `data` |
| 1296 | if "fs" in data: |
| 1297 | fs = int(data.pop("fs")) |
| 1298 | data["utt2fs"] = np.array([fs]) |
| 1299 | else: |
| 1300 | fs = self.sample_rate |
| 1301 | |
| 1302 | sref_name = self.speech_ref_name_prefix + "1" |
| 1303 | if self.flexible_numspk and sref_name in data: |
| 1304 | # The number of speaker varies in each sample. |
| 1305 | # Different speaker signals are stacked in the first dimension. |
| 1306 | dref_name = self.dereverb_ref_name_prefix + "1" |
| 1307 | num_spk = len(data[sref_name]) |
| 1308 | for i in range(2, self.num_spk + 1): |
| 1309 | data.pop(self.speech_ref_name_prefix + str(i), None) |
| 1310 | data.pop(self.dereverb_ref_name_prefix + str(i), None) |
| 1311 | # Divide the stacked signals into single speaker signals for consistency |
| 1312 | for i in range(num_spk - 1, -1, -1): |
| 1313 | idx = str(i + 1) |
| 1314 | # make sure no np.nan paddings are in the data |
| 1315 | assert not np.isnan(np.sum(data[sref_name][i])), uid |
| 1316 | data[self.speech_ref_name_prefix + idx] = data[sref_name][i] |
| 1317 | if dref_name in data: |
| 1318 | # make sure no np.nan paddings are in the data |
| 1319 | assert not np.isnan(np.sum(data[dref_name][i])), uid |
| 1320 | data[self.dereverb_ref_name_prefix + idx] = data[dref_name][i] |
| 1321 | |
| 1322 | if self.train: |
| 1323 | if self.speech_segment is not None: |
| 1324 | speech_segment = self.speech_segment // self.sample_rate * fs |
| 1325 | start, end = self._random_crop_range( |
| 1326 | data, num_spk, speech_segment, uid=uid |
| 1327 | ) |
| 1328 | self._apply_to_all_signals(data, lambda x: x[start:end], num_spk) |
| 1329 | # clean speech signal (Nmic, Time) |
| 1330 | speech_ref = [ |
| 1331 | self._ensure_2d(data[self.speech_ref_name_prefix + str(i + 1)]) |
| 1332 | for i in range(num_spk) |
| 1333 | ] |
| 1334 | |
| 1335 | # dereverberated (noisy) signal (Nmic, Time) |
| 1336 | if self.dereverb_ref_name_prefix + "1" in data: |
no test coverage detected