Process transcript files in parallel. Args: input_files: List of transcript file paths num_processes: Number of processes to use (default: system CPU count) Returns: List of transcript metadata dictionaries
(
input_files: FileList, num_processes: Optional[int] = None
)
| 255 | |
| 256 | |
| 257 | def process_transcripts_parallel( |
| 258 | input_files: FileList, num_processes: Optional[int] = None |
| 259 | ) -> List[TranscriptDict]: |
| 260 | """ |
| 261 | Process transcript files in parallel. |
| 262 | |
| 263 | Args: |
| 264 | input_files: List of transcript file paths |
| 265 | num_processes: Number of processes to use (default: system CPU count) |
| 266 | |
| 267 | Returns: |
| 268 | List of transcript metadata dictionaries |
| 269 | """ |
| 270 | if num_processes is None: |
| 271 | num_processes = multiprocessing.cpu_count() |
| 272 | |
| 273 | print(f"Processing {len(input_files)} files using {num_processes} processes") |
| 274 | |
| 275 | with multiprocessing.Pool(processes=num_processes) as pool: |
| 276 | transcript_dicts = list( |
| 277 | tqdm( |
| 278 | pool.imap_unordered(parallel_process_file, input_files), |
| 279 | total=len(input_files), |
| 280 | desc="Processing transcripts", |
| 281 | ) |
| 282 | ) |
| 283 | |
| 284 | return transcript_dicts |
| 285 | |
| 286 | |
| 287 | def text_to_jsonl( |