Download and process all URLs concurrently. Returns dict mapping original_url -> resolved_local_path. Raises exception immediately if any download fails.
(
unique_urls: dict[str, str], cache_manager: PlatformIOCache
)
| 920 | |
| 921 | |
| 922 | def _download_and_process_urls( |
| 923 | unique_urls: dict[str, str], cache_manager: PlatformIOCache |
| 924 | ) -> dict[str, str]: |
| 925 | """ |
| 926 | Download and process all URLs concurrently. |
| 927 | Returns dict mapping original_url -> resolved_local_path. |
| 928 | Raises exception immediately if any download fails. |
| 929 | """ |
| 930 | if not unique_urls: |
| 931 | return {} |
| 932 | |
| 933 | max_workers = min(4, len(unique_urls)) # Don't create more threads than needed |
| 934 | with ThreadPoolExecutor( |
| 935 | max_workers=max_workers, thread_name_prefix="download" |
| 936 | ) as executor: |
| 937 | # Submit all download tasks |
| 938 | futures: list[Future[ArtifactProcessingResult]] = [] |
| 939 | for url, env_section in unique_urls.items(): |
| 940 | future = executor.submit(_process_artifact, url, env_section, cache_manager) |
| 941 | futures.append(future) |
| 942 | |
| 943 | replacements: dict[str, str] = {} |
| 944 | |
| 945 | try: |
| 946 | for future in as_completed(futures): |
| 947 | try: |
| 948 | result = future.result() |
| 949 | if result.success and result.resolved_path is not None: |
| 950 | replacements[result.url] = result.resolved_path |
| 951 | print(f"✅ Resolved {result.url} -> {result.resolved_path}") |
| 952 | else: |
| 953 | logger.error( |
| 954 | f"❌ Failed to process {result.url}: {result.exception}" |
| 955 | ) |
| 956 | # Re-raise all exceptions - no downloads should fail silently |
| 957 | if result.exception: |
| 958 | raise result.exception |
| 959 | except KeyboardInterrupt as ki: |
| 960 | handle_keyboard_interrupt(ki) |
| 961 | raise |
| 962 | except Exception as e: |
| 963 | logger.error( |
| 964 | f"Future failed with unexpected error: {e}", exc_info=e |
| 965 | ) |
| 966 | raise # Re-raise unexpected exceptions |
| 967 | except KeyboardInterrupt as ki: |
| 968 | handle_keyboard_interrupt(ki) |
| 969 | raise |
| 970 | logger.warning("Processing interrupted, cancelling remaining downloads...") |
| 971 | _global_cancel_event.set() |
| 972 | # The context manager will handle cleanup of the executor |
| 973 | raise |
| 974 | |
| 975 | return replacements |
| 976 | |
| 977 | |
| 978 | def _replace_all_urls( |
no test coverage detected