Method _process_files_concurrently

tools/code_indexer.py:1070–1187

Process files concurrently with semaphore limiting

(self, files_to_analyze: list)

        return file_summaries, all_relationships

    async def _process_files_concurrently(self, files_to_analyze: list) -> tuple:
        """Process files concurrently with semaphore limiting"""
        file_summaries = []
        all_relationships = []

        # Create semaphore to limit concurrent tasks
        semaphore = asyncio.Semaphore(self.max_concurrent_files)
        tasks = []

        async def _process_with_semaphore(file_path: Path, index: int, total: int):
            async with semaphore:
                # Add a small delay to space out concurrent requests
                if index > 1:
                    await asyncio.sleep(
                        self.request_delay * 0.5
                    )  # Reduced delay for concurrent processing
                return await self._analyze_single_file_with_relationships(
                    file_path, index, total
                )

        try:
            # Create tasks for all files
            tasks = [
                _process_with_semaphore(file_path, i, len(files_to_analyze))
                for i, file_path in enumerate(files_to_analyze, 1)
            ]

            # Process tasks and collect results
            if self.verbose_output:
                self.logger.info(
                    f"Starting concurrent analysis of {len(tasks)} files..."
                )

            try:
                results = await asyncio.gather(*tasks, return_exceptions=True)

                for i, result in enumerate(results):
                    if isinstance(result, Exception):
                        self.logger.error(
                            f"Failed to analyze file {files_to_analyze[i]}: {result}"
                        )
                        # Create error summary
                        error_summary = FileSummary(
                            file_path=str(
                                files_to_analyze[i].relative_to(self.code_base_path)
                            ),
                            file_type="error",
                            main_functions=[],
                            key_concepts=[],
                            dependencies=[],
                            summary=f"Concurrent analysis failed: {str(result)}",
                            lines_of_code=0,
                            last_modified="",
                        )
                        file_summaries.append(error_summary)
                    else:
                        file_summary, relationships = result
                        file_summaries.append(file_summary)