Crawler process manager
| 28 | |
| 29 | |
| 30 | class CrawlerManager: |
| 31 | """Crawler process manager""" |
| 32 | |
| 33 | def __init__(self): |
| 34 | self._lock = asyncio.Lock() |
| 35 | self.process: Optional[subprocess.Popen] = None |
| 36 | self.status = "idle" |
| 37 | self.started_at: Optional[datetime] = None |
| 38 | self.current_config: Optional[CrawlerStartRequest] = None |
| 39 | self._log_id = 0 |
| 40 | self._logs: List[LogEntry] = [] |
| 41 | self._read_task: Optional[asyncio.Task] = None |
| 42 | # Project root directory |
| 43 | self._project_root = Path(__file__).parent.parent.parent |
| 44 | # Log queue - for pushing to WebSocket |
| 45 | self._log_queue: Optional[asyncio.Queue] = None |
| 46 | |
| 47 | @property |
| 48 | def logs(self) -> List[LogEntry]: |
| 49 | return self._logs |
| 50 | |
| 51 | def get_log_queue(self) -> asyncio.Queue: |
| 52 | """Get or create log queue""" |
| 53 | if self._log_queue is None: |
| 54 | self._log_queue = asyncio.Queue() |
| 55 | return self._log_queue |
| 56 | |
| 57 | def _create_log_entry(self, message: str, level: str = "info") -> LogEntry: |
| 58 | """Create log entry""" |
| 59 | self._log_id += 1 |
| 60 | entry = LogEntry( |
| 61 | id=self._log_id, |
| 62 | timestamp=datetime.now().strftime("%H:%M:%S"), |
| 63 | level=level, |
| 64 | message=message |
| 65 | ) |
| 66 | self._logs.append(entry) |
| 67 | # Keep last 500 logs |
| 68 | if len(self._logs) > 500: |
| 69 | self._logs = self._logs[-500:] |
| 70 | return entry |
| 71 | |
| 72 | async def _push_log(self, entry: LogEntry): |
| 73 | """Push log to queue""" |
| 74 | if self._log_queue is not None: |
| 75 | try: |
| 76 | self._log_queue.put_nowait(entry) |
| 77 | except asyncio.QueueFull: |
| 78 | pass |
| 79 | |
| 80 | def _parse_log_level(self, line: str) -> str: |
| 81 | """Parse log level""" |
| 82 | line_upper = line.upper() |
| 83 | if "ERROR" in line_upper or "FAILED" in line_upper: |
| 84 | return "error" |
| 85 | elif "WARNING" in line_upper or "WARN" in line_upper: |
| 86 | return "warning" |
| 87 | elif "SUCCESS" in line_upper or "完成" in line or "成功" in line: |
no outgoing calls