| 648 | return total |
| 649 | |
| 650 | def modules_status(self, _log=False): |
| 651 | finished = True |
| 652 | status = {"modules": {}} |
| 653 | |
| 654 | sorted_modules = [] |
| 655 | for module_name, module in self.modules.items(): |
| 656 | if module_name.startswith("_"): |
| 657 | continue |
| 658 | sorted_modules.append(module) |
| 659 | mod_status = module.status |
| 660 | if mod_status["running"]: |
| 661 | finished = False |
| 662 | status["modules"][module_name] = mod_status |
| 663 | |
| 664 | # sort modules by name |
| 665 | sorted_modules.sort(key=lambda m: m.name) |
| 666 | |
| 667 | status["finished"] = finished |
| 668 | |
| 669 | modules_errored = [m for m, s in status["modules"].items() if s["errored"]] |
| 670 | |
| 671 | max_mem_percent = 90 |
| 672 | mem_status = self.helpers.memory_status() |
| 673 | # abort if we don't have the memory |
| 674 | mem_percent = mem_status.percent |
| 675 | if mem_percent > max_mem_percent: |
| 676 | free_memory = mem_status.available |
| 677 | free_memory_human = self.helpers.bytes_to_human(free_memory) |
| 678 | self.warning(f"System memory is at {mem_percent:.1f}% ({free_memory_human} remaining)") |
| 679 | |
| 680 | if _log: |
| 681 | modules_status = [] |
| 682 | for m, s in status["modules"].items(): |
| 683 | running = s["running"] |
| 684 | incoming = s["events"]["incoming"] |
| 685 | outgoing = s["events"]["outgoing"] |
| 686 | tasks = s["tasks"] |
| 687 | total = sum([incoming, outgoing, tasks]) |
| 688 | if running or total > 0: |
| 689 | modules_status.append((m, running, incoming, outgoing, tasks, total)) |
| 690 | modules_status.sort(key=lambda x: x[-1], reverse=True) |
| 691 | |
| 692 | if modules_status: |
| 693 | modules_status_str = ", ".join([f"{m}({i:,}:{t:,}:{o:,})" for m, r, i, o, t, _ in modules_status]) |
| 694 | self.info(f"{self.name}: Modules running (incoming:processing:outgoing) {modules_status_str}") |
| 695 | else: |
| 696 | self.info(f"{self.name}: No modules running") |
| 697 | event_type_summary = sorted(self.stats.events_emitted_by_type.items(), key=lambda x: x[-1], reverse=True) |
| 698 | if event_type_summary: |
| 699 | self.info( |
| 700 | f"{self.name}: Events produced so far: {', '.join([f'{k}: {v}' for k, v in event_type_summary])}" |
| 701 | ) |
| 702 | else: |
| 703 | self.info(f"{self.name}: No events produced yet") |
| 704 | |
| 705 | if modules_errored: |
| 706 | self.verbose( |
| 707 | f"{self.name}: Modules errored: {len(modules_errored):,} ({', '.join(list(modules_errored))})" |