Function render_node

benchmarks/src/postprocessing/monitor.py:579–639

(node: Node)

def render_process_resource_usage(report: ClusterReport, per_process_df: pd.DataFrame):
    def render_node(node: Node) -> Optional[Model]:
        node_data = per_process_df[per_process_df[HOSTNAME_KEY] == node.hostname]
        pids = sorted(node_data[PID_KEY].unique())

        def render_process(pid: int) -> Optional[Model]:
            process_data = node_data[node_data[PID_KEY] == pid]
            max_rss = process_data[RSS_KEY].max()
            avg_cpu = process_data[AVG_CPU_KEY].mean()
            process = [p for p in report.cluster.nodes[node.hostname].processes if p.pid == pid]
            if not process:
                logging.warning(f"Process {node.hostname}/{pid} not found in cluster")
                return
            process = process[0]

            range = get_time_range(process_data[DATETIME_KEY])

            mib_divisor = 1024.0 * 1024

            time = process_data[DATETIME_KEY]
            mem = resample(process_data[RSS_KEY] / mib_divisor, time)
            mem_figure = prepare_time_range_figure(range)
            mem_figure.yaxis[0].formatter = NumeralTickFormatter()
            mem_figure.yaxis.axis_label = "RSS (MiB)"
            mem_figure.y_range = Range1d(0, max_rss / mib_divisor + 100.0)
            mem_figure.add_tools(HoverTool(tooltips=[("RSS (MiB)", "@rss")]))

            data = ColumnDataSource(dict(rss=mem, x=mem.index))
            mem_figure.line(x="x", y="rss", color="red", legend_label="Memory", source=data)

            cpu = resample(process_data[AVG_CPU_KEY], time)
            cpu_figure = prepare_time_range_figure(range)
            cpu_figure.yaxis[0].formatter = NumeralTickFormatter(format="0 %")
            cpu_figure.yaxis.axis_label = "Avg. CPU usage (%)"
            cpu_figure.y_range = Range1d(0, 1)
            cpu_figure.add_tools(HoverTool(tooltips=[("Avg. CPU usage", "@cpu")]))

            data = ColumnDataSource(dict(cpu=cpu / 100.0, x=cpu.index))
            cpu_figure.line(x="x", y="cpu", color="blue", legend_label="CPU usage (%)", source=data)

            cpu_times = [
                ("User CPU time", PROCESS_CPU_TIME_USER_KEY),
                ("System CPU time", PROCESS_CPU_TIME_SYSTEM_KEY),
                ("User + children CPU time", PROCESS_CPU_TIME_USER_CHILDREN_KEY),
                ("System + children CPU time", PROCESS_CPU_TIME_SYSTEM_CHILDREN_KEY),
            ]
            cpu_time_figures = [render_process_cpu_time(process_data, key, time, label) for (label, key) in cpu_times]

            summary = PreText(
                text=f"""
PID: {pid}
Key: {process.key}
Max. RSS: {humanize.naturalsize(max_rss, binary=True)}
Avg. CPU: {avg_cpu:.02f} %
""".strip()
            )

            right_col = Column(children=cpu_time_figures)
            left_col = Column(children=[summary, mem_figure, cpu_figure])