From f61e2f738170946c5eb7cc97a354246cd51ef294 Mon Sep 17 00:00:00 2001 From: Dym03 Date: Sat, 7 Mar 2026 21:12:39 +0100 Subject: [PATCH] Add worker managed cpu utilization --- .../screens/cluster/worker/cpu_util_table.rs | 98 ++++++++++++++++--- .../ui/screens/cluster/worker/mod.rs | 89 ++++++++++++++--- .../src/dashboard/ui/widgets/progressbar.rs | 19 ++++ 3 files changed, 182 insertions(+), 24 deletions(-) diff --git a/crates/hyperqueue/src/dashboard/ui/screens/cluster/worker/cpu_util_table.rs b/crates/hyperqueue/src/dashboard/ui/screens/cluster/worker/cpu_util_table.rs index fc478cf6f..3601ce7c0 100644 --- a/crates/hyperqueue/src/dashboard/ui/screens/cluster/worker/cpu_util_table.rs +++ b/crates/hyperqueue/src/dashboard/ui/screens/cluster/worker/cpu_util_table.rs @@ -1,14 +1,16 @@ use crate::common::format::human_size; +use crate::dashboard::ui::screens::cluster::worker::UtilizationRenderMode; use ratatui::layout::{Constraint, Rect}; use ratatui::style::Style; use ratatui::widgets::{Cell, Row, Table}; use std::cmp; use tako::hwstats::MemoryStats; +use tako::resources::ResourceIndex; use crate::dashboard::ui::styles; use crate::dashboard::ui::terminal::DashboardFrame; use crate::dashboard::ui::widgets::progressbar::{ - ProgressPrintStyle, get_progress_bar_color, render_progress_bar_at, + ProgressPrintStyle, get_progress_bar_color, get_progress_bar_cpu_color, render_progress_bar_at, }; use crate::dashboard::utils::calculate_average; @@ -19,6 +21,8 @@ const CPU_METER_WIDTH: u8 = CPU_METER_PROGRESSBAR_WIDTH + 4; pub fn render_cpu_util_table( cpu_util_list: &[f64], mem_util: &MemoryStats, + used_cpus: &[ResourceIndex], + util_render_mode: &UtilizationRenderMode, rect: Rect, frame: &mut DashboardFrame, table_style: Style, @@ -31,10 +35,15 @@ pub fn render_cpu_util_table( let width = constraints.len(); let height = (cpu_util_list.len() as f64 / width as f64).ceil() as usize; - let mut rows: Vec> = vec![vec![]; height]; - for (position, &cpu_util) in cpu_util_list.iter().enumerate() { - let row = position % height; - rows[row].push((cpu_util, position)); + let mut rows: Vec> = vec![vec![]; height]; + if *util_render_mode == UtilizationRenderMode::Worker { + rows = get_utilization_sorted_by_usage(cpu_util_list, used_cpus, height) + } else { + for (position, &cpu_util) in cpu_util_list.iter().enumerate() { + let row = position % height; + let used = used_cpus.contains(&ResourceIndex::new(position as u32)); + rows[row].push((cpu_util, position, used)); + } } let rows: Vec = rows @@ -42,27 +51,34 @@ pub fn render_cpu_util_table( .map(|targets| { let columns: Vec = targets .into_iter() - .map(|(cpu_util, position)| { + .map(|(cpu_util, position, used)| { let progress = cpu_util / 100.00; + let style = match util_render_mode { + UtilizationRenderMode::Global => get_progress_bar_color(progress), + UtilizationRenderMode::Worker => get_progress_bar_cpu_color(progress, used), + }; + Cell::from(render_progress_bar_at( Some(format!("{position:>3} ")), progress, CPU_METER_PROGRESSBAR_WIDTH, ProgressPrintStyle::default(), )) - .style(get_progress_bar_color(progress)) + .style(style) }) .collect(); Row::new(columns) }) .collect(); - let avg_cpu = calculate_average(cpu_util_list); - let mem_used = mem_util.total - mem_util.free; + let (which_util, num_cpus, avg_cpu) = + create_title_info(cpu_util_list, used_cpus, util_render_mode); + let title = styles::table_title(format!( - "Worker Utilization ({} CPUs), Avg CPU = {:.0}%, Mem = {:.0}% ({}/{})", - cpu_util_list.len(), + "{} Utilization ({} CPUs), Avg CPU = {:.0}%, Mem = {:.0}% ({}/{})", + which_util, + num_cpus, avg_cpu, (mem_used as f64 / mem_util.total as f64) * 100.0, human_size(mem_used), @@ -89,3 +105,63 @@ fn get_column_constraints(rect: Rect, num_cpus: usize) -> Vec { ) .collect() } + +fn get_utilization_sorted_by_usage( + cpu_util_list: &[f64], + used_cpus: &[ResourceIndex], + height: usize, +) -> Vec> { + let mut all_cpus: Vec<(f64, usize, bool)> = cpu_util_list + .iter() + .enumerate() + .map(|(position, &cpu_util)| { + let used = used_cpus.contains(&ResourceIndex::new(position as u32)); + (cpu_util, position, used) + }) + .collect(); + + all_cpus.sort_by_key(|&(_, _, used)| std::cmp::Reverse(used)); + + let mut rows: Vec> = vec![vec![]; height]; + for (index, cpu_data) in all_cpus.into_iter().enumerate() { + let row = index % height; + rows[row].push(cpu_data); + } + rows +} + +fn create_title_info( + cpu_util_list: &[f64], + used_cpus: &[ResourceIndex], + util_render_mode: &UtilizationRenderMode, +) -> (String, usize, f64) { + let which_util = match util_render_mode { + UtilizationRenderMode::Global => "Node".to_string(), + UtilizationRenderMode::Worker => "Worker".to_string(), + }; + + let num_cpus = match util_render_mode { + UtilizationRenderMode::Global => cpu_util_list.len(), + UtilizationRenderMode::Worker => used_cpus.len(), + }; + + let avg_usage = match util_render_mode { + UtilizationRenderMode::Global => calculate_average(cpu_util_list), + UtilizationRenderMode::Worker => { + let used_cpu_util_list: Vec = cpu_util_list + .iter() + .enumerate() + .filter_map(|(idx, utilization)| { + if used_cpus.contains(&ResourceIndex::new(idx as u32)) { + Some(*utilization) + } else { + None + } + }) + .collect(); + calculate_average(&used_cpu_util_list) + } + }; + + (which_util, num_cpus, avg_usage) +} diff --git a/crates/hyperqueue/src/dashboard/ui/screens/cluster/worker/mod.rs b/crates/hyperqueue/src/dashboard/ui/screens/cluster/worker/mod.rs index d5e608e54..d7fca02b5 100644 --- a/crates/hyperqueue/src/dashboard/ui/screens/cluster/worker/mod.rs +++ b/crates/hyperqueue/src/dashboard/ui/screens/cluster/worker/mod.rs @@ -12,6 +12,7 @@ use crate::dashboard::ui::widgets::text::draw_text; use crossterm::event::{KeyCode, KeyEvent}; use ratatui::layout::{Constraint, Direction, Layout, Rect}; use tako::hwstats::MemoryStats; +use tako::resources::{CPU_RESOURCE_NAME, ResourceIndex}; use tako::{JobTaskId, WorkerId}; mod cpu_util_table; @@ -26,6 +27,7 @@ pub struct WorkerDetail { worker_tasks_table: TasksTable, utilization: Option, + utilization_render_mode: UtilizationRenderMode, } impl Default for WorkerDetail { @@ -36,6 +38,29 @@ impl Default for WorkerDetail { worker_config_table: Default::default(), worker_tasks_table: TasksTable::non_interactive(), utilization: None, + utilization_render_mode: UtilizationRenderMode::Worker, + } + } +} + +#[derive(PartialEq)] +enum UtilizationRenderMode { + Global, + Worker, +} + +impl UtilizationRenderMode { + fn next(&mut self) { + *self = match self { + UtilizationRenderMode::Global => UtilizationRenderMode::Worker, + UtilizationRenderMode::Worker => UtilizationRenderMode::Global, + } + } + + fn next_text(&self) -> &str { + match self { + UtilizationRenderMode::Global => "Show worker CPU utilization", + UtilizationRenderMode::Worker => "Show global CPU utilization", } } } @@ -43,6 +68,7 @@ impl Default for WorkerDetail { struct Utilization { cpu: Vec, memory: MemoryStats, + used_cpus: Vec, } impl WorkerDetail { @@ -66,12 +92,24 @@ impl WorkerDetail { frame, style_header_text(), ); - draw_text(": Back", layout.footer, frame, style_footer()); + + draw_text( + format!( + ": Back, : {}", + self.utilization_render_mode.next_text() + ) + .as_str(), + layout.footer, + frame, + style_footer(), + ); if let Some(util) = &self.utilization { render_cpu_util_table( &util.cpu, &util.memory, + &util.used_cpus, + &self.utilization_render_mode, layout.current_utilization, frame, table_style_deselected(), @@ -95,21 +133,45 @@ impl WorkerDetail { if let Some(worker_id) = self.worker_id { self.utilization_history.update(data, worker_id); - if let Some((cpu_util, mem_util)) = data + if let Some(overview) = data .workers() .query_worker_overview_at(worker_id, data.current_time()) - .and_then(|overview| overview.item.hw_state.as_ref()) - .map(|hw_state| { - ( - &hw_state.state.cpu_usage.cpu_per_core_percent_usage, - &hw_state.state.memory_usage, - ) - }) { - self.utilization = Some(Utilization { - cpu: cpu_util.iter().map(|&v| v as f64).collect(), - memory: mem_util.clone(), - }); + let worker_used_cpus: Vec = match self.utilization_render_mode { + UtilizationRenderMode::Worker => overview + .item + .running_tasks + .iter() + .flat_map(|(_id, task_resource_alloc)| { + task_resource_alloc + .resources + .iter() + .filter_map(|resource_alloc| { + if resource_alloc.resource == CPU_RESOURCE_NAME { + Some(resource_alloc.indices.iter().map(|(index, _)| *index)) + } else { + None + } + }) + }) + .flatten() + .collect(), + UtilizationRenderMode::Global => vec![], + }; + + if let Some(hw_state) = overview.item.hw_state.as_ref() { + self.utilization = Some(Utilization { + cpu: hw_state + .state + .cpu_usage + .cpu_per_core_percent_usage + .iter() + .map(|&v| v as f64) + .collect(), + memory: hw_state.state.memory_usage.clone(), + used_cpus: worker_used_cpus, + }) + } } let tasks_info: Vec<(JobTaskId, &TaskInfo)> = @@ -127,6 +189,7 @@ impl WorkerDetail { pub fn handle_key(&mut self, key: KeyEvent) { match key.code { KeyCode::Backspace => self.worker_tasks_table.clear_selection(), + KeyCode::Char('c') => self.utilization_render_mode.next(), _ => self.worker_tasks_table.handle_key(key), } } diff --git a/crates/hyperqueue/src/dashboard/ui/widgets/progressbar.rs b/crates/hyperqueue/src/dashboard/ui/widgets/progressbar.rs index 4d97126f4..ce5c8e155 100644 --- a/crates/hyperqueue/src/dashboard/ui/widgets/progressbar.rs +++ b/crates/hyperqueue/src/dashboard/ui/widgets/progressbar.rs @@ -30,6 +30,25 @@ pub fn get_progress_bar_color(progress: f64) -> Style { } } +pub fn get_progress_bar_cpu_color(progress: f64, used: bool) -> Style { + let color = if !used { + Color::Gray + } else if progress <= GREEN_THRESHOLD { + Color::Green + } else if progress <= YELLOW_THRESHOLD { + Color::Yellow + } else { + Color::Red + }; + + Style { + fg: Some(color), + bg: None, + add_modifier: Modifier::empty(), + sub_modifier: Modifier::empty(), + } +} + /** * Creates a string progress bar for 0 < progress < 1 */