perf: speed up filtering and graph data preparation for large datasets (#364)

.claude/skills/js-bundling/SKILL.md

@@ -90,7 +90,6 @@ Current third-party libraries:
 | @sgratzl/chartjs-chart-boxplot | Box plot chart type |
 | chartjs-chart-matrix | Matrix/heatmap chart type |
 | GridStack | Drag-and-drop dashboard layout |
-| Pako | `pako.inflate()` — decompress embedded data in the browser |
 | Bootstrap | Modals, layout, admin page UI |
 | DataTables (3.x, no jQuery) | Tables page, admin page tables |
 
@@ -128,8 +127,8 @@ json.dumps(data).encode("utf-8")  # serialize
 **JavaScript (browser):**
 ```javascript
 // js/variables/data.js
-const runs = decode_and_decompress("placeholder_runs");
-// atob() → Uint8Array → pako.inflate() → JSON.parse()
+await load_data();   // main() awaits this before anything reads runs/suites/tests/...
+// atob() → Uint8Array → native DecompressionStream("deflate") → Response.json()
 ```
 
-Pako (the JS zlib port) is the only dependency required to decode data, so it must always be loaded before `data.js` runs.
+Decoding needs no library: the browser's `DecompressionStream` inflates the zlib payload. It is async, so the data arrays are empty `let` bindings until `load_data()` resolves — never read them at module load time, only from code that runs after `main()` awaited it.

docs/advanced-cli-examples.md

@@ -209,7 +209,6 @@ The following dependencies are bundled with the package and will be used offline
 | **datatables_css** | CSS | https://cdn.datatables.net/v/bs5/dt-3.0.4/datatables.min.css | Table component styling |
 | **bootstrap_js** | JS | https://cdnjs.cloudflare.com/ajax/libs/twitter-bootstrap/5.3.8/js/bootstrap.bundle.min.js | Bootstrap interactive components |
 | **datatables_js** | JS | https://cdn.datatables.net/v/bs5/dt-3.0.4/datatables.min.js | DataTables functionality |
-| **pako** | JS | https://cdn.jsdelivr.net/npm/pako@3.0.2/dist/browser/pako_inflate.umd.min.js | Compression/decompression used for smaller dashboard file sizes |
 
 ### Behaviour
 

robotframework_dashboard/dependencies.py

@@ -73,12 +73,6 @@
         "local": "dependencies/datatables.js",
         "admin_page": True,
     },
-    "pako": {
-        "type": "js",
-        "cdn": "https://cdn.jsdelivr.net/npm/pako@3.0.2/dist/browser/pako_inflate.umd.min.js",
-        "local": "dependencies/pako.js",
-        "admin_page": False,
-    },
 }
 
 

robotframework_dashboard/js/filter/availability.js

@@ -1,6 +1,5 @@
 import { settings } from '../variables/settings.js';
-import { runs, suites } from '../variables/data.js';
-import { apply_custom_filter_dimension, apply_date_filter, apply_metadata_filter, apply_project_version_filter, apply_run_name_filter, apply_runtag_filter, build_date_range, collect_custom_filter_dimensions, convert_timezone, get_custom_filter_value, get_hidden_custom_filters, get_project_version_value, remove_milliseconds, remove_timezones } from './pipeline.js';
+import { apply_custom_filter_dimension, apply_date_filter, apply_metadata_filter, apply_project_version_filter, apply_run_name_filter, apply_runtag_filter, build_date_range, collect_custom_filter_dimensions, get_custom_filter_value, get_hidden_custom_filters, get_project_version_value, get_transformed_data } from './pipeline.js';
 import { get_metadata_options } from './modal_options.js';
 import { apply_suite_path_run_filter } from './suite_path.js';
 import { capture_current_filters } from './profiles.js';
@@ -11,36 +10,6 @@ import { capture_current_filters } from './profiles.js';
 // The counts follow the usual faceted-search rule: the count of an option in filter X is
 // computed with every filter except X applied, so selecting a value in X never makes the other
 // values of X disappear. The amount filter ("last X runs") is not a category and is left out.
-let filterBaseRunsCache = { key: null, runs: null, suites: null };
-
-// key of the settings that change run_start representation, and with it which runs a date
-// range matches and which suites belong to which run
-function get_filter_base_cache_key() {
-    return `${settings.show.milliseconds}|${settings.show.convertTimezone}|${settings.show.timezones}`;
-}
-
-function apply_run_start_transformations(data) {
-    return remove_timezones(convert_timezone(remove_milliseconds(data)));
-}
-
-// all runs with the same run_start transformations the filter pipeline applies, so the
-// availability computation compares the same timestamps as the real filters do
-function get_filter_base_runs() {
-    const key = get_filter_base_cache_key();
-    if (filterBaseRunsCache.key !== key) {
-        filterBaseRunsCache = { key: key, runs: apply_run_start_transformations(runs), suites: null };
-    }
-    return filterBaseRunsCache.runs;
-}
-
-// suites are only needed while a suite path is selected, so they are transformed on demand
-function get_filter_base_suites() {
-    get_filter_base_runs();
-    if (filterBaseRunsCache.suites === null) {
-        filterBaseRunsCache.suites = apply_run_start_transformations(suites);
-    }
-    return filterBaseRunsCache.suites;
-}
 
 // turn the profile object of capture_current_filters() into the selection shape the apply_*
 // functions take
@@ -85,7 +54,7 @@ function apply_filters_except(runList, selections, facet, dimName = null, suiteL
         result = apply_date_filter(result, selections.dateRange.from, selections.dateRange.to);
     }
     if (selections.suitePath && selections.suitePath !== "All") {
-        result = apply_suite_path_run_filter(result, selections.suitePath, suiteList ?? get_filter_base_suites());
+        result = apply_suite_path_run_filter(result, selections.suitePath, suiteList ?? get_transformed_data("suites"));
     }
     return result;
 }
@@ -164,7 +133,7 @@ function compute_filter_option_availability(runList, selections, suiteList = nul
 // amount filter is left out for the same reason it is left out of the option counts.
 function get_runs_for_date_histogram() {
     const selections = normalize_filter_selections(capture_current_filters());
-    return apply_filters_except(get_filter_base_runs(), selections, "dates");
+    return apply_filters_except(get_transformed_data("runs"), selections, "dates");
 }
 
 // add or update the "(X)" count of one filter option row. The count is a sibling of the
@@ -237,7 +206,7 @@ function refresh_filter_option_availability() {
         return;
     }
     const selections = normalize_filter_selections(capture_current_filters());
-    const availability = compute_filter_option_availability(get_filter_base_runs(), selections);
+    const availability = compute_filter_option_availability(get_transformed_data("runs"), selections);
     apply_availability_to_select(document.getElementById("runs"), availability.runs);
     apply_availability_to_select(document.getElementById("metadata"), availability.metadata);
     apply_availability_to_checkbox_list(document.getElementById("runTag"), availability.runTags);

robotframework_dashboard/js/filter/pipeline.js

@@ -8,11 +8,26 @@ import { filter_runs_by_suite_path, filter_suite_path_data } from './suite_path.
 // Sort an array of run objects by wall-clock run_start (timezone offset stripped),
 // ensuring correct chronological order when timestamps span mixed timezone offsets.
 function sort_wall_clock(data) {
-    return [...data].sort((a, b) => {
-        const ak = strip_tz_suffix(a.run_start);
-        const bk = strip_tz_suffix(b.run_start);
-        return ak < bk ? -1 : ak > bk ? 1 : 0;
-    });
+    return data
+        .map((item, index) => ({ item, index, key: strip_tz_suffix(item.run_start) }))
+        .sort((a, b) => a.key < b.key ? -1 : a.key > b.key ? 1 : a.index - b.index)
+        .map(entry => entry.item);
+}
+
+// the run_start transformations only depend on these settings, so they are cached per combination
+// instead of copying every row on each filter apply; the filter availability uses the same copies
+let transformedDataCache = { key: null, data: {} };
+
+function get_transformed_data(name) {
+    const key = `${settings.show.milliseconds}|${settings.show.convertTimezone}|${settings.show.timezones}`;
+    if (transformedDataCache.key !== key) {
+        transformedDataCache = { key: key, data: {} };
+    }
+    if (!(name in transformedDataCache.data)) {
+        const sourceData = { runs, suites, tests, keywords, exceptions }[name];
+        transformedDataCache.data[name] = remove_timezones(convert_timezone(remove_milliseconds(sourceData)));
+    }
+    return transformedDataCache.data[name];
 }
 
 const dashboardPages = ["overview", "dashboard", "compare", "tables"];
@@ -38,23 +53,11 @@ function apply_custom_filter_visibility() {
 // function updates the data in the graphs whenever filters are updated
 function setup_filtered_data_and_filters() {
     apply_custom_filter_visibility();
-    filteredRuns = remove_milliseconds(runs)
-    filteredSuites = remove_milliseconds(suites)
-    filteredTests = remove_milliseconds(tests)
-    filteredKeywords = remove_milliseconds(keywords)
-    filteredExceptions = remove_milliseconds(exceptions)
-    // convert timezones if enabled (must run before remove_timezones so the offset is still present)
-    filteredRuns = convert_timezone(filteredRuns);
-    filteredSuites = convert_timezone(filteredSuites);
-    filteredTests = convert_timezone(filteredTests);
-    filteredKeywords = convert_timezone(filteredKeywords);
-    filteredExceptions = convert_timezone(filteredExceptions);
-    // remove timezone display if disabled
-    filteredRuns = remove_timezones(filteredRuns);
-    filteredSuites = remove_timezones(filteredSuites);
-    filteredTests = remove_timezones(filteredTests);
-    filteredKeywords = remove_timezones(filteredKeywords);
-    filteredExceptions = remove_timezones(filteredExceptions);
+    filteredRuns = get_transformed_data("runs");
+    filteredSuites = get_transformed_data("suites");
+    filteredTests = get_transformed_data("tests");
+    filteredKeywords = get_transformed_data("keywords");
+    filteredExceptions = get_transformed_data("exceptions");
     // determine filteredRuns with all run-level filters (suite path + amount last)
     filteredRuns = filter_runs(filteredRuns);
     filteredRuns = filter_runtags(filteredRuns);
@@ -437,8 +440,8 @@ function filter_metadata(filteredRuns) {
 
 // function to filter suites/tests/keywords based on the already filtered runs
 function filter_data(data) {
-    const validRunStarts = filteredRuns.map(v => v.run_start);
-    let filteredData = data.filter(v => validRunStarts.includes(v.run_start));
+    const validRunStarts = new Set(filteredRuns.map(v => v.run_start));
+    let filteredData = data.filter(v => validRunStarts.has(v.run_start));
     if (filteredData.length > 0 && "owner" in filteredData[0]) {
         const libraries = settings.libraries || {};
         filteredData = filteredData.filter(item => {
@@ -470,8 +473,10 @@ export {
     get_hidden_custom_filters,
     get_project_version_value,
     get_run_start_date,
+    get_transformed_data,
     parse_custom_filters,
     remove_milliseconds,
     remove_timezones,
     setup_filtered_data_and_filters,
+    sort_wall_clock,
 };

robotframework_dashboard/js/filter/section_selects.js

@@ -103,20 +103,19 @@ function setup_tests_in_select() {
     const testTagsSelect = document.getElementById("testTagsSelect").value;
     const testSelect = document.getElementById("testSelect");
     testSelect.innerHTML = "";
-    const testNames = filteredTests.reduce((names, test) => {
+    const testNames = new Set();
+    for (const test of filteredTests) {
+        if (testNames.has(test.name)) continue;
         const isInSuite = settings.switch.suitePathsTestSection
             ? test.full_name.includes(`${suiteSelectTests}.${test.name}`) || suiteSelectTests === "All"
             : test.full_name.includes(`.${suiteSelectTests}.${test.name}`) || suiteSelectTests === "All"
         const hasTag = testTagsSelect === "All" || test.tags.includes(testTagsSelect);
-
-        if (isInSuite && hasTag && !names.includes(test.name)) {
-            names.push(test.name);
+        if (isInSuite && hasTag) {
+            testNames.add(test.name);
         }
-
-        return names;
-    }, []);
+    }
     testSelect.options.add(new Option("All", "All"));
-    testNames.sort().forEach(testName => testSelect.options.add(new Option(testName, testName)));
+    [...testNames].sort().forEach(testName => testSelect.options.add(new Option(testName, testName)));
 }
 
 // function to update the available testtags to select in the filters

robotframework_dashboard/js/graph_creation/overview.js

@@ -449,15 +449,13 @@ function create_project_cards_container(projectName, projectRuns, percent = null
     }
 
     const container = document.getElementById(`${projectName}RunCardsContainer`);
+    destroy_overview_donuts(container);
     container.innerHTML = '';
     const projectRunsToShow = projectRuns.slice().reverse();
-    // create cards and charts for each run card
+    // create cards and charts for each run card, create_project_run_card also creates the donut
     projectRunsToShow.forEach((run, idx) => {
         const runNumber = projectRunsToShow.length - idx;
-        const createdRunCardId = create_project_run_card(run, projectName, idx, runNumber, passRate, percent, durations, false);
-        const createdRunCard = document.getElementById(createdRunCardId);
-        container.appendChild(createdRunCard);
-        create_overview_run_donut(run, idx, projectName);
+        create_project_run_card(run, projectName, idx, runNumber, passRate, percent, durations, false);
     });
 }
 
@@ -467,6 +465,7 @@ function create_overview_latest_graphs(preFilteredRuns = null) {
     if (!orderEl) return;
     const order = orderEl.value;
     const overviewCardsContainer = document.getElementById("overviewLatestRunCardsContainer");
+    destroy_overview_donuts(overviewCardsContainer);
     overviewCardsContainer.innerHTML = '';
     const allProjects = { ...projects_by_name, ...projects_by_tag };
     const durationsByProject = {};
@@ -527,6 +526,7 @@ function create_overview_latest_graphs(preFilteredRuns = null) {
 function create_overview_total_graphs(preFilteredRuns = null) {
     const overviewCardsContainer = document.getElementById("overviewTotalRunCardsContainer");
     if (!overviewCardsContainer) return;
+    destroy_overview_donuts(overviewCardsContainer);
     overviewCardsContainer.innerHTML = '';
     const allProjects = { ...projects_by_name, ...projects_by_tag };
     const durationsByProject = {};
@@ -621,6 +621,7 @@ function create_project_run_card(run, projectName, runIndex, runNumber, passRate
     )
     const existingRunCard = document.getElementById(`${projectNameForId}Card${runIndex}`);
     if (existingRunCard) {
+        destroy_overview_donuts(existingRunCard);
         // preserves listeners of element
         existingRunCard.replaceWith(document.createRange().createContextualFragment(projectRunCardHTML));
     } else {
@@ -660,7 +661,7 @@ function create_overview_run_donut(run, chartElementPostfix, projectName) {
         );
         return;
     }
-    if (el.chartInstance) el.chartInstance.destroy();
+    destroy_overview_donut(el);
     const chartData = {
         labels: [],
         datasets: [{
@@ -684,7 +685,44 @@ function create_overview_run_donut(run, chartElementPostfix, projectName) {
     const config = get_graph_config('donut', chartData, 'Run Status');
     delete config.options.plugins.datalabels;
     config.options.plugins.legend.display = false;
-    el.chartInstance = new Chart(el, config);
+    // the run cards can hold hundreds of donuts of which only a few are on screen, so a donut
+    // is only created once its card comes near the viewport
+    el.pendingChartConfig = config;
+    get_lazy_donut_observer().observe(el);
+}
+
+let lazyDonutObserver = null;
+
+function get_lazy_donut_observer() {
+    if (!lazyDonutObserver) {
+        lazyDonutObserver = new IntersectionObserver(entries => {
+            for (const entry of entries) {
+                if (!entry.isIntersecting) continue;
+                const el = entry.target;
+                lazyDonutObserver.unobserve(el);
+                if (el.pendingChartConfig) {
+                    el.chartInstance = new Chart(el, el.pendingChartConfig);
+                    delete el.pendingChartConfig;
+                }
+            }
+        }, { rootMargin: "500px" });
+    }
+    return lazyDonutObserver;
+}
+
+function destroy_overview_donut(el) {
+    lazyDonutObserver?.unobserve(el);
+    delete el.pendingChartConfig;
+    if (el.chartInstance) {
+        el.chartInstance.destroy();
+        delete el.chartInstance;
+    }
+}
+
+// Chart.js keeps a reference to every chart until it is destroyed, so the donuts have to be
+// destroyed before their cards are removed from the page
+function destroy_overview_donuts(container) {
+    container.querySelectorAll(".overview-canvas canvas").forEach(destroy_overview_donut);
 }
 
 function update_overview_latest_heading() {

robotframework_dashboard/js/graph_creation/suite.js

@@ -3,7 +3,7 @@ import { get_statistics_graph_data } from '../graph_data/statistics.js';
 import { get_duration_graph_data } from '../graph_data/duration.js';
 import { get_graph_config } from '../graph_data/graph_config.js';
 import { build_tooltip_meta, lookup_tooltip_meta, format_status } from '../graph_data/tooltip_helpers.js';
-import { exclude_from_suite_data } from '../graph_data/helpers.js';
+import { get_suite_data_exclusion } from '../graph_data/helpers.js';
 import { setup_suites_in_suite_select } from '../filter/section_selects.js';
 import { format_duration } from '../common.js';
 import { dataLabelConfig } from '../variables/chartconfig.js';
@@ -120,7 +120,7 @@ function _build_suite_statistics_config() {
     const callbackData = data[1]
     const suiteSelectSuites = document.getElementById("suiteSelectSuites").value;
     const isCombined = suiteSelectSuites === "All Suites Combined";
-    const relevantSuites = filteredSuites.filter(s => !exclude_from_suite_data("suite", s));
+    const relevantSuites = get_relevant_suites();
     const tooltipMeta = build_tooltip_meta(relevantSuites, 'elapsed_s', isCombined);
     var config;
     if (settings.graphTypes.suiteStatisticsGraphType == "line") {
@@ -179,7 +179,7 @@ function _build_suite_duration_config() {
     const suiteSelectSuites = document.getElementById("suiteSelectSuites").value;
     const isCombined = suiteSelectSuites === "All Suites Combined";
     // Filter suites the same way get_duration_graph_data does, so tooltip meta matches
-    const relevantSuites = filteredSuites.filter(s => !exclude_from_suite_data("suite", s));
+    const relevantSuites = get_relevant_suites();
     const tooltipMeta = build_tooltip_meta(relevantSuites, 'elapsed_s', isCombined);
     var config;
     if (settings.graphTypes.suiteDurationGraphType == "bar") {
@@ -197,9 +197,14 @@ function _build_suite_duration_config() {
     return config;
 }
 
+function get_relevant_suites() {
+    const exclude_from_suite_data = get_suite_data_exclusion("suite");
+    return filteredSuites.filter(suite => !exclude_from_suite_data(suite));
+}
+
 function _get_suite_most_filtered_data() {
     if (!settings.switch.sectionFiltersApplySuite) return filteredSuites;
-    return filteredSuites.filter(s => !exclude_from_suite_data("suite", s));
+    return get_relevant_suites();
 }
 
 function _build_suite_most_failed_config() {