From 9eb1d1c42eaf36dbb78713aa85345dbce1d8e8a8 Mon Sep 17 00:00:00 2001 From: Paul Wise Date: Thu, 7 Nov 2024 13:16:46 +0800 Subject: [PATCH 01/29] Document the job header and its contents --- dashboard/dashboard.html | 3 +++ 1 file changed, 3 insertions(+) diff --git a/dashboard/dashboard.html b/dashboard/dashboard.html index a0304387..28187761 100644 --- a/dashboard/dashboard.html +++ b/dashboard/dashboard.html @@ -417,6 +417,9 @@

This page shows all of the crawls that ArchiveBot is currently running.

+

+ Each job has a header with the URL, note, request/queue stats, options and an identifier. +

To show or hide a job, click anywhere on its stats line. From 24ac72b0a954fef9614856ef5f3992ecab3f7ab0 Mon Sep 17 00:00:00 2001 From: Paul Wise Date: Thu, 27 Mar 2025 11:09:59 +0800 Subject: [PATCH 02/29] Refer to the job header rather than stats line --- dashboard/dashboard.html | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/dashboard/dashboard.html b/dashboard/dashboard.html index 28187761..077585bc 100644 --- a/dashboard/dashboard.html +++ b/dashboard/dashboard.html @@ -421,9 +421,9 @@ Each job has a header with the URL, note, request/queue stats, options and an identifier.

- To show or hide a job, click anywhere on its stats line. + To show or hide a job, click anywhere on its header. - The color coding for the job stats line is: + The color coding for the job header is: in progress, finished normally, finished with abort, From dc28751098cc5fd5900245810e0dcfa0fe5c5ad6 Mon Sep 17 00:00:00 2001 From: Paul Wise Date: Thu, 27 Mar 2025 11:11:29 +0800 Subject: [PATCH 03/29] Clarify that the filter operates on job logs and is a regex --- dashboard/dashboard.html | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/dashboard/dashboard.html b/dashboard/dashboard.html index 077585bc..31bf1fbf 100644 --- a/dashboard/dashboard.html +++ b/dashboard/dashboard.html @@ -398,7 +398,7 @@ ArchiveBot tracking 0 crawls. See also pipeline or job reports. - Show: + Logs @@ -442,12 +442,12 @@ Keyboard shortcuts:

From dfde2a7e4198c6911f5a55b8902aa2b65f75757c Mon Sep 17 00:00:00 2001 From: Paul Wise Date: Thu, 27 Mar 2025 12:50:27 +0800 Subject: [PATCH 04/29] Document the title URL parameter --- dashboard/dashboard.html | 1 + 1 file changed, 1 insertion(+) diff --git a/dashboard/dashboard.html b/dashboard/dashboard.html index 31bf1fbf..83593486 100644 --- a/dashboard/dashboard.html +++ b/dashboard/dashboard.html @@ -467,6 +467,7 @@

  • To retain more lines in the log windows, add ?historyLines=1000 to the dashboard URL. The default is 500, or 250 on mobile.
  • To update the dashboard more frequently, add ?batchTimeWhenVisible=33 to the dashboard URL. The default is 125 (8 Hz).
  • To skip loading of recent (buffered) log data for jobs, add ?loadRecent=0 to the dashboard URL. Inactive jobs will not appear.
    • +
  • To append some text to the page title, add ?title=TEXT to the dashboard URL.

    • To use ArchiveBot, drop by #archivebot on hackint. Issue commands by typing them into the channel. You will need channel operator (@) or voice (+) status to issue archiving jobs; just ask for help or leave a message with the website you want to archive. From c607ced7f27bc7f2429cae36cfcf8736195c1583 Mon Sep 17 00:00:00 2001 From: Paul Wise Date: Thu, 27 Mar 2025 12:50:52 +0800 Subject: [PATCH 05/29] Document the host URL parameter --- dashboard/dashboard.html | 1 + 1 file changed, 1 insertion(+) diff --git a/dashboard/dashboard.html b/dashboard/dashboard.html index 83593486..9516842c 100644 --- a/dashboard/dashboard.html +++ b/dashboard/dashboard.html @@ -468,6 +468,7 @@

  • To update the dashboard more frequently, add ?batchTimeWhenVisible=33 to the dashboard URL. The default is 125 (8 Hz).
  • To skip loading of recent (buffered) log data for jobs, add ?loadRecent=0 to the dashboard URL. Inactive jobs will not appear.
  • To append some text to the page title, add ?title=TEXT to the dashboard URL.
    • +
  • To override the hostname of the logs server, add ?host=TEXT to the dashboard URL.

    • To use ArchiveBot, drop by #archivebot on hackint. Issue commands by typing them into the channel. You will need channel operator (@) or voice (+) status to issue archiving jobs; just ask for help or leave a message with the website you want to archive. From 2a78e648d46f81fa080a976a05ef0258524115cd Mon Sep 17 00:00:00 2001 From: Paul Wise Date: Thu, 27 Mar 2025 12:51:20 +0800 Subject: [PATCH 06/29] Document the dumpMax URL parameter --- dashboard/dashboard.html | 1 + 1 file changed, 1 insertion(+) diff --git a/dashboard/dashboard.html b/dashboard/dashboard.html index 9516842c..42585ff6 100644 --- a/dashboard/dashboard.html +++ b/dashboard/dashboard.html @@ -469,6 +469,7 @@

  • To skip loading of recent (buffered) log data for jobs, add ?loadRecent=0 to the dashboard URL. Inactive jobs will not appear.
  • To append some text to the page title, add ?title=TEXT to the dashboard URL.
  • To override the hostname of the logs server, add ?host=TEXT to the dashboard URL.
    • +
  • To view the first two JSON messages of the logs stream, add ?dumpMax=2 to the dashboard URL.

    • To use ArchiveBot, drop by #archivebot on hackint. Issue commands by typing them into the channel. You will need channel operator (@) or voice (+) status to issue archiving jobs; just ask for help or leave a message with the website you want to archive. From 41a32e6ec11a02f65698edc2547f3f35cc7e4797 Mon Sep 17 00:00:00 2001 From: Paul Wise Date: Thu, 27 Mar 2025 12:51:34 +0800 Subject: [PATCH 07/29] Document the debug URL parameter --- dashboard/dashboard.html | 1 + 1 file changed, 1 insertion(+) diff --git a/dashboard/dashboard.html b/dashboard/dashboard.html index 42585ff6..c562c306 100644 --- a/dashboard/dashboard.html +++ b/dashboard/dashboard.html @@ -470,6 +470,7 @@

  • To append some text to the page title, add ?title=TEXT to the dashboard URL.
  • To override the hostname of the logs server, add ?host=TEXT to the dashboard URL.
  • To view the first two JSON messages of the logs stream, add ?dumpMax=2 to the dashboard URL.
    • +
  • To enable debug messages in the browser console, add ?debug=1 to the dashboard URL.

    • To use ArchiveBot, drop by #archivebot on hackint. Issue commands by typing them into the channel. You will need channel operator (@) or voice (+) status to issue archiving jobs; just ask for help or leave a message with the website you want to archive. From d734e9f433f451f43c4a3c6bfac915df74ed7540 Mon Sep 17 00:00:00 2001 From: Paul Wise Date: Thu, 27 Mar 2025 11:23:53 +0800 Subject: [PATCH 08/29] Add job notes to job URL title Makes it easier to access the notes when not in aligned mode. --- dashboard/assets/scripts/dashboard.js | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/dashboard/assets/scripts/dashboard.js b/dashboard/assets/scripts/dashboard.js index f570f438..86d020df 100644 --- a/dashboard/assets/scripts/dashboard.js +++ b/dashboard/assets/scripts/dashboard.js @@ -225,10 +225,11 @@ class JobsTracker { } class JobRenderInfo { - constructor(logWindow, logSegment, statsElements, jobNote, lineCountWindow, lineCountSegments) { + constructor(logWindow, logSegment, statsElements, jobUrl, jobNote, lineCountWindow, lineCountSegments) { this.logWindow = logWindow; this.logSegment = logSegment; this.statsElements = statsElements; + this.jobUrl = jobUrl; this.jobNote = jobNote; this.lineCountWindow = lineCountWindow; this.lineCountSegments = lineCountSegments; @@ -411,13 +412,14 @@ class JobsRenderer { ], ), ]); + const jobUrl = statsElements.jobInfo.querySelector(".job-url"); const logWindow = h("div", logWindowAttrs, logSegment); const div = h("div", { className: "log-container", id: `log-container-${ident}` }, [ h("div", { className: "job-header" }, [statsElements.jobInfo, h("span", { className: "job-ident" }, ident)]), logWindow, ]); - this.renderInfo[ident] = new JobRenderInfo(logWindow, logSegment, statsElements, jobNote, 0, [0]); + this.renderInfo[ident] = new JobRenderInfo(logWindow, logSegment, statsElements, jobUrl, jobNote, 0, [0]); this.container.insertBefore(div, beforeElement); // Filter hasn't changed, but we might need to filter out the new job, or // add/remove log-window-expanded class @@ -580,6 +582,11 @@ class JobsRenderer { // Update note info.jobNote.textContent = isBlank(jobData.note) ? "" : ` (${jobData.note})`; + if (isBlank(jobData.note)) { + info.jobUrl.removeAttribute("title"); + } else { + info.jobUrl.title = jobData.note; + } info.lineCountWindow += linesRendered; info.lineCountSegments[info.lineCountSegments.length - 1] += linesRendered; From baddaa01a595b9cab80482e0f7ac0504905f95e6 Mon Sep 17 00:00:00 2001 From: Paul Wise Date: Thu, 27 Mar 2025 15:48:57 +0800 Subject: [PATCH 09/29] Document the mouseover info for the URL and queue count --- dashboard/dashboard.html | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dashboard/dashboard.html b/dashboard/dashboard.html index c562c306..55eb8389 100644 --- a/dashboard/dashboard.html +++ b/dashboard/dashboard.html @@ -430,7 +430,7 @@ finished with fatal exception.

    - Mouse over the job start date or the response count for additional information. + Mouse over the job URL, start date, response count or the queue count for additional information.

    To pause scrolling, move your mouse inside a log window. From aa2f62399a4c8ef5f1c4d42f5a8c15fa90ee69cc Mon Sep 17 00:00:00 2001 From: Paul Wise Date: Wed, 16 Oct 2024 13:02:38 +0800 Subject: [PATCH 10/29] Compile the filter regex only once when filtering jobs Previously it was compiled once for each job being filtered. Suggested-by: JustAnotherArchivist Fixes: commit 990b70d2d62b6528e1762b91e8baf6fbc0066807 --- dashboard/assets/scripts/dashboard.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/dashboard/assets/scripts/dashboard.js b/dashboard/assets/scripts/dashboard.js index 86d020df..0cdbe656 100644 --- a/dashboard/assets/scripts/dashboard.js +++ b/dashboard/assets/scripts/dashboard.js @@ -628,14 +628,14 @@ class JobsRenderer { } applyFilter() { - const query = this.filterBox.value; + const query = RegExp(this.filterBox.value); let matches = 0; const matchedWindows = []; const unmatchedWindows = []; this.firstFilterMatch = null; for (const job of this.jobs.sorted) { const w = this.renderInfo[job.ident].logWindow; - if (!RegExp(query).test(job.url)) { + if (!query.test(job.url)) { w.classList.add("log-window-hidden"); unmatchedWindows.push(w); From c3b462893b8a988750052a15581e098186479605 Mon Sep 17 00:00:00 2001 From: Paul Wise Date: Thu, 27 Mar 2025 11:13:28 +0800 Subject: [PATCH 11/29] Add identifiers for the filter modification buttons --- dashboard/dashboard.html | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/dashboard/dashboard.html b/dashboard/dashboard.html index 55eb8389..aafe75bc 100644 --- a/dashboard/dashboard.html +++ b/dashboard/dashboard.html @@ -399,8 +399,8 @@ tracking 0 crawls. See also pipeline or job reports. Logs - - + +

    😊
    From aae91b934469e3006c826fdf65447c3978451587 Mon Sep 17 00:00:00 2001 From: Paul Wise Date: Thu, 27 Mar 2025 11:21:58 +0800 Subject: [PATCH 12/29] Add a button and key to revert to the initial filter When the initial filter is your nick, this makes it easier to switch between monitoring only your personal jobs and monitoring all jobs or other jobs. --- dashboard/assets/scripts/dashboard.js | 18 ++++++++++++++++-- dashboard/dashboard.html | 1 + 2 files changed, 17 insertions(+), 2 deletions(-) diff --git a/dashboard/assets/scripts/dashboard.js b/dashboard/assets/scripts/dashboard.js index 0cdbe656..f508402b 100644 --- a/dashboard/assets/scripts/dashboard.js +++ b/dashboard/assets/scripts/dashboard.js @@ -1027,7 +1027,7 @@ class Dashboard { const batchMaxItems = args.batchMaxItems ? Number(args.batchMaxItems) : 250; const showNicks = args.showNicks ? Boolean(Number(args.showNicks)) : false; const contextMenu = args.contextMenu ? Boolean(Number(args.contextMenu)) : true; - const initialFilter = args.initialFilter ?? "^$"; + this.initialFilter = args.initialFilter ?? "^$"; const loadRecent = args.loadRecent ? Boolean(Number(args.loadRecent)) : true; this.debug = args.debug ? Boolean(Number(args.debug)) : false; @@ -1076,7 +1076,19 @@ class Dashboard { addPageStyles(".job-nick-aligned { width: 0; }"); } - this.setFilter(initialFilter); + if (args.initialFilter != null) { + byId("set-filter-none").after( + h("input", { + className: "button", + type: "button", + id: "set-filter-initial", + onclick: () => { ds.setFilter(ds.initialFilter) }, + value: "Initial", + }) + ); + byId("set-filter-none").after("\n"); + } + this.setFilter(this.initialFilter); const finishSetup = () => { byId("meta-info").innerHTML = ""; @@ -1198,6 +1210,8 @@ ${String(kbPerSec).padStart(3, "0")} KB/s`; ev.preventDefault(); byId("filter-box").focus(); byId("filter-box").select(); + } else if (ev.which === 105 /* i */) { + ds.setFilter(ds.initialFilter); } else if (ev.which === 118 /* v */) { window.open(this.jobsRenderer.firstFilterMatch.url); } diff --git a/dashboard/dashboard.html b/dashboard/dashboard.html index aafe75bc..a9592db0 100644 --- a/dashboard/dashboard.html +++ b/dashboard/dashboard.html @@ -447,6 +447,7 @@
  • a - show all job log
  • n - hide all job log
  • f - move focus to filter box +
  • i - use the initial job log filter
  • v - open the job URL of the first-shown job log
  • ? - show/hide help text From b2183dc45ba8a6670b1515d5a1f0bf472ca7c136 Mon Sep 17 00:00:00 2001 From: Paul Wise Date: Thu, 27 Mar 2025 12:31:37 +0800 Subject: [PATCH 13/29] Allow hiding the job headers for hidden job logs This saves a lot of space when a lot of job logs are hidden. --- dashboard/assets/scripts/dashboard.js | 10 ++++++++++ dashboard/dashboard.html | 10 ++++++++++ 2 files changed, 20 insertions(+) diff --git a/dashboard/assets/scripts/dashboard.js b/dashboard/assets/scripts/dashboard.js index f508402b..4d11f0bd 100644 --- a/dashboard/assets/scripts/dashboard.js +++ b/dashboard/assets/scripts/dashboard.js @@ -1028,6 +1028,7 @@ class Dashboard { const showNicks = args.showNicks ? Boolean(Number(args.showNicks)) : false; const contextMenu = args.contextMenu ? Boolean(Number(args.contextMenu)) : true; this.initialFilter = args.initialFilter ?? "^$"; + const showAllHeaders = args.showAllHeaders ? Boolean(Number(args.showAllHeaders)) : true; const loadRecent = args.loadRecent ? Boolean(Number(args.loadRecent)) : true; this.debug = args.debug ? Boolean(Number(args.debug)) : false; @@ -1090,6 +1091,8 @@ class Dashboard { } this.setFilter(this.initialFilter); + this.showAllHeaders(showAllHeaders); + const finishSetup = () => { byId("meta-info").innerHTML = ""; @@ -1214,6 +1217,8 @@ ${String(kbPerSec).padStart(3, "0")} KB/s`; ds.setFilter(ds.initialFilter); } else if (ev.which === 118 /* v */) { window.open(this.jobsRenderer.firstFilterMatch.url); + } else if (ev.which === 104 /* h */) { + ds.showAllHeaders(!byId("show-all-headers").checked); } } @@ -1270,6 +1275,11 @@ ${String(kbPerSec).padStart(3, "0")} KB/s`; byId("filter-box").value = value; byId("filter-box").onchange(); } + + showAllHeaders(value) { + byId('show-all-headers').checked = value; + byId('hide-headers').sheet.disabled = value; + } } const ds = new Dashboard(); diff --git a/dashboard/dashboard.html b/dashboard/dashboard.html index a9592db0..5ff08c1b 100644 --- a/dashboard/dashboard.html +++ b/dashboard/dashboard.html @@ -391,6 +391,12 @@ top: 0; } +
    @@ -401,6 +407,8 @@ Logs + +
    😊
    @@ -449,6 +457,7 @@
  • f - move focus to filter box
  • i - use the initial job log filter
  • v - open the job URL of the first-shown job log +
  • h - show/hide headers for hidden job logs
  • ? - show/hide help text

    @@ -465,6 +474,7 @@

    • To specify an initial filter, add ?initialFilter=TEXT to the dashboard URL. The default is ^$.
      • +
    • To initially hide headers for hidden job logs, add ?showAllHeaders=0 to the dashboard URL. The default is to show them.
    • To retain more lines in the log windows, add ?historyLines=1000 to the dashboard URL. The default is 500, or 250 on mobile.
    • To update the dashboard more frequently, add ?batchTimeWhenVisible=33 to the dashboard URL. The default is 125 (8 Hz).
    • To skip loading of recent (buffered) log data for jobs, add ?loadRecent=0 to the dashboard URL. Inactive jobs will not appear.
      • From 951135968e7a188fba976b01def57619280ebfdc Mon Sep 17 00:00:00 2001 From: Paul Wise Date: Thu, 27 Mar 2025 12:33:01 +0800 Subject: [PATCH 14/29] Add an option to specify the port for the recent logs Allows using archivebot-dashboard-repeater on another port on localhost. See-also: https://github.com/iakat/archivebot-dashboard-repeater --- dashboard/assets/scripts/dashboard.js | 4 +++- dashboard/dashboard.html | 1 + 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/dashboard/assets/scripts/dashboard.js b/dashboard/assets/scripts/dashboard.js index 4d11f0bd..037ed60e 100644 --- a/dashboard/assets/scripts/dashboard.js +++ b/dashboard/assets/scripts/dashboard.js @@ -1038,6 +1038,8 @@ class Dashboard { } this.host = args.host ? args.host : location.hostname; + this.port = args.port ? `:${Number(args.port)}` : ''; + this.dumpTraffic = args.dumpMax && Number(args.dumpMax) > 0; if (this.dumpTraffic) { this.dumpMax = Number(args.dumpMax); @@ -1180,7 +1182,7 @@ ${String(kbPerSec).padStart(3, "0")} KB/s`; const size_mb = Math.round((100 * ev.total) / 1e6) / 100; byId("meta-info").textContent = `Recent data: ${percent}% (${size_mb}MB)`; }; - xhr.open("GET", `//${this.host}/logs/recent?cb=${Date.now()}${Math.random()}`); + xhr.open("GET", `//${this.host}${this.port}/logs/recent?cb=${Date.now()}${Math.random()}`); xhr.setRequestHeader("Accept", "application/json"); xhr.send(""); }); diff --git a/dashboard/dashboard.html b/dashboard/dashboard.html index 5ff08c1b..58bcb2f4 100644 --- a/dashboard/dashboard.html +++ b/dashboard/dashboard.html @@ -480,6 +480,7 @@
    • To skip loading of recent (buffered) log data for jobs, add ?loadRecent=0 to the dashboard URL. Inactive jobs will not appear.
    • To append some text to the page title, add ?title=TEXT to the dashboard URL.
    • To override the hostname of the logs server, add ?host=TEXT to the dashboard URL.
      • +
    • To override the port of the recent logs server, add ?port=TEXT to the dashboard URL.
    • To view the first two JSON messages of the logs stream, add ?dumpMax=2 to the dashboard URL.
    • To enable debug messages in the browser console, add ?debug=1 to the dashboard URL.
    From d00a6d20f686c559b4141917333d970e55476e0c Mon Sep 17 00:00:00 2001 From: Paul Wise Date: Thu, 27 Mar 2025 12:54:36 +0800 Subject: [PATCH 15/29] Add an option to specify the URL for the logs stream Allows using archivebot-dashboard-repeater on localhost. See-also: https://github.com/iakat/archivebot-dashboard-repeater --- dashboard/assets/scripts/dashboard.js | 6 +++--- dashboard/dashboard.html | 1 + 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/dashboard/assets/scripts/dashboard.js b/dashboard/assets/scripts/dashboard.js index 037ed60e..3f5fdee5 100644 --- a/dashboard/assets/scripts/dashboard.js +++ b/dashboard/assets/scripts/dashboard.js @@ -1039,6 +1039,8 @@ class Dashboard { this.host = args.host ? args.host : location.hostname; this.port = args.port ? `:${Number(args.port)}` : ''; + const wsproto = window.location.protocol === "https:" ? "wss:" : "ws:"; + this.websocketUrl = args.websocketUrl ?? `${wsproto}//${this.host}:4568/stream`; this.dumpTraffic = args.dumpMax && Number(args.dumpMax) > 0; if (this.dumpTraffic) { @@ -1233,9 +1235,7 @@ ${String(kbPerSec).padStart(3, "0")} KB/s`; } connectWebSocket() { - const wsproto = window.location.protocol === "https:" ? "wss:" : "ws:"; - - this.ws = new WebSocket(`${wsproto}//${this.host}:4568/stream`); + this.ws = new WebSocket(this.websocketUrl); this.ws.onmessage = (ev) => { this.newItemsReceived += 1; diff --git a/dashboard/dashboard.html b/dashboard/dashboard.html index 58bcb2f4..adfcca52 100644 --- a/dashboard/dashboard.html +++ b/dashboard/dashboard.html @@ -481,6 +481,7 @@
  • To append some text to the page title, add ?title=TEXT to the dashboard URL.
  • To override the hostname of the logs server, add ?host=TEXT to the dashboard URL.
  • To override the port of the recent logs server, add ?port=TEXT to the dashboard URL.
    • +
  • To override the URL of the logs stream, add ?websocketUrl=ws://localhost:4568/stream to the dashboard URL.
  • To view the first two JSON messages of the logs stream, add ?dumpMax=2 to the dashboard URL.
  • To enable debug messages in the browser console, add ?debug=1 to the dashboard URL.
    • From a3e3298f3fba616a8246393a02383c9105df1116 Mon Sep 17 00:00:00 2001 From: Paul Wise Date: Thu, 27 Mar 2025 15:28:02 +0800 Subject: [PATCH 16/29] Add a slight delay when typing into the job log filter Avoids applying the filter multiple times while typing, since that can be too slow and block the UI sometimes. --- dashboard/assets/scripts/dashboard.js | 28 ++++++++++++++++++--------- 1 file changed, 19 insertions(+), 9 deletions(-) diff --git a/dashboard/assets/scripts/dashboard.js b/dashboard/assets/scripts/dashboard.js index 3f5fdee5..18197f38 100644 --- a/dashboard/assets/scripts/dashboard.js +++ b/dashboard/assets/scripts/dashboard.js @@ -122,14 +122,6 @@ function regExpEscape(s) { return escaped; } -function addAnyChangeListener(elem, func) { - // DOM0 handler for convenient use by Clear button - elem.onchange = func; - elem.addEventListener("keydown", func, false); - elem.addEventListener("paste", func, false); - elem.addEventListener("input", func, false); -} - function scrollToBottom(elem) { // Scroll to the bottom. To avoid serious performance problems in Firefox, // use a big number instead of elem.scrollHeight. @@ -284,7 +276,25 @@ class JobsRenderer { constructor(container, filterBox, historyLines, showNicks, contextMenuRenderer) { this.container = container; this.filterBox = filterBox; - addAnyChangeListener(this.filterBox, () => this.applyFilter()); + this.filterTimeout = null; + this.filterBox.onchange = (e) => { + const repeats = [ + "insertText", + "deleteContent", + "deleteContentForward", + "deleteContentBackward", + ]; + let ms = e && e.inputType && repeats.includes(e.inputType) ? 100 : 0; + ms = !this.filterBox.value ? 0 : ms; + clearTimeout(this.filterTimeout); + this.filterTimeout = setTimeout(() => { + if (this.filterBox.value !== this.filterBox.old) { + this.applyFilter(); + this.filterBox.old = this.filterBox.value; + } + }, ms); + }; + this.filterBox.oninput = this.filterBox.onchange; this.filterBox.onkeypress = (ev) => { // Don't let `j` or `k` in the filter box cause the job window to switch ev.stopPropagation(); From 69f076f37c1e39c3bf4e0a8c46bd8ad92510aa2d Mon Sep 17 00:00:00 2001 From: Paul Wise Date: Thu, 27 Mar 2025 15:42:52 +0800 Subject: [PATCH 17/29] Allow hiding jobs based on their status Hiding finished jobs can save space when there are a lot of them. --- dashboard/assets/scripts/dashboard.js | 37 ++++++++++++++++++ dashboard/dashboard.html | 54 +++++++++++++++++++++++++++ 2 files changed, 91 insertions(+) diff --git a/dashboard/assets/scripts/dashboard.js b/dashboard/assets/scripts/dashboard.js index 18197f38..ec45ff7e 100644 --- a/dashboard/assets/scripts/dashboard.js +++ b/dashboard/assets/scripts/dashboard.js @@ -1039,6 +1039,10 @@ class Dashboard { const contextMenu = args.contextMenu ? Boolean(Number(args.contextMenu)) : true; this.initialFilter = args.initialFilter ?? "^$"; const showAllHeaders = args.showAllHeaders ? Boolean(Number(args.showAllHeaders)) : true; + const showRunningJobs = args.showRunningJobs ? Boolean(Number(args.showRunningJobs)) : true; + const showFinishedJobs = args.showFinishedJobs ? Boolean(Number(args.showFinishedJobs)) : true; + const showFatalJobs = args.showFatalJobs ? Boolean(Number(args.showFatalJobs)) : true; + const showAbortedJobs = args.showAbortedJobs ? Boolean(Number(args.showAbortedJobs)) : true; const loadRecent = args.loadRecent ? Boolean(Number(args.loadRecent)) : true; this.debug = args.debug ? Boolean(Number(args.debug)) : false; @@ -1107,6 +1111,11 @@ class Dashboard { this.showAllHeaders(showAllHeaders); + this.showRunningJobs(showRunningJobs); + this.showFinishedJobs(showFinishedJobs); + this.showFatalJobs(showFatalJobs); + this.showAbortedJobs(showAbortedJobs); + const finishSetup = () => { byId("meta-info").innerHTML = ""; @@ -1233,6 +1242,14 @@ ${String(kbPerSec).padStart(3, "0")} KB/s`; window.open(this.jobsRenderer.firstFilterMatch.url); } else if (ev.which === 104 /* h */) { ds.showAllHeaders(!byId("show-all-headers").checked); + } else if (ev.which === 114 /* r */) { + ds.showRunningJobs(!byId("show-running-jobs").checked); + } else if (ev.which === 100 /* d */) { + ds.showFinishedJobs(!byId("show-finished-jobs").checked); + } else if (ev.which === 99 /* c */) { + ds.showFatalJobs(!byId("show-fatal-jobs").checked); + } else if (ev.which === 115 /* s */) { + ds.showAbortedJobs(!byId("show-aborted-jobs").checked); } } @@ -1292,6 +1309,26 @@ ${String(kbPerSec).padStart(3, "0")} KB/s`; byId('show-all-headers').checked = value; byId('hide-headers').sheet.disabled = value; } + + showRunningJobs(value) { + byId('show-running-jobs').checked = value; + byId('hide-running').sheet.disabled = value; + } + + showFinishedJobs(value) { + byId('show-finished-jobs').checked = value; + byId('hide-done').sheet.disabled = value; + } + + showFatalJobs(value) { + byId('show-fatal-jobs').checked = value; + byId('hide-fatal').sheet.disabled = value; + } + + showAbortedJobs(value) { + byId('show-aborted-jobs').checked = value; + byId('hide-aborted').sheet.disabled = value; + } } const ds = new Dashboard(); diff --git a/dashboard/dashboard.html b/dashboard/dashboard.html index adfcca52..e6ec18a6 100644 --- a/dashboard/dashboard.html +++ b/dashboard/dashboard.html @@ -57,6 +57,11 @@ font-size: 18px; } +.drop-down { + display: inline flex; + flex-direction: column; +} + .padded-page { padding: 20px 27px 20px 27px; } @@ -397,6 +402,34 @@ content-visibility: hidden; } + + + +
    @@ -409,6 +442,17 @@ +
    + Jobs + +
    + +
    + +
    + +
    +
    😊
    @@ -458,6 +502,10 @@
  • i - use the initial job log filter
  • v - open the job URL of the first-shown job log
  • h - show/hide headers for hidden job logs +
  • r - show/hide header+log for running jobs +
  • d - show/hide header+log for finished jobs +
  • c - show/hide header+log for fatal jobs +
  • s - show/hide header+log for aborted jobs
  • ? - show/hide help text

    @@ -475,6 +523,12 @@

    • To specify an initial filter, add ?initialFilter=TEXT to the dashboard URL. The default is ^$.
    • To initially hide headers for hidden job logs, add ?showAllHeaders=0 to the dashboard URL. The default is to show them.
      • +
    • To initially hide different job types, add these to the dashboard URL. The default is to show them. + ?showRunningJobs=0 + ?showFinishedJobs=0 + ?showFatalJobs=0 + ?showAbortedJobs=0 +
    • To retain more lines in the log windows, add ?historyLines=1000 to the dashboard URL. The default is 500, or 250 on mobile.
    • To update the dashboard more frequently, add ?batchTimeWhenVisible=33 to the dashboard URL. The default is 125 (8 Hz).
    • To skip loading of recent (buffered) log data for jobs, add ?loadRecent=0 to the dashboard URL. Inactive jobs will not appear.
      • From 033eb830aff25c2019e3cdd9b642916af251ec91 Mon Sep 17 00:00:00 2001 From: Paul Wise Date: Thu, 27 Mar 2025 16:00:37 +0800 Subject: [PATCH 18/29] Fix detection of finished jobs Fixes: commit e179725059e7752f6d97a5e72f4556256e78fa12 --- dashboard/assets/scripts/dashboard.js | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/dashboard/assets/scripts/dashboard.js b/dashboard/assets/scripts/dashboard.js index ec45ff7e..45509c4c 100644 --- a/dashboard/assets/scripts/dashboard.js +++ b/dashboard/assets/scripts/dashboard.js @@ -492,10 +492,12 @@ class JobsRenderer { logSegment.appendChild(h("div", Reusable.obj_className_line_stdout, line)); renderedLines += 1; - // Check for 'Finished RsyncUpload for Item' - // instead of 'Starting MarkItemAsDone for Item' - // because the latter is often missing - if (/^Finished RsyncUpload for Item/.test(line)) { + // Check for several completion messages + // because some of them are often missing + // Ignore error jobs as they get done messages. + if (!info.statsElements.jobInfo.classList.contains("job-info-fatal") && + !info.statsElements.jobInfo.classList.contains("job-info-aborted") && + /^ *[1-9][0-9]* bytes\.$|^Starting (RelabelIfAborted|MarkItemAsDone) for Item$|^Finished (WgetDownload|MoveFiles|StopHeartbeat) for Item$/.test(line)) { info.statsElements.jobInfo.classList.add("job-info-done"); this.jobs.markFinished(ident); } else if ( From b752be2309fff7cf41d904ea732acde1edfab155 Mon Sep 17 00:00:00 2001 From: Paul Wise Date: Thu, 27 Mar 2025 15:47:53 +0800 Subject: [PATCH 19/29] Detect jobs that failed and allow filtering them out --- dashboard/assets/scripts/dashboard.js | 13 +++++++++++++ dashboard/dashboard.html | 16 ++++++++++++++++ 2 files changed, 29 insertions(+) diff --git a/dashboard/assets/scripts/dashboard.js b/dashboard/assets/scripts/dashboard.js index 45509c4c..a9ceed08 100644 --- a/dashboard/assets/scripts/dashboard.js +++ b/dashboard/assets/scripts/dashboard.js @@ -497,9 +497,12 @@ class JobsRenderer { // Ignore error jobs as they get done messages. if (!info.statsElements.jobInfo.classList.contains("job-info-fatal") && !info.statsElements.jobInfo.classList.contains("job-info-aborted") && + !info.statsElements.jobInfo.classList.contains("job-info-failed") && /^ *[1-9][0-9]* bytes\.$|^Starting (RelabelIfAborted|MarkItemAsDone) for Item$|^Finished (WgetDownload|MoveFiles|StopHeartbeat) for Item$/.test(line)) { info.statsElements.jobInfo.classList.add("job-info-done"); this.jobs.markFinished(ident); + } else if (/^ *0 bytes\.$/.test(line)) { + info.statsElements.jobInfo.classList.add("job-info-failed"); } else if ( /^CRITICAL (Sorry|Please report)|^ERROR Fatal exception|No space left on device|^Fatal Python error:|^(Thread|Current thread) 0x/.test( line, @@ -520,6 +523,7 @@ class JobsRenderer { } else if (/^Received item /.test(line)) { // Clear other statuses if a job restarts with the same job ID info.statsElements.jobInfo.classList.remove("job-info-done"); + info.statsElements.jobInfo.classList.remove("job-info-failed"); info.statsElements.jobInfo.classList.remove("job-info-fatal"); info.statsElements.jobInfo.classList.remove("job-info-aborted"); this.jobs.markUnfinished(ident); @@ -1043,6 +1047,7 @@ class Dashboard { const showAllHeaders = args.showAllHeaders ? Boolean(Number(args.showAllHeaders)) : true; const showRunningJobs = args.showRunningJobs ? Boolean(Number(args.showRunningJobs)) : true; const showFinishedJobs = args.showFinishedJobs ? Boolean(Number(args.showFinishedJobs)) : true; + const showFailedJobs = args.showFailedJobs ? Boolean(Number(args.showFailedJobs)) : true; const showFatalJobs = args.showFatalJobs ? Boolean(Number(args.showFatalJobs)) : true; const showAbortedJobs = args.showAbortedJobs ? Boolean(Number(args.showAbortedJobs)) : true; const loadRecent = args.loadRecent ? Boolean(Number(args.loadRecent)) : true; @@ -1115,6 +1120,7 @@ class Dashboard { this.showRunningJobs(showRunningJobs); this.showFinishedJobs(showFinishedJobs); + this.showFailedJobs(showFailedJobs); this.showFatalJobs(showFatalJobs); this.showAbortedJobs(showAbortedJobs); @@ -1248,6 +1254,8 @@ ${String(kbPerSec).padStart(3, "0")} KB/s`; ds.showRunningJobs(!byId("show-running-jobs").checked); } else if (ev.which === 100 /* d */) { ds.showFinishedJobs(!byId("show-finished-jobs").checked); + } else if (ev.which === 98 /* b */) { + ds.showFailedJobs(!byId("show-failed-jobs").checked); } else if (ev.which === 99 /* c */) { ds.showFatalJobs(!byId("show-fatal-jobs").checked); } else if (ev.which === 115 /* s */) { @@ -1322,6 +1330,11 @@ ${String(kbPerSec).padStart(3, "0")} KB/s`; byId('hide-done').sheet.disabled = value; } + showFailedJobs(value) { + byId('show-failed-jobs').checked = value; + byId('hide-failed').sheet.disabled = value; + } + showFatalJobs(value) { byId('show-fatal-jobs').checked = value; byId('hide-fatal').sheet.disabled = value; diff --git a/dashboard/dashboard.html b/dashboard/dashboard.html index e6ec18a6..1d61d99b 100644 --- a/dashboard/dashboard.html +++ b/dashboard/dashboard.html @@ -133,6 +133,10 @@ color: #DD0000 !important; } +.job-info-failed { + color: #CC7676 !important; +} + .inline-stat { /* Needed for 'Align!' feature */ display: inline-block; @@ -405,6 +409,7 @@ +