From 186d580df38dbef5da8fd2ddd89c830f9499a6be Mon Sep 17 00:00:00 2001 From: Vasyl Osypchuk Date: Sat, 19 Sep 2026 07:41:35 +0300 Subject: [PATCH 1/7] Harden first application monitoring readiness and reference integration --- .github/workflows/ci.yml | 10 + CHANGELOG.md | 11 + README.md | 13 +- architecture.md | 5 +- design.md | 5 +- src/monitoring/agents/dispatch.zig | 2 +- src/monitoring/agents/signals.py | 35 +-- src/monitoring/agents/status.zig | 2 +- src/monitoring/agents/tests.zig | 37 ++++ src/monitoring/apps/dispatch.zig | 4 +- src/monitoring/apps/station.zig | 33 ++- src/monitoring/readiness.zig | 3 + tests/agent_checks_test.py | 33 ++- tests/agent_ingestion_test.py | 37 ++++ tests/app_station_test.py | 26 +++ tests/ingress_proxy_fixture.py | 12 +- tests/integration/README.md | 3 +- tests/integration/application.md | 2 +- tests/integration/doers-validation.md | 106 ++++++++++ tests/integration/doers_runtime.py | 293 ++++++++++++++++++++++++++ tests/integration/render_doers.py | 67 ++++++ tests/integration/vector_outage.py | 68 ++++++ tools/fetch_doers_fixture.py | 72 +++++++ 23 files changed, 847 insertions(+), 32 deletions(-) create mode 100644 tests/integration/doers-validation.md create mode 100644 tests/integration/doers_runtime.py create mode 100644 tests/integration/render_doers.py create mode 100644 tests/integration/vector_outage.py create mode 100644 tools/fetch_doers_fixture.py diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index d575b99..b6f774c 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -24,6 +24,16 @@ jobs: python3 -I -B tools/fetch_caddy_fixture.py --output "$RUNNER_TEMP/dragontools-caddy-2.11.4" python3 -I -B tests/agent_ingestion_test.py --caddy "$RUNNER_TEMP/dragontools-caddy-2.11.4" - run: python3 -I -B tests/release_test.py + - name: Doers reference signals and alert recovery (isolated Linux processes) + if: runner.os == 'Linux' + timeout-minutes: 12 + run: | + fixture="$RUNNER_TEMP/dragontools-doers" + python3 -I -B tests/integration/render_doers.py "$fixture" + python3 -I -B tools/fetch_doers_fixture.py --output "$fixture" + cp zig-out/bin/dragontool-agent "$fixture/dragontool-agent" + python3 -I -B tests/integration/agent_ingestion_pipeline.py --prepare-credentials "$fixture" + sudo unshare --net sh -c 'ip link set lo up; exec python3 -I -B tests/integration/doers_runtime.py "$1"' fixture "$fixture" - name: Linux helper filesystem lifecycle (temporary paths only) if: runner.os == 'Linux' run: sudo python3 -I -B tests/helper_install_test.py --fixture "$PWD/zig-out/bin/dragontool-pki-fixture" --agent "$PWD/zig-out/bin/dragontool-agent" diff --git a/CHANGELOG.md b/CHANGELOG.md index 766ac75..647a965 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,17 @@ ## 0.1.0-dev — unreleased +- Accept fresh vmagent failed-scrape telemetry when the application is down; + require fresh non-scrape payload on successful targets and reject stale/missing + data. Report pipeline readiness independently of application availability. +- Give app publication, scraper reload and finalization fixed semantic check IDs; + do not suggest reinstalling ingress for app namespace/rule failures. +- Add the production-rendered Doers reference process gate: zero-app station, + real pinned Caddy/Vector/vmagent/blackbox/VM/VL/vmalert, quiet logs and trusted + labels, the real two-minute alert hold and recovery, scoped probe removal and + unchanged publication. Add the isolated Linux gate to CI, retaining shared + Linux/macOS tests and a fixture-only early HTTP rejection portability fix. + - Move station CA/server PKI, Caddy and private ingress authorization ownership to `monitoring install`. Add `--ingress-hostname` / `[ingress].hostname`, independent of SSH; reuse an existing managed endpoint when omitted. Install/verify/status diff --git a/README.md b/README.md index 77b41a8..43ce0a1 100644 --- a/README.md +++ b/README.md @@ -875,6 +875,13 @@ app rule glob, and zero rule samples are valid. `monitoring apply` and expected managed rules to be loaded and healthy. Rule readiness uses bounded retries after restart; it does not require an alert expression to match samples. A failing HTTP target is valid monitoring data; a broken probe pipeline fails. +The same distinction applies to vmagent: fresh `up=0` proves scrape reporting and +remote delivery while the target is down. Success reports pipeline readiness, +not application availability. An `up=1` target still needs fresh application payload. +The [Doers reference validation](tests/integration/doers-validation.md) records +isolated signal, alert recovery, namespace and rerun evidence and remaining host +checks. Publication/reload failures identify `application_publish`, +`application_scrape_reload` or `application_finalize` without raw remote output. Verify/status never resolve station secrets or send test notifications. Live alert evaluation remains active independently. @@ -1262,8 +1269,10 @@ and the main journald configuration remain untouched. Conflicting later override are refused. These bounds do not cover applications writing their own log files. Install verifies service/configuration/hardening, the secured endpoint, recent -host metrics, every selected log stream, and each app target's successful scrape -and recent non-scrape metric. Install/verify require samples newer than the current +host metrics, every selected log stream, and fresh scrape telemetry for each app +target. A target reporting `up=1` must also supply a recent non-scrape metric. +Fresh `up=0` is valid monitoring while the application is down; missing/stale +`up`, missing payload from an `up=1` target, or an unreachable station still fail. Install/verify require samples newer than the current agent process start as well as their freshness windows (90 seconds for metrics, two minutes for logs), preventing old data from proving a changed URL works. Keep both hosts' clocks synchronized because Vector timestamps originate on the agent. diff --git a/architecture.md b/architecture.md index 57419dc..fcb9565 100644 --- a/architecture.md +++ b/architecture.md @@ -471,7 +471,10 @@ The shared Vector host rules preserve application/environment/host grouping; application log rules use exact scoped fields and bounded counts/windows. Each probe has one default alert or one explicit override. Native probe telemetry with `probe_success=0` is a successful monitoring mechanism, not apply failure. -No notification tests or synthetic application errors run during apply/verify. +Fresh vmagent `up=0` likewise proves delivery while the target is down; `up=1` +requires both a fresh scrape result and application payload. Missing/stale samples +and station outages fail bounded readiness. No notification tests or synthetic +application errors run during apply/verify. Dashboards remain unimplemented. Future generated dashboards must use folder `DragonTools / ` and deterministic UIDs, with explicit ownership. diff --git a/design.md b/design.md index a024b74..aab0fff 100644 --- a/design.md +++ b/design.md @@ -1173,8 +1173,9 @@ Deterministic binary/configuration/user/hardening/argument/listener errors fail without retry. Runtime checks use the common bounded readiness policy (15 seconds active, 30 seconds HTTP, 45 seconds signal checks; 500ms first retry then one second). Station queries require host samples within 90 seconds, selected log -streams within two minutes, and each application's `up=1` plus a recent real -non-scrape metric. Install/verify require these samples to postdate the current +streams within two minutes, and each application's fresh scrape result. `up=0` +is valid target-down telemetry; `up=1` additionally requires a recent real +non-scrape metric. Missing/stale scrape results and empty successful targets fail. Install/verify require these samples to postdate the current agent process start, so stale data cannot prove a changed target URL works. Both host clocks must be synchronized; Vector uses agent-side timestamps. Install never claims success for only local configuration. diff --git a/src/monitoring/agents/dispatch.zig b/src/monitoring/agents/dispatch.zig index a410245..8ba3070 100644 --- a/src/monitoring/agents/dispatch.zig +++ b/src/monitoring/agents/dispatch.zig @@ -90,6 +90,6 @@ pub fn run(init: std.process.Init, options: cli.Options) !void { print(init.io, report.enrollmentSummary()); if (options.command == .agents_install and report.state.changes == 0) print(init.io, "No changes required.\n"); print(init.io, "Vector\n active; enabled\n host metrics flowing\n selected log stream identities flowing (quiet-service metadata included)\n"); - print(init.io, if (registration.metrics_targets.len > 0) "vmagent\n active; enabled\n application metrics flowing\n" else "vmagent\n not required (no application metrics targets)\n"); + print(init.io, if (registration.metrics_targets.len > 0) "vmagent\n active; enabled\n application metrics pipeline verified (fresh scrape telemetry; target may be down)\n" else "vmagent\n not required (no application metrics targets)\n"); print(init.io, "OTel traces: unavailable. No test errors or notifications generated.\n"); } diff --git a/src/monitoring/agents/signals.py b/src/monitoring/agents/signals.py index 2e693db..255dbd3 100644 --- a/src/monitoring/agents/signals.py +++ b/src/monitoring/agents/signals.py @@ -9,14 +9,17 @@ def request(port, path, fields): conn = http.client.HTTPConnection('127.0.0.1', port, timeout=5) - conn.request('POST', path, urllib.parse.urlencode(fields), {'Content-Type': 'application/x-www-form-urlencoded'}) - response = conn.getresponse() - body = response.read(1024 * 1024 + 1) - if response.status in (500, 502, 503, 504): - raise ConnectionError() - if response.status != 200 or len(body) > 1024 * 1024: - raise ValueError('invalid query response') - return body + try: + conn.request('POST', path, urllib.parse.urlencode(fields), {'Content-Type': 'application/x-www-form-urlencoded'}) + response = conn.getresponse() + body = response.read(1024 * 1024 + 1) + if response.status in (500, 502, 503, 504): + raise ConnectionError() + if response.status != 200 or len(body) > 1024 * 1024: + raise ValueError('invalid query response') + return body + finally: + conn.close() def metric(query): @@ -31,6 +34,16 @@ def check(mode, registration, since=0): def fresh(selector): stamp = 'timestamp(' + selector + ')' return metric('(' + stamp + ' >= ' + str(float(since)) + ') and (' + stamp + ' > time()-90)') + def scrape_ready(labels): + # A fresh failed scrape proves vmagent -> station delivery while the + # application is down. Missing/stale/pre-restart samples prove nothing. + # A successful scrape must still contain real application payload. + up = 'up' + labels + '}' + if not fresh(up): + return False + if metric(up + ' == 1'): + return fresh(labels + ',__name__!~"up|scrape_.*"}') + return metric(up + ' == 0') applications = registration.get('applications', []) if applications: for application in applications: @@ -60,7 +73,7 @@ def fresh(selector): if service['metrics_url'] is None: continue labels = '{host=' + host + ',agent="vmagent"' + identity + ',service=' + json.dumps(service['name']) - if not metric('up' + labels + '} == 1') or not fresh(labels + ',__name__!~"up|scrape_.*"}'): + if not scrape_ready(labels): return False else: raise ValueError('invalid signal check') @@ -89,9 +102,7 @@ def fresh(selector): if mode == 'app': for target in registration['metrics_targets']: labels = '{host=' + host + ',agent="vmagent",app=' + json.dumps(target['name']) - if not metric('up' + labels + '} == 1'): - return False - if not fresh(labels + ',__name__!~"up|scrape_.*"}'): + if not scrape_ready(labels): return False return True raise ValueError('invalid signal check') diff --git a/src/monitoring/agents/status.zig b/src/monitoring/agents/status.zig index edc7b9f..ae87658 100644 --- a/src/monitoring/agents/status.zig +++ b/src/monitoring/agents/status.zig @@ -33,5 +33,5 @@ fn statusSelected(a: std.mem.Allocator, app: remote.Remote, station: remote.Remo const vm_active = try state(a, app, "dragontools-vmagent.service", "is-active"); const vm_enabled = try state(a, app, "dragontools-vmagent.service", "is-enabled"); const flowing = try signal(a, station, selected, "app"); - return std.fmt.allocPrint(a, "{s}vmagent\n {s}\n {s}\n configured targets: {d}\n remote write {s}\nOTel traces: unavailable.\n", .{ vector, if (vm_active) "active" else "inactive", if (vm_enabled) "enabled" else "disabled", selected.metricsCount(), if (flowing) "healthy (recent application metrics)" else "unverified (targets or station unavailable)" }); + return std.fmt.allocPrint(a, "{s}vmagent\n {s}\n {s}\n configured targets: {d}\n remote write {s}\nOTel traces: unavailable.\n", .{ vector, if (vm_active) "active" else "inactive", if (vm_enabled) "enabled" else "disabled", selected.metricsCount(), if (flowing) "healthy (fresh scrape telemetry; target may be down)" else "unverified (no recent telemetry or station unavailable)" }); } diff --git a/src/monitoring/agents/tests.zig b/src/monitoring/agents/tests.zig index cbdf300..6dbba1e 100644 --- a/src/monitoring/agents/tests.zig +++ b/src/monitoring/agents/tests.zig @@ -841,3 +841,40 @@ test "per-signal endpoint diagnostics preserve bounded retries and skip unselect } } } + +test "Doers reference enrollment rerun preserves agents certificates and station ingress" { + var arena = std.heap.ArenaAllocator.init(std.testing.allocator); + defer arena.deinit(); + const a = arena.allocator(); + var value = try @import("../../config/application.zig").load(a, std.testing.io, "examples/doers-monitoring.toml"); + defer value.deinit(); + try std.testing.expectEqualStrings("softwarelanding", value.target_ssh_host); + try std.testing.expectEqualStrings("monitoring", value.station_ssh_host); + try std.testing.expectEqualStrings("monitoring.baptizeddragon.com", value.station_hostname); + try std.testing.expectEqualStrings("doers", value.application.name); + try std.testing.expectEqualStrings("production", value.application.environment); + try std.testing.expectEqualStrings("doers.service", value.services[0].systemd); + try std.testing.expect(value.services[0].logs); + try std.testing.expectEqualStrings("http://127.0.0.1:16005/metrics", value.services[0].metrics_url.?); + try std.testing.expectEqualStrings("https://doers.business/healthz", value.probes[0].url); + const scopes = [_]model.ApplicationScope{try @import("../apps/dispatch.zig").scope(a, value)}; + const reference: model.Registration = .{ .host = registration.host, .station = value.station_hostname, .services = &.{"doers.service"}, .metrics_targets = &.{}, .applications = &scopes }; + var report: model.Report = .{ .application = value.application.name }; + var fake: Fake = .{ .allocator = a, .report = &report, .station_hostname = value.station_hostname }; + try install.install(a, fake.asRemote(), fake.asRemote(), &report, reference); + const before = fake.mutations; + report = .{ .application = value.application.name }; + try install.install(a, fake.asRemote(), fake.asRemote(), &report, reference); + try std.testing.expectEqual(@as(usize, 0), report.state.changes); + try std.testing.expectEqual(before, fake.mutations); + try std.testing.expectEqual(@as(usize, 1), fake.enrollments); + for ([_]model.Component{ .vector, .vmagent }) |kind| { + try std.testing.expectEqual(@as(usize, 1), fake.state(kind).restarts); + try std.testing.expectEqual(@as(usize, 1), fake.state(kind).credential_writes); + try std.testing.expect(!fake.state(kind).pending); + } + for ([_]model.Component{ .caddy, .ingestion, .host_rules }) |kind| { + try std.testing.expectEqual(@as(usize, 0), fake.state(kind).restarts); + for (fake.state(kind).commands) |command| try std.testing.expect(command == null); + } +} diff --git a/src/monitoring/apps/dispatch.zig b/src/monitoring/apps/dispatch.zig index 374b9ec..8704929 100644 --- a/src/monitoring/apps/dispatch.zig +++ b/src/monitoring/apps/dispatch.zig @@ -61,7 +61,7 @@ pub fn execute(a: std.mem.Allocator, app: remote.Remote, station: remote.Remote, if (command == .app_apply and report.state.changes == 0) "No changes required.\n" else "", report.enrollmentSummary(), if (logs > 0) "selected service logs flowing (quiet-service metadata included)" else "selected service logs: disabled", - if (metrics > 0) "application metrics flowing" else "application metrics: not configured", + if (metrics > 0) "application metrics pipeline verified (fresh scrape telemetry; target may be down)" else "application metrics: not configured", value.probes.len, }); } @@ -124,7 +124,7 @@ pub fn run(init: std.process.Init, options: cli.Options) !void { if (options.command == .app_apply) "Completed changes may remain; pending intent is preserved. Correct the cause and rerun the same application config." else "This command is read-only; no configuration or restart intent was changed.", })); print(init.io, try report.state.credentialDiagnostics(a)); - if (report.component == .ingestion or report.component == .caddy or report.component == .station) print(init.io, "Station ingress is owned by monitoring install. Run monitoring install on the station with its configured --ingress-hostname, then retry. Application apply does not bootstrap or repair base ingress.\n"); + if (report.component == .ingestion or report.component == .caddy or report.state.check == .station_ingress_required) print(init.io, "Station ingress is owned by monitoring install. Run monitoring install on the station with its configured --ingress-hostname, then retry. Application apply does not bootstrap or repair base ingress.\n"); if (report.state.check == .dns_unresolved) print(init.io, "Monitoring station hostname does not resolve. DragonTools does not manage DNS. Configure the DNS record and rerun the same command.\n"); if (report.state.check == .tcp_metrics_unreachable or report.state.check == .tcp_logs_unreachable) { const port: u16 = if (report.state.check == .tcp_logs_unreachable) 9444 else 9443; diff --git a/src/monitoring/apps/station.zig b/src/monitoring/apps/station.zig index 88b9620..f5079e6 100644 --- a/src/monitoring/apps/station.zig +++ b/src/monitoring/apps/station.zig @@ -28,8 +28,15 @@ fn rulesCommand(a: std.mem.Allocator, config: Config, identity: []const u8, kind pub fn preflight(a: std.mem.Allocator, r: remote.Remote, report: *workflow.Report, config: Config, identity: []const u8) !void { _ = try readiness.deterministic(a, r, report, .application_ownership, try command(a, config, identity, "preflight", false)); } +fn mutate(r: remote.Remote, report: *workflow.Report, op: remote.Operation, check: readiness.Check, cmd: []const u8) !void { + _ = report.call(r, op, cmd) catch |err| { + // A fixed substage survives failures without forwarding remote output. + report.check = check; + return err; + }; +} pub fn apply(a: std.mem.Allocator, r: remote.Remote, report: *workflow.Report, arch: host.Arch, config: Config, identity: []const u8) !void { - _ = try report.call(r, .config, try command(a, config, identity, "publish", true)); + try mutate(r, report, .config, .application_publish, try command(a, config, identity, "publish", true)); // First integration updates generated units once. Exact fixed rule packs are // preserved, and each evaluator keeps its own restart/finalization boundary. for ([_]vmalert.Kind{ .logs, .metrics }) |kind| { @@ -37,9 +44,9 @@ pub fn apply(a: std.mem.Allocator, r: remote.Remote, report: *workflow.Report, a try vmalert.installWithRuleCheck(a, r, report, arch, kind, try rulesCommand(a, config, identity, kind)); } report.component = .victoriametrics; - _ = try report.call(r, .activate, try command(a, config, identity, "activate", true)); + try mutate(r, report, .activate, .application_scrape_reload, try command(a, config, identity, "activate", true)); try verify(a, r, report, arch, config, identity); - _ = try report.call(r, .finalize, try command(a, config, identity, "finalize", true)); + try mutate(r, report, .finalize, .application_finalize, try command(a, config, identity, "finalize", true)); } pub fn verify(a: std.mem.Allocator, r: remote.Remote, report: *workflow.Report, arch: host.Arch, config: Config, identity: []const u8) !void { _ = try readiness.deterministic(a, r, report, .managed_state, try command(a, config, identity, "managed", false)); @@ -152,3 +159,23 @@ test "station base readiness is independent while application rules gate verific try std.testing.expectEqual(@as(usize, 0), report.changes); try std.testing.expectEqual(restarts, fake.restarts); } + +test "application mutation failures retain semantic substage without raw output" { + const Fake = struct { + fn call(_: *anyopaque, _: remote.Operation, _: []const u8) !remote.Result { + return .{ .code = 1, .output = "PRIVATE fixture payload" }; + } + }; + var context: u8 = 0; + const r: remote.Remote = .{ .context = &context, .execute = Fake.call }; + const operations = [_]remote.Operation{ .config, .activate, .finalize }; + const checks = [_]readiness.Check{ .application_publish, .application_scrape_reload, .application_finalize }; + for (operations, checks) |op, check| { + var report: workflow.Report = .{}; + try std.testing.expectError(error.RemoteOperationFailed, mutate(r, &report, op, check, "fixed operation")); + try std.testing.expectEqual(check, report.check.?); + try std.testing.expectEqual(op, report.phase); + try std.testing.expectEqual(@as(usize, 0), report.changes); + try std.testing.expectEqualStrings("", try report.credentialDiagnostics(std.testing.allocator)); + } +} diff --git a/src/monitoring/readiness.zig b/src/monitoring/readiness.zig index 64fe0bc..e0700f0 100644 --- a/src/monitoring/readiness.zig +++ b/src/monitoring/readiness.zig @@ -12,6 +12,9 @@ pub const Check = enum { caddy_service, legacy_ingress_conflict, application_ownership, + application_publish, + application_scrape_reload, + application_finalize, plugin_integrity, service_active, http_ready, diff --git a/tests/agent_checks_test.py b/tests/agent_checks_test.py index e9c5dc8..e807515 100644 --- a/tests/agent_checks_test.py +++ b/tests/agent_checks_test.py @@ -189,6 +189,9 @@ def getresponse(self): body = b'x' * (1024 * 1024 + 1) return types.SimpleNamespace(status=status, read=lambda limit: body[:limit]) + def close(self): + pass + with patch('http.client.HTTPConnection', Connection): result = entrypoint(SIGNALS, ['signals.py', mode, json.dumps(REGISTRATION), str(SINCE)]) return result, queries @@ -208,10 +211,10 @@ def test_host_metrics_require_all_contract_signals_and_app_requires_payload(self self.assertIn('agent="vector"', query) result, queries = self.run_signal('app') self.assertEqual(result, 0) - self.assertEqual(len(queries), 2) + self.assertEqual(len(queries), 3) self.assertIn('up{', queries[0]) - self.assertIn('__name__!~"up|scrape_.*"', queries[1]) - self.assertIn('agent="vmagent",app="software"', queries[1]) + self.assertIn('__name__!~"up|scrape_.*"', queries[2]) + self.assertIn('agent="vmagent",app="software"', queries[2]) def test_station_unavailable_retries_but_invalid_responses_fail_deterministically(self): for status in (500, 502, 503, 504): @@ -237,6 +240,28 @@ def module(self): def registration(self): return dict(REGISTRATION, services=['doers.service'], metrics_targets=[], applications=[dict(name='doers', environment='production', services=[dict(name='web', systemd='doers.service', logs=True, metrics_url='http://127.0.0.1:16005/metrics')])]) + def test_failed_scrape_is_valid_but_stale_absent_and_empty_success_are_not(self): + for registration in (REGISTRATION, self.registration()): + for up, fresh_up, payload, expected in ( + (0, True, False, True), (1, True, True, True), + (1, True, False, False), (0, False, True, False), + (1, False, True, False), (None, False, False, False), + (2, True, True, False)): + with self.subTest(application=bool(registration.get('applications')), up=up, + fresh_up=fresh_up, payload=payload): + functions = self.module() + def metric(query): + if query.startswith('(timestamp(up{'): + self.assertIn(' >= ' + str(float(SINCE)), query) + self.assertIn(' > time()-90)', query) + return fresh_up + if query.startswith('up{'): + return up == (1 if query.endswith(' == 1') else 0) + self.assertIn('__name__!~"up|scrape_.*"', query) + return payload + functions['metric'] = metric + self.assertEqual(functions['check']('app', registration, SINCE), expected) + def test_all_checks_scope_application_environment_and_service(self): functions = self.module() metrics, logs = [], [] @@ -249,7 +274,7 @@ def request(port, path, fields): registration = self.registration() for mode in ('host', 'logs', 'app'): self.assertTrue(functions['check'](mode, registration, SINCE)) - self.assertEqual(len(metrics), 5) + self.assertEqual(len(metrics), 6) self.assertEqual(len(logs), 1) for query in metrics: self.assertIn('application="doers"', query) diff --git a/tests/agent_ingestion_test.py b/tests/agent_ingestion_test.py index 8933f82..592df28 100644 --- a/tests/agent_ingestion_test.py +++ b/tests/agent_ingestion_test.py @@ -4,6 +4,7 @@ helper receives client private keys and no SSH/real monitoring host is used. """ import contextlib +import email.message import hashlib import http.client import http.server @@ -18,6 +19,7 @@ import tempfile import threading import time +import types import argparse from unittest.mock import patch @@ -99,7 +101,42 @@ def serve(server): return thread +def early_proxy_rejection(): + """Force the body-write race, independent of socket scheduling or OS.""" + for status in (404, 204): + handler = object.__new__(proxy.Proxy) + handler.connection = types.SimpleNamespace(getpeercert=lambda binary_form=False: b'public fixture' if binary_form else { + 'subject': ((('commonName', HOST),),), 'subjectAltName': (('URI', IDENTITY),)}) + handler.headers = email.message.Message() + handler.headers['Content-Length'] = '4' + handler.rfile = io.BytesIO(b'test') + handler.server = types.SimpleNamespace(upstream='fixture.sock') + handler.command, handler.path = 'POST', '/rejected' + statuses, closed = [], [] + handler.send_response = statuses.append + handler.send_header = lambda *_: None + handler.end_headers = lambda: None + def rejected(_): + raise BrokenPipeError('upstream already replied') + connection = types.SimpleNamespace(putrequest=lambda *_: None, putheader=lambda *_: None, + endheaders=rejected, getresponse=lambda: types.SimpleNamespace(status=status, read=lambda: b''), + close=lambda: closed.append(True)) + with patch.object(proxy, 'UnixConnection', return_value=connection): + if status == 404: + handler.forward() + assert statuses == [404] + else: + try: + handler.forward() + except BrokenPipeError: + pass + else: + raise AssertionError('Broken body write accepted as success') + assert closed == [True] + + def main(binary=None): + early_proxy_rejection() with tempfile.TemporaryDirectory(prefix='dt-ingress-', dir='/tmp') as temporary, contextlib.ExitStack() as stack: root = Path(temporary) ca = root / 'station-ca' diff --git a/tests/app_station_test.py b/tests/app_station_test.py index e6cbd84..d9bbf86 100644 --- a/tests/app_station_test.py +++ b/tests/app_station_test.py @@ -92,6 +92,32 @@ def test_unchanged_publication_does_not_validate_rewrite_or_touch_markers(self): self.assertFalse(self.n['app_publish'](self.config)) self.assertEqual(before, self.snapshot(Path(self.temp.name))) + def test_probe_removal_changes_only_own_scrape_and_metrics_rules_then_noop(self): + self.n['app_publish'](self.config) + self.n['app_publish'](config('other')) + root = Path(self.temp.name) + manual = root / 'manual-probes.yml' + rules = root / 'manual.rules.yml' + manual.write_text('administrator probe outside app namespaces\n') + rules.write_text('administrator rule outside app namespaces\n') + station = root / 'prometheus.yml' + station.write_text('existing station probes\n') + for path in self.n['APP_PENDING'].values(): + if os.path.exists(path): + os.unlink(path) + before = self.snapshot(root) + self.config['probes'] = [] + self.assertTrue(self.n['app_publish'](self.config)) + after = self.snapshot(root) + changed = {path for path in before if before[path] != after[path]} + self.assertEqual(changed, {'apps/doers/manifest.json', 'apps/doers/scrape.yml', 'apps/doers/metrics.rules.yml'}) + self.assertEqual(json.loads(self.app_path('scrape.yml').read_bytes()), []) + self.assertEqual(json.loads(self.app_path('metrics.rules.yml').read_bytes()), {'groups': []}) + self.assertEqual({key for key, path in self.n['APP_PENDING'].items() if os.path.exists(path)}, {'metrics.rules.yml', 'scrape.yml'}) + self.assertEqual(set(Path(self.n['APP_ROOT']).iterdir()), {self.app_path('manifest.json').parent, self.app_path('manifest.json', 'other').parent}) + self.assertFalse(self.n['app_publish'](self.config)) + self.assertEqual(after, self.snapshot(root)) + def test_alert_probe_and_endpoint_edits_only_dirty_affected_consumers(self): self.n['app_publish'](self.config) for path in self.n['APP_PENDING'].values(): diff --git a/tests/ingress_proxy_fixture.py b/tests/ingress_proxy_fixture.py index b89cef1..b59fe2a 100644 --- a/tests/ingress_proxy_fixture.py +++ b/tests/ingress_proxy_fixture.py @@ -61,8 +61,18 @@ def names(kind): # Oversize/ambiguous frames are rejected by the real helper before # reading. Forward only the header to avoid a blind fixture read. body = b'' if size > 4 * 1024 * 1024 or len(self.headers.get_all('Content-Length', [])) > 1 or 'Transfer-Encoding' in self.headers else self.rfile.read(size) - connection.endheaders(body) + # A real reverse proxy can receive an early route/auth rejection + # before finishing its upstream body write. macOS exposes this race + # reliably; retain and forward the actual rejection response. An + # absent response or a successful status still fails the fixture. + write_error = None + try: + connection.endheaders(body) + except BrokenPipeError as error: + write_error = error response = connection.getresponse() + if write_error is not None and response.status < 400: + raise write_error response.read() self.send_response(response.status) self.send_header('Content-Length', '0') diff --git a/tests/integration/README.md b/tests/integration/README.md index 2f2ab95..bf351c4 100644 --- a/tests/integration/README.md +++ b/tests/integration/README.md @@ -613,8 +613,7 @@ zig build Expected status includes Vector `active`, `enabled`, `log forwarding healthy (recent stream identity)`, `host metrics flowing`; configured vmagent reports -`active`, `enabled`, `configured targets: 1` and `remote write healthy (recent -application metrics)`. Treat these as expected outputs until observed. +`active`, `enabled`, `configured targets: 1` and `remote write healthy (fresh scrape telemetry; target may be down)`. Treat these as expected outputs until observed. 1. Query stored station metrics/logs for the machine-ID-derived `dt-<32 hex>` host. Inspect the actual CPU/memory/filesystem/inode metric names against diff --git a/tests/integration/application.md b/tests/integration/application.md index 96ad5f7..c0ba93a 100644 --- a/tests/integration/application.md +++ b/tests/integration/application.md @@ -41,7 +41,7 @@ dragontool monitoring apply dragontools-ingestion.service is installed. Expected first apply includes `host metrics flowing`, `selected service logs flowing (quiet-service metadata included)`, - `application metrics flowing`, `probes registered: 1` and + `application metrics pipeline verified (fresh scrape telemetry; target may be down)`, `probes registered: 1` and `application alerts loaded`. Second apply must include `No changes required.` Record PIDs/start timestamps, cert/config hashes and pending markers before and after; unchanged apply must not restart services or rewrite credentials. diff --git a/tests/integration/doers-validation.md b/tests/integration/doers-validation.md new file mode 100644 index 0000000..aefd40a --- /dev/null +++ b/tests/integration/doers-validation.md @@ -0,0 +1,106 @@ +# Doers reference hardening — 2026-09-19 + +**Disposable-host integration not run.** No `monitoring` or `softwarelanding` SSH +connection was made and the production Doers service was not stopped. This follows +the operator's instruction to finish locally and leave deployment validation to +them. Process fixtures do not prove real systemd hardening, journal access, DNS or +provider-firewall reachability. + +## Reference and coverage + +`render_doers.py` reads `examples/doers-monitoring.toml` with the production Zig +parser and renders Vector, vmagent, station documents and base rule packs. The +fixture checks the exact application/environment, SSH aliases, station DNS, +`doers.service`, `127.0.0.1:16005/metrics` and `https://doers.business/healthz`. +Only fixture runtime DNS/probe destinations and filesystem paths become local; +journal input becomes stdin because the container has no systemd journal. The +source TOML is unchanged. No production branch depends on the application name. + +The network-isolated process gate runs the existing reviewed binaries: +VictoriaMetrics **v1.151.0**, VictoriaLogs **v1.52.0**, Caddy **v2.11.4**, Vector +**0.58.0**, vmagent/vmalert **v1.152.0**, blackbox_exporter **0.28.0**. Downloads use +archive and extracted-binary SHA-256 from the existing component modules. No pin +or service unit changes. A local discard HTTP sink receives vmalert notifications; +this fixture does not test Alertmanager delivery, Telegram or human receipt. + +Observed checks: + +- Empty client registry with absent, then empty, app directory; Caddy mTLS on + fixture loopback 9443/9444 and both base rule packs verify without enrollment. + Production IPv4 listener ownership is covered separately by runtime fixtures. +- Native app publication creates only `apps/doers`; app probes never become + station probes. The loaded scraper and both evaluators accept the generated + fragments. No app scrape match is valid before publication or after removal. +- Real Vector host metrics and vmagent payload traverse Caddy metrics ingress to + VM; selected logs traverse logs ingress to VL. Forged application/environment/ + host/service labels are overwritten with the expected identities. +- Quiet metadata alone proves logs flow before any ordinary fixture event. It is + `type=dragontools_stream`, level info; no synthetic application errors are used. +- Closing the local application listener yields real blackbox `probe_success=0` + and vmagent `up=0`. Production readiness checks accept these fresh failed-target + samples. The actual default **120-second** hold is preserved: the alert becomes + pending, fires, then resolves after reopening the listener. +- Identical app publication leaves bytes and mtimes unchanged. Removing the + probe changes only its scrape fragment, metrics rule and manifest, with only + scraper/metrics-evaluator intent. Manual files outside app namespaces, logs + rules, client credentials and agent/Caddy processes remain unchanged. + +Controller/native/temp-file tests separately cover certificate enrollment no-op, +independent restart finalization, cross-app preservation and interrupted recovery. +The Doers fake-remote test checks one enrollment, one initial Vector/vmagent start, +zero base-ingress writes/restarts and no changes on an identical rerun. It is not +an SSH deployment test. + +## Commands actually run + +Local macOS arm64, Zig **0.16.0**: + +```sh +export ZIG_GLOBAL_CACHE_DIR=/tmp/dragontools-zig-cache +zig fmt build.zig src +zig build +zig build test --summary all +python3 tests/cli_smoke.py +python3 -I -B tests/agent_checks_test.py +python3 -I -B tests/app_station_test.py +python3 -I -B tests/agent_journald_test.py +python3 -I -B tests/agent_ingestion_test.py +python3 -I -B tests/agent_ingestion_test.py --caddy /tmp/dragontools-caddy-2.11.4 +python3 -I -B tests/release_test.py +python3 tools/verify_crypto_vendor.py +python3 tests/version_test.py +zig fmt --check build.zig src +git diff --check +python3 -I -B tests/integration/render_doers.py /tmp/dragontools-caddy-pipeline +python3 -I -B tools/fetch_doers_fixture.py --output /tmp/dragontools-caddy-pipeline --arch arm64 +``` + +Results: **390/390 Zig tests**, **269 CLI checks** (Fish unavailable), **9 signal/ +runtime**, **17 app station**, **9 journald lifecycle**, **2 release** tests; vendor +integrity and version/diagnostic checks passed. The first full macOS run exposed a +pre-existing race in the test-only TLS proxy: an upstream route rejection can +arrive before its body write completes. The fixture now reads and requires the +actual rejection response. Broken writes cannot count as success. Empty stderr +assertions and production ingress are unchanged. + +The following ran against an existing local Ubuntu 24.04 arm64 fixture image: + +```sh +docker run --rm --network none --read-only --tmpfs /tmp:rw,exec,size=1g \ + -v "$PWD:/work:ro" -v /tmp/dragontools-caddy-pipeline:/fixture:ro \ + -w /tmp dragontools-pki-test:ubuntu24.04 \ + python3 -I -B /work/tests/integration/doers_runtime.py /fixture +``` + +All seven process gates above passed. All binaries and temporary localhost +credentials were supplied read-only; writable state was disposable `/tmp`. + +## Deployment gate still required + +Follow [the two-host procedure](application.md) using operator-approved hosts. +Record service PIDs/start times, public certificate fingerprints and owned file +hashes before/after the deliberate unchanged apply. Exercise actual journald +collection, stricter administrator limits and capacity on those hosts. Queue and +journal bounds constrain DragonTools-managed storage, not unrelated application +files or unlimited lossless retention; provision disk headroom for both Vector +buffers, vmagent's 1 GiB queue and the configured journal budget. diff --git a/tests/integration/doers_runtime.py b/tests/integration/doers_runtime.py new file mode 100644 index 0000000..822dd0c --- /dev/null +++ b/tests/integration/doers_runtime.py @@ -0,0 +1,293 @@ +"""Isolated Linux reference integration using pinned processes and real mTLS. + +Render with render_doers.py first. Run with --network none in a disposable +container, not on a station. No SSH/systemd, live Doers, journal access, external +notifications or provider changes. The reference's DNS/probe are redirected to +localhost and journal input is replaced with stdin; identity, buffers, relabeling, +rules, scrape/evaluation intervals and the two-minute alert hold are unchanged. +""" +import copy +import datetime +import hashlib +import http.server +import importlib.util +import json +import os +from pathlib import Path +import ssl +import subprocess +import sys +import tempfile +import threading +import time +import urllib.parse + +ROOT = Path(__file__).resolve().parents[2] + + +def load(name, path): + spec = importlib.util.spec_from_file_location(name, path) + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +pipeline = load('pipeline', ROOT / 'tests/integration/agent_ingestion_pipeline.py') +request = pipeline.request + + +def wait(check, label, seconds=45): + deadline = time.monotonic() + seconds + delay = .5 + while True: + try: + result = check() + if result: + return result + except (ConnectionError, TimeoutError, OSError): + pass + left = deadline - time.monotonic() + if left <= 0: + raise AssertionError('Deadline: ' + label) + time.sleep(min(delay, left)) + delay = 1 + + +def snapshot(root): + return {str(p.relative_to(root)): (hashlib.sha256(p.read_bytes()).hexdigest(), p.stat().st_mtime_ns) + for p in root.rglob('*') if p.is_file()} + + +def main(fixture): + for name, component in (('victoria-metrics-prod', 'victoriametrics'), ('victoria-logs-prod', 'victorialogs'), + ('vector', 'vector'), ('vmagent', 'vmagent'), ('caddy', 'caddy'), + ('vmalert', 'vmalert'), ('blackbox_exporter', 'blackbox_exporter')): + digest = hashlib.sha256((fixture / name).read_bytes()).hexdigest() + assert digest in (ROOT / ('src/components/' + component + '.zig')).read_text(), name + ' pin mismatch' + config = json.loads((fixture / 'doers-station.json').read_text()) + registration = json.loads((fixture / 'doers-registration.json').read_text()) + assert json.loads((fixture / 'doers-transport.json').read_text()) == dict( + target_ssh_host='softwarelanding', station_ssh_host='monitoring', station_hostname='monitoring.baptizeddragon.com') + assert config['application'] == 'doers' and config['environment'] == 'production' + assert config['services'] == [dict(name='doers', systemd='doers.service', logs=True, metrics_url='http://127.0.0.1:16005/metrics')] + assert config['probes'] == [dict(name='web', url='https://doers.business/healthz')] + assert registration['services'] == ['doers.service'] + registration['station'] = 'localhost' + config['probes'][0]['url'] = 'http://127.0.0.1:16005/healthz' + n = {'__name__': 'doers_station_fixture'} + for name in ('apps/station_model.py', 'scrape.py', 'vmalert_rules.py', 'apps/station_read.py', 'apps/station_mutate.py'): + exec(compile((ROOT / 'src/monitoring' / name).read_text(), name, 'exec'), n) + signals = load('signals', ROOT / 'src/monitoring/agents/signals.py') + health = load('ingress_health', ROOT / 'src/monitoring/ingress_health.py') + ingestion = load('ingestion', ROOT / 'src/monitoring/agents/ingestion.py') + proxy = load('proxy_fixture', ROOT / 'tests/ingress_proxy_fixture.py') + processes, servers = [], [] + ingress = None + with tempfile.TemporaryDirectory(prefix='doers-runtime-') as temporary: + root = Path(temporary) + apps, registry = root / 'apps', root / 'registry' + registry.mkdir() + main_config = root / 'prometheus.yml' + n['APP_ROOT'] = str(apps) + n['APP_VM_CONFIG'] = str(main_config) + n['APP_INCLUDE'] = "scrape_config_files: ['" + str(apps / '*' / 'scrape.yml') + "']\n" + n['APP_PENDING'] = {name: str(root / (name + '.pending')) for name in n['APP_FILES']} + n['APP_VM_BINARY'] = str(fixture / 'victoria-metrics-prod') + n['APP_ALERT_BINARY'] = str(fixture / 'vmalert') + main_config.write_text('# Managed by DragonTools\nglobal:\n scrape_interval: 30s\n scrape_timeout: 5s\n' + n['APP_INCLUDE'] + 'scrape_configs: []\n') + main_config.chmod(0o644) + # These represent administrator files outside the app glob. Never passed + # to an app renderer, never adopted as managed inputs. + manual = root / 'manual'; manual.mkdir() + (manual / 'probes.yml').write_text('administrator-owned probe\n') + (manual / 'rules.yml').write_text('administrator-owned rules\n') + protected = snapshot(manual) + credentials = snapshot(fixture / 'certs') + def start(argv, stdin=False): + p = subprocess.Popen(argv, stdin=subprocess.PIPE if stdin else subprocess.DEVNULL, + stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, + env=dict(os.environ, GOMAXPROCS='2')) + processes.append(p) + return p + def query(expression): + return json.loads(request(8428, 'GET', '/api/v1/query?' + urllib.parse.urlencode(dict(query=expression, nocache=1))))['data']['result'] + def checked(fn): + try: + fn() + return True + except n['NotReady']: + return False + def rules(kind): + return json.loads(request(8880 if kind == 'logs' else 8881, 'GET', '/api/v1/rules')) + def base_ready(kind): + response = rules(kind) + # Only fixture file locations differ from the production policy. + for group in response['data']['groups']: + if group['file'] == str(fixture / ('base-' + kind + '.rules.yml')): + group['file'] = '/etc/dragontools/vmalert-' + kind + '/rules.yml' + for rule in group['rules']: + rule['file'] = group['file'] + return checked(lambda: n['validate'](response, kind)) + def evaluator(kind): + return start([str(fixture / 'vmalert'), '-rule=' + str(fixture / ('base-' + kind + '.rules.yml')), + '-rule=' + str(apps / '*' / (kind + '.rules.yml')), + '-datasource.url=http://127.0.0.1:' + ('9428' if kind == 'logs' else '8428'), + '-notifier.url=http://127.0.0.1:19093', '-httpListenAddr=127.0.0.1:' + ('8880' if kind == 'logs' else '8881'), + '-group.maxStartDelay=1s', '-loggerLevel=ERROR']) + def application(): + server = http.server.ThreadingHTTPServer(('127.0.0.1', 16005), pipeline.Application) + servers.append(server) + threading.Thread(target=server.serve_forever, daemon=True).start() + return server + class NotificationSink(http.server.BaseHTTPRequestHandler): + # Local discard receiver; no Telegram or outside request. + def do_POST(self): + self.rfile.read(int(self.headers.get('Content-Length', 0))) + self.send_response(200); self.end_headers() + def log_message(self, *_): + pass + try: + sink = http.server.ThreadingHTTPServer(('127.0.0.1', 19093), NotificationSink) + servers.append(sink) + threading.Thread(target=sink.serve_forever, daemon=True).start() + vm = start([str(fixture / 'victoria-metrics-prod'), '-httpListenAddr=127.0.0.1:8428', '-storageDataPath=' + str(root / 'vm'), + '-promscrape.config=' + str(main_config), '-search.latencyOffset=0s', '-memory.allowedBytes=67108864']) + start([str(fixture / 'victoria-logs-prod'), '-httpListenAddr=127.0.0.1:9428', '-storageDataPath=' + str(root / 'vl'), '-memory.allowedBytes=67108864']) + for port in (8428, 9428): + wait(lambda: request(port, 'GET', '/health') is not None, 'backend health') + ingress = proxy.Harness(root, fixture / 'certs', registry, ingestion, dict(metrics=8428, logs=9428), str(fixture / 'caddy'), ports=dict(metrics=9443, logs=9444)) + assert list(registry.iterdir()) == [] and not apps.exists() + health.tls(fixture / 'certs', 'localhost', (9443, 9444)) + health.authorization([str(root / 'metrics.sock'), str(root / 'logs.sock')]) + evaluators = {kind: evaluator(kind) for kind in ('logs', 'metrics')} + for kind in evaluators: + wait(lambda: base_ready(kind), 'zero-app base ' + kind + ' rules') + wait(lambda: checked(lambda: n['app_probe_ready']()), 'empty scrape glob') + apps.mkdir(mode=0o755) + assert list(apps.glob('*/scrape.yml')) == [] + assert checked(lambda: n['app_probe_ready']()) + print('PASS: zero clients, absent/empty app tree, Caddy 9443/9444 mTLS and healthy base rules.', flush=True) + + cert = (fixture / 'certs/client.crt').read_text() + registration.update(certificate_sha256=hashlib.sha256(ssl.PEM_cert_to_DER_cert(cert)).hexdigest(), + certificate_identity='dragontools://hosts/' + registration['host']) + record = registry / (registration['host'] + '.json') + record.write_text(json.dumps(registration)); record.chmod(0o640) + caddy_pid = ingress.process.pid + assert n['app_publish'](config) + assert sorted(p.name for p in apps.iterdir()) == ['doers'] + assert n['app_base_probes']() == [] # App TOML never becomes a station probe. + for kind in evaluators: + evaluators[kind].terminate(); evaluators[kind].wait(timeout=10) + evaluators[kind] = evaluator(kind) + start([str(fixture / 'blackbox_exporter'), '--config.file=' + str(fixture / 'blackbox.yml'), + '--web.listen-address=127.0.0.1:9115', '--history.limit=0', '--log.prober=error']) + app = application() + request(8428, 'POST', '/-/reload') + for kind in evaluators: + wait(lambda: checked(lambda: n['app_rules_ready'](config, (kind,))), 'scoped ' + kind + ' rules') + wait(lambda: checked(lambda: n['app_probe_ready']()), 'app probe pipeline') + expected = n['application_definitions']() + wait(lambda: list(n['stored_states'](expected, True).values()) == ['healthy'], 'healthy blackbox probe') + for path in n['APP_PENDING'].values(): + Path(path).unlink(missing_ok=True) # Fixture activation finished; no systemd claim. + print('PASS: only Doers namespace published; live blackbox scraping and both scoped rule APIs ready.', flush=True) + + original = (fixture / 'doers-vector.yaml').read_text() + assert original.count('max_size: 268435488') == 2 and original.count('when_full: block') == 2 + subprocess.run([str(fixture / 'vector'), 'validate', '--no-environment', '--skip-healthchecks', str(fixture / 'doers-vector.yaml')], check=True, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL) + begin, end = original.index(' journal:\n'), original.index(' stream_0:\n') + vector_config = original[:begin] + ' journal:\n type: stdin\n decoding:\n codec: json\n' + original[end:] + for old, new in (('/var/lib/dragontools/vector', str(root / 'vector')), ('/etc/dragontools/vector', str(fixture / 'certs')), + ('monitoring.baptizeddragon.com', 'localhost'), ('/opt/dragontools/agent/current/dragontool-agent', str(fixture / 'dragontool-agent'))): + vector_config = vector_config.replace(old, new) + (root / 'vector').mkdir() + (root / 'vector.yaml').write_text(vector_config) + since = time.time() + vector = start([str(fixture / 'vector'), '--config', str(root / 'vector.yaml')], stdin=True) + wait(lambda: signals.check('host', registration, since), 'trusted host metrics') + wait(lambda: signals.check('logs', registration, since), 'quiet stream metadata') + def logs(query): + body = request(9428, 'POST', '/select/logsql/query', urllib.parse.urlencode(dict(query=query)), headers={'Content-Type': 'application/x-www-form-urlencoded'}) + return [json.loads(line) for line in body.splitlines()] + quiet = logs('application:="doers" service:="doers" | limit 10') + assert quiet and all(row['type'] == 'dragontools_stream' and row['level'] == 'info' for row in quiet) + trusted = dict(application='doers', environment='production', host=registration['host'], service='doers') + assert all(all(row[k] == v for k, v in trusted.items()) for row in quiet) + event = dict(_SYSTEMD_UNIT='doers.service', message=json.dumps(dict(application='forged', environment='forged', host='forged', service='forged', level='info', request_id='reference-fixture', message='normal fixture event')), PRIORITY='6') + vector.stdin.write((json.dumps(event) + '\n').encode()); vector.stdin.flush() + actual = wait(lambda: logs('request_id:="reference-fixture" | limit 1'), 'normal structured log') + assert all(actual[0][k] == v for k, v in trusted.items()) + vm_since = time.time() + vmagent = start([str(fixture / 'vmagent'), '-httpListenAddr=127.0.0.1:8429', '-promscrape.config=' + str(fixture / 'doers-prometheus.yml'), + '-remoteWrite.url=https://localhost:9443/api/v1/write', '-remoteWrite.forcePromProto=true', + '-remoteWrite.tlsCAFile=' + str(fixture / 'certs/ca.crt'), '-remoteWrite.tlsCertFile=' + str(fixture / 'certs/client.crt'), + '-remoteWrite.tlsKeyFile=' + str(fixture / 'certs/client.key'), '-remoteWrite.tmpDataPath=' + str(root / 'vmagent'), '-remoteWrite.maxDiskUsagePerURL=1GiB']) + wait(lambda: signals.check('app', registration, vm_since), 'application metrics') + selector = '{' + ','.join(k + '=' + json.dumps(v) for k, v in trusted.items()) + '}' + assert query('fixture_requests_total' + selector) + assert not query('fixture_requests_total{application="forged"}') + assert not signals.check('app', registration, time.time() + 3600) + print('PASS: Vector host/logs and vmagent metrics traverse real Caddy with trusted labels; quiet logs need no fake error.', flush=True) + + def alert_state(): + group = next(g for g in rules('metrics')['data']['groups'] if g['name'] == 'dragontools-app-doers-metrics') + return group['rules'][0] + app.shutdown(); app.server_close(); servers.remove(app) + wait(lambda: list(n['stored_states'](expected, True).values()) == ['unhealthy'], 'probe_success=0') + wait(lambda: query('up' + selector + ' == 0'), 'vmagent failed scrape') + assert signals.check('app', registration, vm_since) + assert checked(lambda: n['app_probe_ready']()) + pending = wait(lambda: (r if (r := alert_state())['state'] == 'pending' else None), 'pending alert', 65) + assert pending['duration'] == 120 and pending['health'] == 'ok' + print('PASS: stopped fixture target has probe_success=0 and up=0; monitoring readiness passes, alert pending for 2m.', flush=True) + firing = wait(lambda: (r if (r := alert_state())['state'] == 'firing' else None), 'firing alert after configured hold', 180) + active = datetime.datetime.fromisoformat(firing['alerts'][0]['activeAt'].replace('Z', '+00:00')).timestamp() + assert time.time() - active >= 120 + app = application() + wait(lambda: list(n['stored_states'](expected, True).values()) == ['healthy'], 'probe recovery') + wait(lambda: alert_state()['state'] == 'inactive', 'alert resolution', 75) + wait(lambda: query('up' + selector + ' == 1') and signals.check('app', registration, vm_since), 'metrics recovery') + print('PASS: ServiceProbeFailed fired after the real 2m hold, then resolved after target recovery.', flush=True) + + before = snapshot(apps) + assert not n['app_publish'](config) and snapshot(apps) == before + assert snapshot(fixture / 'certs') == credentials + reduced = copy.deepcopy(config); reduced['probes'] = [] + logs_before = snapshot(apps)['doers/logs.rules.yml'] + assert n['app_publish'](reduced) + assert snapshot(apps)['doers/logs.rules.yml'] == logs_before + assert set(p.name for p in root.glob('*.pending')) == {'metrics.rules.yml.pending', 'scrape.yml.pending'} + request(8428, 'POST', '/-/reload') + evaluators['metrics'].terminate(); evaluators['metrics'].wait(timeout=10) + evaluators['metrics'] = evaluator('metrics') + wait(lambda: checked(lambda: n['app_probe_ready']()), 'removed app probe') + wait(lambda: checked(lambda: n['app_rules_ready'](reduced)), 'removed app rule') + assert not json.loads(request(8428, 'GET', '/api/v1/targets?state=active'))['data']['activeTargets'] + assert snapshot(manual) == protected and n['app_base_probes']() == [] + after = snapshot(apps) + assert not n['app_publish'](reduced) and snapshot(apps) == after + assert all(p.poll() is None for p in (vm, vector, vmagent, ingress.process)) + assert ingress.process.pid == caddy_pid and snapshot(fixture / 'certs') == credentials + assert not logs('application:="doers" level:in(error,critical,fatal) | limit 1') + print('PASS: identical publication is a no-op; probe removal only dirties its scraper/metrics rules; agents, Caddy, credentials and manual files preserved.', flush=True) + if '--outage' in sys.argv: + load('vector_outage', ROOT / 'tests/integration/vector_outage.py').exercise( + vector, ingress.process, root / 'vector', request, wait, logs) + finally: + for p in reversed(processes): + if p.poll() is None: + p.terminate() + for p in reversed(processes): + try: + p.wait(timeout=10) + except subprocess.TimeoutExpired: + p.kill(); p.wait() + if ingress: + ingress.close() + for server in servers: + server.shutdown(); server.server_close() + + +if __name__ == '__main__': + main(Path(sys.argv[1]).resolve()) diff --git a/tests/integration/render_doers.py b/tests/integration/render_doers.py new file mode 100644 index 0000000..57a1cf4 --- /dev/null +++ b/tests/integration/render_doers.py @@ -0,0 +1,67 @@ +"""Render the committed Doers reference through production Zig parsers/renderers. + +Local files only. No SSH, secret lookup, DNS, or production endpoint requests. +Usage: python3 -I -B tests/integration/render_doers.py /tmp/doers-fixture +""" +from pathlib import Path +import subprocess +import sys +import tempfile + +ROOT = Path(__file__).resolve().parents[2] +SOURCE = r''' +const std = @import("std"); +const application = @import("config/application.zig"); +const config = @import("monitoring/agents/config.zig"); +const model = @import("monitoring/agents/model.zig"); +fn write(init: std.process.Init, directory: []const u8, name: []const u8, bytes: []const u8) !void { + const path = try std.fmt.allocPrint(init.arena.allocator(), "{s}/{s}", .{ directory, name }); + const file = try std.Io.Dir.cwd().createFile(init.io, path, .{}); + defer file.close(init.io); + try file.writeStreamingAll(init.io, bytes); +} +pub fn main(init: std.process.Init) !void { + const a = init.arena.allocator(); + const args = try init.minimal.args.toSlice(a); + if (args.len != 2) return error.OutputDirectoryRequired; + var value = try application.load(a, init.io, "examples/doers-monitoring.toml"); + defer value.deinit(); + const services = try a.alloc(model.AppService, value.services.len); + var units: std.ArrayList([]const u8) = .empty; + for (value.services, services) |service, *selected| { + selected.* = .{ .name = service.name, .systemd = service.systemd, .logs = service.logs, .metrics_url = service.metrics_url }; + if (service.logs) try units.append(a, service.systemd); + } + const registration: model.Registration = .{ .host = "dt-aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", .station = value.station_hostname, + .services = units.items, .metrics_targets = &.{}, .applications = &.{.{ .name = value.application.name, .environment = value.application.environment, .services = services }} }; + try write(init, args[1], "doers-registration.json", try registration.json(a)); + try write(init, args[1], "doers-vector.yaml", try config.renderVectorRegistration(a, registration)); + try write(init, args[1], "doers-prometheus.yml", try config.renderVmagentRegistration(a, registration)); + try write(init, args[1], "doers-station.json", try std.json.Stringify.valueAlloc(a, .{ + .application = value.application.name, .environment = value.application.environment, .host = registration.host, + .services = value.services, .probes = value.probes, .alerts = value.alerts, + }, .{})); + try write(init, args[1], "doers-transport.json", try std.json.Stringify.valueAlloc(a, .{ + .target_ssh_host = value.target_ssh_host, .station_ssh_host = value.station_ssh_host, .station_hostname = value.station_hostname, + }, .{})); + try write(init, args[1], "blackbox.yml", @import("components/blackbox_exporter.zig").config); + try write(init, args[1], "base-logs.rules.yml", try @import("monitoring/vmalert.zig").rules(a, .logs)); + try write(init, args[1], "base-metrics.rules.yml", try @import("monitoring/vmalert.zig").rules(a, .metrics)); +} +''' + + +def main(): + output = Path(sys.argv[1]).resolve() + output.mkdir(parents=True, exist_ok=True) + with tempfile.NamedTemporaryFile(mode='w', suffix='.zig', prefix='.doers-fixture-', dir=ROOT / 'src', delete=False) as source: + source.write(SOURCE) + try: + subprocess.run(['zig', 'run', source.name, '--', str(output)], cwd=ROOT, check=True) + finally: + Path(source.name).unlink() + print('PASS: Doers reference parsed and rendered through production code (no network).') + + +if __name__ == '__main__': + main() diff --git a/tests/integration/vector_outage.py b/tests/integration/vector_outage.py new file mode 100644 index 0000000..cc4a9c5 --- /dev/null +++ b/tests/integration/vector_outage.py @@ -0,0 +1,68 @@ +"""Opt-in saturation check inside the isolated Doers process fixture. + +Pause only that fixture's Caddy, fill the production Vector log buffer, observe +backpressure and bounded data files, then resume delivery. No systemd/real disks. +""" +import json +import os +from pathlib import Path +import select +import signal +import time + +LIMIT = 268435488 + + +def exercise(vector, caddy, data_dir, request, wait, logs): + def buffer_size(): + text = request(8686, 'GET', '/metrics').decode() + rows = [line for line in text.splitlines() if line.startswith('vector_buffer_size_bytes{') and 'component_id="logs"' in line] + return max((float(line.rsplit(' ', 1)[1]) for line in rows), default=0) + # Keep each line bounded and ordinary info-level; it is synthetic fixture + # traffic, never an application error or installer-generated event. + event = dict(_SYSTEMD_UNIT='doers.service', PRIORITY='6', message=json.dumps(dict( + message='x' * 16000, level='info', request_id='outage-fixture'))) + payload = (json.dumps(event) + '\n').encode() + descriptor = vector.stdin.fileno() + os.set_blocking(descriptor, False) + os.kill(caddy.pid, signal.SIGSTOP) + offset = sent = 0 + full_since = None + sizes = [] + deadline = time.monotonic() + 120 + try: + while True: + assert vector.poll() is None and caddy.poll() is None + _, writable, _ = select.select([], [descriptor], [], .1) + if writable: + try: + count = os.write(descriptor, payload[offset:]) + sent += count + offset = (offset + count) % len(payload) + except BlockingIOError: + pass + now = time.monotonic() + if not sizes or now - sizes[-1][0] >= 1: + size = buffer_size() + disk = sum(p.stat().st_size for p in Path(data_dir).rglob('*') if p.is_file()) + # Two configured disk buffers plus bounded ledger/segment slack. + assert disk < 2 * LIMIT + 16 * 1024**2, 'Vector disk exceeded configured budget' + sizes.append((now, size, disk, sent)) + if size >= LIMIT - 1024 * 1024 and full_since is None: + full_since = now + if full_since is not None and now - full_since >= 5: + # Source progress must stall while the full queue blocks. + assert sizes[-1][3] == sizes[-3][3], 'Full buffer did not backpressure stdin' + break + if now >= deadline: + raise AssertionError('Vector buffer saturation deadline') + finally: + os.kill(caddy.pid, signal.SIGCONT) + os.set_blocking(descriptor, True) + # Finish a partial source line after backpressure releases; stdin stays open. + if offset: + vector.stdin.write(payload[offset:]); vector.stdin.flush() + wait(lambda: buffer_size() < LIMIT // 2, 'Vector buffer drains after station recovery', 90) + wait(lambda: logs('request_id:="outage-fixture" | limit 1'), 'queued logs arrive after outage', 45) + assert vector.poll() is None and caddy.poll() is None + print('PASS: station outage saturates bounded Vector buffer, blocks source reads, and resumes delivery without agent restart.', flush=True) diff --git a/tools/fetch_doers_fixture.py b/tools/fetch_doers_fixture.py new file mode 100644 index 0000000..555a498 --- /dev/null +++ b/tools/fetch_doers_fixture.py @@ -0,0 +1,72 @@ +#!/usr/bin/env python3 +"""Explicit opt-in Linux fixture download using the existing production pins. + +Never invoked by zig build/test. No version discovery or system installation. +Only the named regular archive member is extracted; both digests must match. +""" +import argparse +import hashlib +import io +from pathlib import Path +import platform +import re +import tarfile +import urllib.request + +ROOT = Path(__file__).resolve().parents[1] +# (production component, local fixture name, upstream repo, archive name, member) +ARTIFACTS = ( + ('victoriametrics', 'victoria-metrics-prod', 'VictoriaMetrics/VictoriaMetrics', 'victoria-metrics-linux-{arch}-{version}.tar.gz', 'victoria-metrics-prod'), + ('victorialogs', 'victoria-logs-prod', 'VictoriaMetrics/VictoriaLogs', 'victoria-logs-linux-{arch}-{version}.tar.gz', 'victoria-logs-prod'), + ('vmalert', 'vmalert', 'VictoriaMetrics/VictoriaMetrics', 'vmutils-linux-{arch}-{version}.tar.gz', 'vmalert-prod'), + ('vmagent', 'vmagent', 'VictoriaMetrics/VictoriaMetrics', 'vmutils-linux-{arch}-{version}.tar.gz', 'vmagent-prod'), + ('vector', 'vector', 'vectordotdev/vector', 'vector-{bare}-{cpu}-unknown-linux-musl.tar.gz', './vector-{cpu}-unknown-linux-musl/bin/vector'), + ('blackbox_exporter', 'blackbox_exporter', 'prometheus/blackbox_exporter', 'blackbox_exporter-{bare}.linux-{arch}.tar.gz', 'blackbox_exporter-{bare}.linux-{arch}/blackbox_exporter'), + ('caddy', 'caddy', 'caddyserver/caddy', 'caddy_{bare}_linux_{arch}.tar.gz', 'caddy'), +) + + +def fetch(directory, arch): + directory.mkdir(parents=True, exist_ok=True) + cache = {} + for component, name, repo, archive, member in ARTIFACTS: + source = (ROOT / ('src/components/' + component + '.zig')).read_text() + version = re.search(r'pub const version = "([v0-9.]+)";', source)[1] + block = re.search(r'\.' + arch + r' => \.\{(.*?)\}', source, re.S)[1] + archive_hash = re.search(r'\.archive_sha256 = "([a-f0-9]{64})"', block)[1] + binary_hash = re.search(r'\.binary_sha256 = "([a-f0-9]{64})"', block)[1] + path = directory / name + if path.is_symlink(): + raise ValueError('Symlink fixture refused') + if path.exists(): + if not path.is_file() or hashlib.sha256(path.read_bytes()).hexdigest() != binary_hash: + raise ValueError('Existing fixture pin mismatch: ' + name) + continue + fields = dict(version=version, bare=version.removeprefix('v'), arch=arch, cpu='aarch64' if arch == 'arm64' else 'x86_64') + url = 'https://github.com/' + repo + '/releases/download/v' + fields['bare'] + '/' + archive.format(**fields) + if url not in cache: + with urllib.request.urlopen(url, timeout=180) as response: + data = response.read(200_000_001) + if len(data) > 200_000_000 or hashlib.sha256(data).hexdigest() != archive_hash: + raise ValueError('Archive pin mismatch: ' + name) + cache[url] = data + data = cache[url] + if hashlib.sha256(data).hexdigest() != archive_hash: + raise ValueError('Shared archive pin mismatch') + with tarfile.open(fileobj=io.BytesIO(data), mode='r:gz') as tar: + entry = tar.getmember(member.format(**fields)) + if not entry.isfile() or entry.size > 200_000_000: + raise ValueError('Invalid regular archive member') + binary = tar.extractfile(entry).read() + if hashlib.sha256(binary).hexdigest() != binary_hash: + raise ValueError('Binary pin mismatch: ' + name) + path.write_bytes(binary); path.chmod(0o755) + print('PASS: all seven Linux fixture binaries match production pins.') + + +if __name__ == '__main__': + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument('--output', type=Path, required=True) + parser.add_argument('--arch', choices=('arm64', 'amd64'), default='arm64' if platform.machine() in ('arm64', 'aarch64') else 'amd64') + args = parser.parse_args() + fetch(args.output, args.arch) From 4c85f755e877055e12bd9aa09f448706c6728a23 Mon Sep 17 00:00:00 2001 From: Vasyl Osypchuk Date: Sat, 19 Sep 2026 07:43:19 +0300 Subject: [PATCH 2/7] Observe controlled alert baseline and pending before signal polling --- tests/integration/doers_runtime.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/tests/integration/doers_runtime.py b/tests/integration/doers_runtime.py index 822dd0c..85262d3 100644 --- a/tests/integration/doers_runtime.py +++ b/tests/integration/doers_runtime.py @@ -233,13 +233,16 @@ def logs(query): def alert_state(): group = next(g for g in rules('metrics')['data']['groups'] if g['name'] == 'dragontools-app-doers-metrics') return group['rules'][0] + wait(lambda: alert_state()['state'] == 'inactive', 'healthy alert baseline', 90) app.shutdown(); app.server_close(); servers.remove(app) + # Observe pending first, before separate signal queries. Otherwise + # their polling can miss part of the hold on a loaded CI runner. + pending = wait(lambda: (r if (r := alert_state())['state'] == 'pending' else None), 'pending alert', 90) + assert pending['duration'] == 120 and pending['health'] == 'ok' wait(lambda: list(n['stored_states'](expected, True).values()) == ['unhealthy'], 'probe_success=0') wait(lambda: query('up' + selector + ' == 0'), 'vmagent failed scrape') assert signals.check('app', registration, vm_since) assert checked(lambda: n['app_probe_ready']()) - pending = wait(lambda: (r if (r := alert_state())['state'] == 'pending' else None), 'pending alert', 65) - assert pending['duration'] == 120 and pending['health'] == 'ok' print('PASS: stopped fixture target has probe_success=0 and up=0; monitoring readiness passes, alert pending for 2m.', flush=True) firing = wait(lambda: (r if (r := alert_state())['state'] == 'firing' else None), 'firing alert after configured hold', 180) active = datetime.datetime.fromisoformat(firing['alerts'][0]['activeAt'].replace('Z', '+00:00')).timestamp() From bd0ac004d2bfab9639a037c0bf048cc03df7bf6b Mon Sep 17 00:00:00 2001 From: Vasyl Osypchuk Date: Sat, 19 Sep 2026 07:53:27 +0300 Subject: [PATCH 3/7] Read TLS client-auth rejection before HTTP writes and bound outage recovery checks --- CHANGELOG.md | 2 ++ src/monitoring/ingress_health.py | 10 +++++++--- tests/agent_checks_test.py | 25 +++++++++++++++++++++++++ tests/integration/doers-validation.md | 11 +++++++++-- tests/integration/doers_runtime.py | 4 ++++ tests/integration/vector_outage.py | 21 +++++++++++++++++---- 6 files changed, 64 insertions(+), 9 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 647a965..4637866 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,8 @@ - Accept fresh vmagent failed-scrape telemetry when the application is down; require fresh non-scrape payload on successful targets and reject stale/missing data. Report pipeline readiness independently of application availability. +- Read the post-handshake TLS client-auth alert before sending HTTP, avoiding + OpenSSL write-side EOF races without accepting EOF/timeouts as proof of mTLS. - Give app publication, scraper reload and finalization fixed semantic check IDs; do not suggest reinstalling ingress for app namespace/rule failures. - Add the production-rendered Doers reference process gate: zero-app station, diff --git a/src/monitoring/ingress_health.py b/src/monitoring/ingress_health.py index 02fae98..0f7dded 100644 --- a/src/monitoring/ingress_health.py +++ b/src/monitoring/ingress_health.py @@ -28,13 +28,17 @@ def tls(server, hostname, ports): try: with socket.create_connection(('127.0.0.1', port), timeout=3) as raw: with context.wrap_socket(raw, server_hostname=hostname) as conn: - conn.sendall(b'GET /health HTTP/1.1\r\nHost: localhost\r\nConnection: close\r\n\r\n') - conn.recv(1) # TLS 1.3's client-auth alert follows its handshake. + # TLS 1.3's client-auth alert follows the handshake. Read + # it before writing HTTP data: OpenSSL can otherwise expose + # a write-side EOF instead of the authenticated TLS alert. + # A silent peer, ordinary EOF or application data never + # proves mandatory client authentication. + conn.recv(1) except ssl.SSLCertVerificationError: raise except ssl.SSLError as error: # TLS 1.3 and TLS 1.2 names are stable SSL alert semantics, not text. - if error.reason not in ('TLSV13_ALERT_CERTIFICATE_REQUIRED', 'SSLV3_ALERT_HANDSHAKE_FAILURE'): + if getattr(error, 'reason', None) not in ('TLSV13_ALERT_CERTIFICATE_REQUIRED', 'SSLV3_ALERT_HANDSHAKE_FAILURE'): raise else: raise ValueError('Client authentication not enforced') diff --git a/tests/agent_checks_test.py b/tests/agent_checks_test.py index e807515..859b4a5 100644 --- a/tests/agent_checks_test.py +++ b/tests/agent_checks_test.py @@ -231,6 +231,31 @@ def test_station_unavailable_retries_but_invalid_responses_fail_deterministicall self.assertEqual(self.run_signal('host', oversized=True)[0], 1) +class StationTlsProof(unittest.TestCase): + def test_only_explicit_client_certificate_alert_proves_mtls(self): + namespace = {'__name__': 'fixture'} + exec(compile((ROOT / 'src/monitoring/ingress_health.py').read_text(), 'ingress_health.py', 'exec'), namespace) + class Alert(ssl.SSLError): + reason = 'TLSV13_ALERT_CERTIFICATE_REQUIRED' + class WrongAlert(ssl.SSLError): + reason = 'TLSV1_ALERT_INTERNAL_ERROR' + for outcome in (Alert(), WrongAlert(), ssl.SSLEOFError(), ssl.SSLCertVerificationError(), TimeoutError(), b'', b'H'): + with self.subTest(outcome=type(outcome).__name__): + def receive(_): + if isinstance(outcome, Exception): + raise outcome + return outcome + peer = types.SimpleNamespace(recv=receive, sendall=lambda _: self.fail('write can obscure TLS 1.3 alert')) + context = types.SimpleNamespace(wrap_socket=lambda raw, server_hostname: contextlib.nullcontext(peer)) + with patch('ssl.create_default_context', return_value=context), \ + patch('socket.create_connection', return_value=contextlib.nullcontext()): + if isinstance(outcome, Alert): + namespace['tls'](Path('/public-fixture'), 'localhost', (9443, 9444)) + else: + with self.assertRaises((ssl.SSLError, TimeoutError, ValueError)): + namespace['tls'](Path('/public-fixture'), 'localhost', (9443, 9444)) + + class ApplicationSignals(unittest.TestCase): def module(self): namespace = {'__name__': 'fixture'} diff --git a/tests/integration/doers-validation.md b/tests/integration/doers-validation.md index aefd40a..35adab1 100644 --- a/tests/integration/doers-validation.md +++ b/tests/integration/doers-validation.md @@ -75,13 +75,20 @@ python3 -I -B tests/integration/render_doers.py /tmp/dragontools-caddy-pipeline python3 -I -B tools/fetch_doers_fixture.py --output /tmp/dragontools-caddy-pipeline --arch arm64 ``` -Results: **390/390 Zig tests**, **269 CLI checks** (Fish unavailable), **9 signal/ +Results: **390/390 Zig tests**, **269 CLI checks** (Fish unavailable), **10 signal/ runtime**, **17 app station**, **9 journald lifecycle**, **2 release** tests; vendor integrity and version/diagnostic checks passed. The first full macOS run exposed a pre-existing race in the test-only TLS proxy: an upstream route rejection can arrive before its body write completes. The fixture now reads and requires the actual rejection response. Broken writes cannot count as success. Empty stderr -assertions and production ingress are unchanged. +assertions and production ingress routing are unchanged. + +Linux Actions also exposed a TLS 1.3 write-side EOF in station health: the server +has already rejected the certificate-less handshake when the verifier writes its +HTTP request. Health now reads the post-handshake alert first; only the explicit +certificate-required/handshake-failure alerts count. Ordinary EOF, unexpected +alerts, a silent peer or application data still fail. Real Caddy and the TLS +stand-in pass on Ubuntu 24.04 and macOS with this change. The following ran against an existing local Ubuntu 24.04 arm64 fixture image: diff --git a/tests/integration/doers_runtime.py b/tests/integration/doers_runtime.py index 85262d3..c1c62b8 100644 --- a/tests/integration/doers_runtime.py +++ b/tests/integration/doers_runtime.py @@ -229,6 +229,10 @@ def logs(query): assert not query('fixture_requests_total{application="forged"}') assert not signals.check('app', registration, time.time() + 3600) print('PASS: Vector host/logs and vmagent metrics traverse real Caddy with trusted labels; quiet logs need no fake error.', flush=True) + if '--outage-only' in sys.argv: + load('vector_outage', ROOT / 'tests/integration/vector_outage.py').exercise( + vector, ingress.process, root / 'vector', request, wait, logs) + return def alert_state(): group = next(g for g in rules('metrics')['data']['groups'] if g['name'] == 'dragontools-app-doers-metrics') diff --git a/tests/integration/vector_outage.py b/tests/integration/vector_outage.py index cc4a9c5..e311d39 100644 --- a/tests/integration/vector_outage.py +++ b/tests/integration/vector_outage.py @@ -53,16 +53,29 @@ def buffer_size(): if full_since is not None and now - full_since >= 5: # Source progress must stall while the full queue blocks. assert sizes[-1][3] == sizes[-3][3], 'Full buffer did not backpressure stdin' + print('PASS: Vector outage buffer saturated at ' + str(int(size)) + + ' bytes; data files ' + str(disk) + ' bytes; source backpressure observed.', flush=True) break if now >= deadline: raise AssertionError('Vector buffer saturation deadline') finally: os.kill(caddy.pid, signal.SIGCONT) - os.set_blocking(descriptor, True) # Finish a partial source line after backpressure releases; stdin stays open. - if offset: - vector.stdin.write(payload[offset:]); vector.stdin.flush() - wait(lambda: buffer_size() < LIMIT // 2, 'Vector buffer drains after station recovery', 90) + def finish_line(): + nonlocal offset + if offset: + try: + offset = (offset + os.write(descriptor, payload[offset:])) % len(payload) + except BlockingIOError: + return False + return offset == 0 + try: + wait(finish_line, 'Vector source resumes after station recovery', 90) + finally: + os.set_blocking(descriptor, True) + # Prove resumed consumption, not an arbitrary throughput benchmark against + # shared CI CPUs. Retrying connections and exporter refresh remain bounded. + wait(lambda: buffer_size() < sizes[-1][1] - 1024 * 1024, 'Vector buffer drains after station recovery', 90) wait(lambda: logs('request_id:="outage-fixture" | limit 1'), 'queued logs arrive after outage', 45) assert vector.poll() is None and caddy.poll() is None print('PASS: station outage saturates bounded Vector buffer, blocks source reads, and resumes delivery without agent restart.', flush=True) From 567bd1d1283885503b243e2335f476437a2b88c4 Mon Sep 17 00:00:00 2001 From: Vasyl Osypchuk Date: Sat, 19 Sep 2026 08:00:28 +0300 Subject: [PATCH 4/7] Verify actual Vector queue bounds and recovery in Linux CI --- .github/workflows/ci.yml | 4 ++-- tests/integration/doers-validation.md | 34 +++++++++++++++++++++++++-- tests/integration/vector_outage.py | 18 +++++++++----- 3 files changed, 46 insertions(+), 10 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index b6f774c..e0468a5 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -24,7 +24,7 @@ jobs: python3 -I -B tools/fetch_caddy_fixture.py --output "$RUNNER_TEMP/dragontools-caddy-2.11.4" python3 -I -B tests/agent_ingestion_test.py --caddy "$RUNNER_TEMP/dragontools-caddy-2.11.4" - run: python3 -I -B tests/release_test.py - - name: Doers reference signals and alert recovery (isolated Linux processes) + - name: Doers signals, alerts and outage recovery (isolated Linux processes) if: runner.os == 'Linux' timeout-minutes: 12 run: | @@ -33,7 +33,7 @@ jobs: python3 -I -B tools/fetch_doers_fixture.py --output "$fixture" cp zig-out/bin/dragontool-agent "$fixture/dragontool-agent" python3 -I -B tests/integration/agent_ingestion_pipeline.py --prepare-credentials "$fixture" - sudo unshare --net sh -c 'ip link set lo up; exec python3 -I -B tests/integration/doers_runtime.py "$1"' fixture "$fixture" + sudo unshare --net sh -c 'ip link set lo up; exec python3 -I -B tests/integration/doers_runtime.py "$1" --outage' fixture "$fixture" - name: Linux helper filesystem lifecycle (temporary paths only) if: runner.os == 'Linux' run: sudo python3 -I -B tests/helper_install_test.py --fixture "$PWD/zig-out/bin/dragontool-pki-fixture" --agent "$PWD/zig-out/bin/dragontool-agent" diff --git a/tests/integration/doers-validation.md b/tests/integration/doers-validation.md index 35adab1..6447b4c 100644 --- a/tests/integration/doers-validation.md +++ b/tests/integration/doers-validation.md @@ -99,8 +99,38 @@ docker run --rm --network none --read-only --tmpfs /tmp:rw,exec,size=1g \ python3 -I -B /work/tests/integration/doers_runtime.py /fixture ``` -All seven process gates above passed. All binaries and temporary localhost -credentials were supplied read-only; writable state was disposable `/tmp`. +All seven process gates above passed. The same seven gates also passed on +Ubuntu 26.04 after changing the fixture to establish an inactive alert baseline +and observe pending before its other signal queries. This avoids missing the +pending window; the two-minute hold and production intervals are unchanged. +All binaries and temporary localhost credentials were supplied read-only; +writable state was disposable `/tmp`. + +The extra outage fixture uses `--outage-only` (baseline signals, then outage) or +`--outage` (full alert cycle, then outage). It pauses only its local Caddy process, +attempts continuous ordinary info-log input, and checks bounded data files plus +sustained source backpressure. It reads the gauge value before an optional +Prometheus timestamp. The configured maximum includes segment/acknowledgement +headroom and is not assumed to equal usable queue capacity. Resumption must show +source progress, a declining backlog and actual queued logs in VictoriaLogs; +there is no fixed drain-throughput requirement. + +The corrected outage gate passed on Ubuntu 26.04 arm64 with this command: + +```sh +docker run --rm --network none --read-only --tmpfs /tmp:rw,exec,size=2g \ + -v "$PWD:/work:ro" -v /tmp/dragontools-caddy-pipeline:/fixture:ro \ + -w /tmp dragontools-pki-test:ubuntu26.04 \ + python3 -I -B /work/tests/integration/doers_runtime.py /fixture --outage-only +``` + +Observed backlog: **131,325,192 bytes**; Vector data files: **132,455,328 bytes**. +Input stopped advancing under backpressure, then resumed with a declining queue +and queued logs arriving after Caddy resumed, without an agent restart. Earlier +outage attempts failed because the fixture parsed the optional metric timestamp +as the gauge, then assumed all configured buffer bytes were usable. Those +attempts are not counted as successful outage validation. The final Linux CI +gate runs the full alert cycle and corrected outage check together (`--outage`). ## Deployment gate still required diff --git a/tests/integration/vector_outage.py b/tests/integration/vector_outage.py index e311d39..c8badfb 100644 --- a/tests/integration/vector_outage.py +++ b/tests/integration/vector_outage.py @@ -17,7 +17,9 @@ def exercise(vector, caddy, data_dir, request, wait, logs): def buffer_size(): text = request(8686, 'GET', '/metrics').decode() rows = [line for line in text.splitlines() if line.startswith('vector_buffer_size_bytes{') and 'component_id="logs"' in line] - return max((float(line.rsplit(' ', 1)[1]) for line in rows), default=0) + # Prometheus exposition may include a trailing millisecond timestamp. + # The value is the first token after the label block, never the last. + return max((float(line.split('}', 1)[1].split()[0]) for line in rows), default=0) # Keep each line bounded and ordinary info-level; it is synthetic fixture # traffic, never an application error or installer-generated event. event = dict(_SYSTEMD_UNIT='doers.service', PRIORITY='6', message=json.dumps(dict( @@ -27,7 +29,7 @@ def buffer_size(): os.set_blocking(descriptor, False) os.kill(caddy.pid, signal.SIGSTOP) offset = sent = 0 - full_since = None + last_progress = time.monotonic() sizes = [] deadline = time.monotonic() + 120 try: @@ -39,21 +41,25 @@ def buffer_size(): count = os.write(descriptor, payload[offset:]) sent += count offset = (offset + count) % len(payload) + if count: + last_progress = time.monotonic() except BlockingIOError: pass now = time.monotonic() if not sizes or now - sizes[-1][0] >= 1: size = buffer_size() + assert 0 <= size <= LIMIT + 65536, 'Unexpected buffer gauge value' disk = sum(p.stat().st_size for p in Path(data_dir).rglob('*') if p.is_file()) # Two configured disk buffers plus bounded ledger/segment slack. assert disk < 2 * LIMIT + 16 * 1024**2, 'Vector disk exceeded configured budget' sizes.append((now, size, disk, sent)) - if size >= LIMIT - 1024 * 1024 and full_since is None: - full_since = now - if full_since is not None and now - full_since >= 5: + # Disk segments awaiting acknowledgement can stop writes below + # max_size. Prove a substantial queued backlog and sustained + # blocked input, without treating max_size as usable capacity. + if size >= LIMIT // 4 and now - last_progress >= 10: # Source progress must stall while the full queue blocks. assert sizes[-1][3] == sizes[-3][3], 'Full buffer did not backpressure stdin' - print('PASS: Vector outage buffer saturated at ' + str(int(size)) + + print('PASS: Vector outage backlog ' + str(int(size)) + ' bytes; data files ' + str(disk) + ' bytes; source backpressure observed.', flush=True) break if now >= deadline: From 9ddd8dd86bb6d79a7133b98b0d459511f9f23042 Mon Sep 17 00:00:00 2001 From: Vasyl Osypchuk Date: Sat, 19 Sep 2026 08:08:28 +0300 Subject: [PATCH 5/7] Account for evaluator delay and alignment in recovery fixture --- tests/integration/doers-validation.md | 9 +++++++++ tests/integration/doers_runtime.py | 8 ++++++-- 2 files changed, 15 insertions(+), 2 deletions(-) diff --git a/tests/integration/doers-validation.md b/tests/integration/doers-validation.md index 6447b4c..27faf35 100644 --- a/tests/integration/doers-validation.md +++ b/tests/integration/doers-validation.md @@ -106,6 +106,15 @@ pending window; the two-minute hold and production intervals are unchanged. All binaries and temporary localhost credentials were supplied read-only; writable state was disposable `/tmp`. +An Actions run then exposed an undersized recovery deadline in the fixture. +The pinned evaluator applies a 30-second query delay, aligns query time to the +30-second group interval, then evaluates on its next tick. That can require up +to 90 seconds after a recovery sample. The fixture now polls for 105 seconds +(90 plus 15 seconds of runner margin), and 135 seconds for pending because a +stopped target must also wait for its next scrape. No production timeout, rule, +hold duration or evaluation setting changed. See the pinned +[vmalert scheduling implementation](https://github.com/VictoriaMetrics/VictoriaMetrics/blob/v1.152.0/app/vmalert/rule/group.go). + The extra outage fixture uses `--outage-only` (baseline signals, then outage) or `--outage` (full alert cycle, then outage). It pauses only its local Caddy process, attempts continuous ordinary info-log input, and checks bounded data files plus diff --git a/tests/integration/doers_runtime.py b/tests/integration/doers_runtime.py index c1c62b8..5d81437 100644 --- a/tests/integration/doers_runtime.py +++ b/tests/integration/doers_runtime.py @@ -241,7 +241,11 @@ def alert_state(): app.shutdown(); app.server_close(); servers.remove(app) # Observe pending first, before separate signal queries. Otherwise # their polling can miss part of the hold on a loaded CI runner. - pending = wait(lambda: (r if (r := alert_state())['state'] == 'pending' else None), 'pending alert', 90) + # vmalert's default 30s evaluation delay, 30s query-time alignment + # and next 30s evaluation tick can add up to 90s after a sample. + # Stopping the target also waits for the next 30s scrape. Keep all + # production timings and poll within those bounds plus 15s margin. + pending = wait(lambda: (r if (r := alert_state())['state'] == 'pending' else None), 'pending alert', 135) assert pending['duration'] == 120 and pending['health'] == 'ok' wait(lambda: list(n['stored_states'](expected, True).values()) == ['unhealthy'], 'probe_success=0') wait(lambda: query('up' + selector + ' == 0'), 'vmagent failed scrape') @@ -253,7 +257,7 @@ def alert_state(): assert time.time() - active >= 120 app = application() wait(lambda: list(n['stored_states'](expected, True).values()) == ['healthy'], 'probe recovery') - wait(lambda: alert_state()['state'] == 'inactive', 'alert resolution', 75) + wait(lambda: alert_state()['state'] == 'inactive', 'alert resolution', 105) wait(lambda: query('up' + selector + ' == 1') and signals.check('app', registration, vm_since), 'metrics recovery') print('PASS: ServiceProbeFailed fired after the real 2m hold, then resolved after target recovery.', flush=True) From d8393b75c2424c2d56e8c16c8b3ae15c97e290e3 Mon Sep 17 00:00:00 2001 From: Vasyl Osypchuk Date: Sat, 19 Sep 2026 09:57:44 +0300 Subject: [PATCH 6/7] Fix station config defaults and native ingress bootstrap --- .github/workflows/ci.yml | 3 + AGENTS.md | 5 + CHANGELOG.md | 15 +++ README.md | 98 ++++++++++++++---- architecture.md | 9 +- design.md | 36 +++++-- examples/monitoring.toml | 2 +- examples/station.toml | 2 +- src/agent/diagnostics.zig | 26 ++++- src/agent/files.zig | 57 ++++++++++- src/agent/protocol.zig | 1 + src/agent/runtime.zig | 10 +- src/agent/station.zig | 66 +++++++++--- src/agent_main.zig | 2 + src/agent_tests.zig | 81 ++++++++++++++- src/cli/help.zig | 9 +- src/cli/parse.zig | 77 +++++++++++++- src/cli/spec.zig | 5 +- src/config/monitoring.zig | 62 ++++++++++-- src/main.zig | 6 +- src/monitoring/agents/tests.zig | 37 +++++++ src/monitoring/install.zig | 26 ++++- src/monitoring/plan.zig | 10 +- src/monitoring/readiness.zig | 11 ++ tests/cli_smoke.py | 92 ++++++++++++----- tests/station_bootstrap_test.py | 174 ++++++++++++++++++++++++++++++++ 26 files changed, 823 insertions(+), 99 deletions(-) create mode 100644 tests/station_bootstrap_test.py diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index e0468a5..8d8ec5f 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -37,6 +37,9 @@ jobs: - name: Linux helper filesystem lifecycle (temporary paths only) if: runner.os == 'Linux' run: sudo python3 -I -B tests/helper_install_test.py --fixture "$PWD/zig-out/bin/dragontool-pki-fixture" --agent "$PWD/zig-out/bin/dragontool-agent" + - name: Native station bootstrap (isolated filesystem, exact production ownership) + if: runner.os == 'Linux' + run: sudo python3 -I -B tests/station_bootstrap_test.py --agent "$PWD/zig-out/bin/dragontool-agent" cross-build: runs-on: ubuntu-24.04 strategy: diff --git a/AGENTS.md b/AGENTS.md index 7172a5c..34f182b 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -7,6 +7,11 @@ configuration and login shell. `wizard` and `completion` are local UX entry poin No generic resource DSL, shell hooks or provider framework. Read README.md, architecture.md and design.md before changing workflow behavior. +Station install/verify/status/notify-test default only to CWD `./station.toml`; +explicit `--config` replaces it and CLI fields override it. `[station].hostname` +is canonical; conflicting v1 `[ingress].hostname` aliases fail. Application +commands never load station config. Plans resolve no secrets. + The primary application workflow is `monitoring apply` with strict version-1 `./monitoring.toml`, or one explicit `--config`. Keep application configuration separate from central station configuration and secrets. Application/environment diff --git a/CHANGELOG.md b/CHANGELOG.md index 4637866..e885dfb 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,21 @@ ## 0.1.0-dev — unreleased +- Default station install/verify/status/notify-test to CWD `./station.toml`, with + explicit config replacement and CLI precedence. Add canonical `[station].hostname`, + retain the compatible v1 `[ingress].hostname` alias and reject conflicts. Plans + show the SSH connection and TLS hostname/9443/9444 without resolving secrets; + application commands retain their separate `./monitoring.toml` default. +- Fix Linux native bootstrap's free-lock failure: Zig 0.16 `O_PATH` descriptors + cannot be flocked. Use a readable directory descriptor with unchanged exclusive, + nonblocking semantics; report contention as OperationBusy/96. Add granular safe + filesystem/CA/server checks and preserve redaction and CA maintenance semantics. +- Converge absent/empty/interrupted station PKI state, recover proven unpublished + candidates, reuse published CA/server identity and restrict registry repair to + the known 0700-to-0750 generation. Add portable lifecycle/lock tests and a real + helper chroot regression with exact root/dt-ingest production ownership and no-op + rerun. No application registration, ambient OpenSSL or architecture changes. + - Accept fresh vmagent failed-scrape telemetry when the application is down; require fresh non-scrape payload on successful targets and reject stale/missing data. Report pipeline readiness independently of application availability. diff --git a/README.md b/README.md index 43ce0a1..756ed61 100644 --- a/README.md +++ b/README.md @@ -6,7 +6,8 @@ Install the monitoring station once, then keep each application's monitoring contract in its own repository: ```bash -dragontool monitoring install --config station.toml +cd monitoring-infra # Directory containing station.toml +dragontool monitoring install cd my-application dragontool monitoring apply --plan @@ -15,6 +16,7 @@ dragontool monitoring app-verify dragontool monitoring apply # Deliberate unchanged rerun ``` +Station `install`, `verify`, `status` and `notify-test` read `./station.toml`. Application commands read only `./monitoring.toml` by default, or one explicit `--config PATH`. They configure host metrics, selected journal logs, optional application metrics, station HTTP probes and application alerts. The strict @@ -129,14 +131,18 @@ while anonymous access, auth proxy and signup stay disabled. ## Monitoring configuration and Grafana credentials -`--config` reads one explicit, small TOML file for monitoring `install`, `verify`, -`status`, and `notify-test`. There is no implicit file discovery. The version-1 +`monitoring install`, `verify`, `status` and `notify-test` load **`./station.toml`** +when present; install `--plan` uses it too. `--config PATH` selects one different +file instead of the default. Only the current working directory is used: no parent, +home, XDG or `/etc` search. Application commands use only `./monitoring.toml`. The version-1 schema accepts an OpenSSH alias, an ingress TLS DNS hostname, Grafana/Telegram secret references and bounded named HTTP/HTTPS probes; component tuning, literal passwords, unknown keys and duplicate keys are rejected. -The checked-in [examples/monitoring.toml](examples/monitoring.toml) contains these -**example** references. Replace the alias and vault/item/field paths with your own: +Save central station intent as `station.toml`, separately from application +repositories. [examples/station.toml](examples/station.toml) is a minimal template. +The following hostname, SSH alias and optional secret references are examples; +replace them with your own: ```toml version = 1 @@ -144,12 +150,16 @@ version = 1 [connection] ssh_host = "monitoring" -[ingress] -hostname = "monitoring.example.com" +[station] +hostname = "monitoring.baptizeddragon.com" [grafana] username = { op = "op://BaptizedDragon/Grafana/username" } password = { op = "op://BaptizedDragon/Grafana/password" } + +[telegram] +bot_token = { op = "op://BaptizedDragon/DragonTools/alarms-telegram-bot-token" } +chat_id = { op = "op://BaptizedDragon/DragonTools/alarms-telegram-chat-id" } ``` A reference contains no resolved secret and is safe to keep in configuration or @@ -164,17 +174,44 @@ its CLI, or its session credentials. op signin zig build -Doptimize=ReleaseSafe -./zig-out/bin/dragontool monitoring install --config examples/monitoring.toml --plan -./zig-out/bin/dragontool monitoring install --config examples/monitoring.toml -./zig-out/bin/dragontool monitoring verify --config examples/monitoring.toml -./zig-out/bin/dragontool monitoring status --config examples/monitoring.toml +./zig-out/bin/dragontool monitoring install --plan +./zig-out/bin/dragontool monitoring install +./zig-out/bin/dragontool monitoring verify +./zig-out/bin/dragontool monitoring status # Desired credentials already work: no password reset or service restart. -./zig-out/bin/dragontool monitoring install --config examples/monitoring.toml +./zig-out/bin/dragontool monitoring install +``` + +Precedence is explicit CLI flags, then the selected file, then CLI defaults. An +explicit `--config other.toml` ignores `./station.toml` completely. A missing +default file still permits CLI-only usage; without a connection, the error points +to `./station.toml`, `--config` and CLI options. Present malformed files fail +without exposing their contents. + +`connection.ssh_host` is an administrative OpenSSH alias. `station.hostname` is +the independent DNS/mTLS identity, overridden by `--ingress-hostname`. It must be +a DNS name without a scheme, port or path. Version 1 continues accepting deprecated +`[ingress].hostname`; specifying both forms with different values fails. Plan shows +the connection, hostname and metrics/logs ports without resolving any secrets. + +For example, from a directory without `station.toml`, CLI-only operation is: + +```bash +dragontool monitoring install --ssh-host monitoring \ + --ingress-hostname monitoring.baptizeddragon.com ``` -Explicit CLI values override the corresponding configuration values. The direct -CLI equivalent is: +`station.toml` describes the station; `monitoring.toml` describes one application: + +```bash +cd monitoring-infra +dragontool monitoring install +cd ../doers +dragontool monitoring apply +``` + +The Grafana CLI equivalent is: ```bash ./zig-out/bin/dragontool monitoring install \ @@ -569,14 +606,14 @@ Run with the same enrolled SSH alias throughout: ```bash zig build -Doptimize=ReleaseSafe -./zig-out/bin/dragontool monitoring install --config monitoring.toml --plan -./zig-out/bin/dragontool monitoring install --config monitoring.toml -./zig-out/bin/dragontool monitoring verify --config monitoring.toml -./zig-out/bin/dragontool monitoring status --config monitoring.toml +./zig-out/bin/dragontool monitoring install --config station.toml --plan +./zig-out/bin/dragontool monitoring install --config station.toml +./zig-out/bin/dragontool monitoring verify --config station.toml +./zig-out/bin/dragontool monitoring status --config station.toml # Deliberate unchanged rerun: -./zig-out/bin/dragontool monitoring install --config monitoring.toml +./zig-out/bin/dragontool monitoring install --config station.toml # Explicitly sends a test alert through Alertmanager to its configured receiver: -./zig-out/bin/dragontool monitoring notify-test --config monitoring.toml +./zig-out/bin/dragontool monitoring notify-test --config station.toml ``` Only GET with expected HTTP 2xx is supported. Probe names are unique, at most 63 @@ -1004,7 +1041,7 @@ for local test evidence and remaining deployment checks. `monitoring install` owns all ten station services, including Caddy and private registry authorization, plus the native helper and station PKI. A fresh station -requires `--ingress-hostname DNS` or `[ingress].hostname` in the central station +requires `--ingress-hostname DNS` or `[station].hostname` in the central station config. The CLI flag overrides the file. Without either, an existing exactly managed server bundle supplies its saved identity; a fresh station fails with `ingress_hostname_required`. The SSH alias never supplies the TLS hostname. @@ -1225,6 +1262,25 @@ CA state. Correct the reported cause and rerun station install for base PKI/ingr or the same application config for client enrollment; empty managed bootstrap parents remain safe to reuse. +Station install accepts an absent ingestion tree, the empty root-owned +`pki/clients/registry` skeleton and interrupted unpublished bundles. It validates +native Mbed TLS CA/server candidates before atomic publication and fsync, reuses +an already published valid CA, and never rotates an invalid existing CA silently. +Only recognized private staging with known names, marker and metadata is removed; +unrecognized files and symlinks are preserved and refused. The known +root:dt-ingest registry mode `0700` migrates to `0750`; other ownership/type or +unknown mode conflicts fail. No app directory or registration is required. + +Fixed checks now distinguish `ingestion_root_invalid`, `pki_directory_invalid`, +`clients_directory_invalid`, `registry_directory_invalid`, `state_directory_invalid`, +`unexpected_managed_file`, `unexpected_symlink`, `ca_bundle_invalid` and +`server_identity_invalid`. Missing CA in an uninitialized tree is the internal +`ca_missing_bootstrap_allowed` state, not corruption. A held operation lock reports +`operation_busy` (native exit 96); it never reports a filesystem contradiction. +The Linux lock regression used Zig 0.16's `O_PATH` directory descriptor, which +`flock` rejects even without contention. Using a readable directory handle preserves +the same exclusive, nonblocking advisory lock and read-only verification behavior. + This is a private ingestion channel, so DragonTools intentionally uses its own CA rather than Let's Encrypt. The station certificate includes its configured DNS/IP SAN. Vector checks both certificate and hostname, and vmagent retains diff --git a/architecture.md b/architecture.md index fcb9565..321f76a 100644 --- a/architecture.md +++ b/architecture.md @@ -14,7 +14,12 @@ provider abstraction, arbitrary shell hooks, or general plugin framework. Application repositories use strict `monitoring.toml` v1 and `monitoring apply`. The controller reads only the current directory's file or an explicit `--config`. `app-verify` and `app-status` are read-only application commands; station -`install/verify/status` retain their separate central configuration and secrets. +`install/verify/status/notify-test` load optional `./station.toml`, separately from +application configuration and secrets. Explicit `--config` replaces the default. +Neither workflow searches parents, home, XDG or `/etc`; CLI-only station operation +remains valid when the default is absent. `[station].hostname` supplies TLS identity +independent of the SSH alias; the deprecated v1 `[ingress].hostname` alias cannot +contradict it. Plans show the resolved connection/hostname and never resolve secrets. `cli/parse.zig` merges explicit monitoring configuration and validates all supplied inputs before SSH. CLI values override file values; the small version-1 TOML @@ -541,7 +546,7 @@ readiness budget, not fixed sleeps. Missing signals fail installation. Station `monitoring install` provisions the ingestion/Caddy accounts, native helper, CA/server bundle, Caddy binary/config/unit and private authorization helper. Explicit -`[ingress].hostname` or `--ingress-hostname` supplies the TLS DNS identity on first +`[station].hostname` or `--ingress-hostname` supplies the TLS DNS identity on first install; later runs may reuse the managed server endpoint. No SSH alias inference or application registration is involved. Station verify checks PKI and transport, including mandatory client authentication with an empty registry; it never enrolls diff --git a/design.md b/design.md index aab0fff..4bbadfd 100644 --- a/design.md +++ b/design.md @@ -577,11 +577,16 @@ unchecked; plans and status never resolve secrets or execute these API requests. ## Monitoring configuration and Grafana credentials -Station `--config PATH` is explicit and supported for monitoring install, verify -and status. That schema has no conventional-path search, includes or -interpolation; application config uses the separate contract above. +Station install, verify, status and notify-test load the optional `./station.toml` +from CWD, including install plans. Explicit `--config PATH` replaces the default; +CLI fields override that selected file, then existing CLI defaults apply. Only +absence is optional; present invalid/unreadable config fails safely. No parent, +home, XDG or `/etc` search, includes or interpolation is performed. Application +commands retain the independent `./monitoring.toml` contract above. `[station].hostname` +is canonical; the v1 `[ingress].hostname` alias remains accepted, and contradictory +values fail before CLI merging or SSH. `config/monitoring.zig` accepts a bounded 64 KiB version-1 TOML subset: -`version = 1`, `[connection].ssh_host`, `[ingress].hostname`, and `[grafana]` username/password inline +`version = 1`, `[connection].ssh_host`, `[station].hostname`, and `[grafana]` username/password inline `{ op = "op://vault/item/field" }` references. Single-line basic/literal strings, basic escapes and comments are supported. Unknown or duplicate keys/tables, literal secret values, array/dotted/nested/multiline forms and unsupported sources @@ -914,7 +919,7 @@ The helper never terminates TLS or reads server keys. Local roots remain trusted Station install owns accounts, the native helper, CA/server PKI, Caddy and private Unix-socket authorization. A fresh station requires explicit `--ingress-hostname` -or `[ingress].hostname`; an omitted value reuses only an exactly managed server +or `[station].hostname`; an omitted value reuses only an exactly managed server bundle's saved endpoint. CLI overrides the config; neither infers a TLS name from SSH. Native `station-ensure` accepts only the endpoint, no app registration. `station-verify` is read-only; the enrollment `ensure` action now only checks an @@ -984,18 +989,18 @@ replaces keys, suppresses a failure or relies on platform-specific output text. New CA material is fully validated in memory before root-private staging and atomic rename; validation failure removes staging and leaves no published CA directory. Parent directories may remain. New directories explicitly receive their requested mode -after creation under the helper's private umask. Apply reconciles the fixed +after creation under the helper's private umask. Station install reconciles the fixed ingestion registry directory to root:dt-ingest 0750, including older mode-0700 directories on initialized stations. Directory handles opened without following -symlinks anchor the repair under the owned ingestion tree. Only mode is repaired +symlinks anchor the repair under the owned ingestion tree. Only the known 0700 mode is repaired after root ownership and the dt-ingest group are proven; unsafe paths, other file -types and incompatible ownership fail with `ingestion / registry_permissions`. +types and incompatible ownership fail with `Ingress authorization / registry_directory_invalid` (legacy helpers retain `registry_permissions`). Contents and credentials are preserved; no service restart is requested for a mode-only repair, and correct permissions are a no-op. Read-only verification reports drift without repairing it. Empty pki/clients/registry parents remain a valid bootstrap starting point. An existing CA is never replaced on validation failure; missing CA material on an initialized station still requires -explicit maintenance. Rerunning apply needs no manual cleanup of empty parents. +explicit maintenance. Rerunning station install needs no manual cleanup of empty parents. The bounded native stdin decoder treats Zig 0.16 `error.EndOfStream` as normal request completion. It still rejects empty, malformed, oversized and unexpected @@ -1004,7 +1009,18 @@ station enrollment failing before CA generation when the controller closes stdin `ingestion.zig` invokes the installed native helper using a fixed command and bounded public JSON on stdin. SSH transports only public inspection, CSR and -certificate payloads. The root operation lock serializes local generations. A CSR is at most 8192 bytes and must be strict PEM +certificate payloads. The root operation lock serializes local generations. It opens the existing +`/etc/dragontools` directory with `iterate=true`: Zig 0.16 otherwise uses Linux +`O_PATH`, which cannot be flocked (EBADF). Contention maps to `OperationBusy`/96; +other lock failures are distinct from filesystem invariants. The lock remains +exclusive/nonblocking and creates no lock file. Bootstrap explicitly validates +each managed parent with separate safe stages, and records missing CA as an +allowed bootstrap condition only before any server/client/registry state exists. +Only root-owned unpublished `.bundle-<32 lowercase hex>` and `.credential-...` +candidates with known metadata/contents are discarded; active CA/server bundles +are validated and reused. Unknown files, links or malformed active bundles fail +closed. Recovery validates every candidate entry before unlinking any; marker +and file mode transitions from interrupted writes are recognized. A CSR is at most 8192 bytes and must be strict PEM PKCS#10. The station checks its signature, P-256 public-key validity, exact CN and sole host URI SAN with a narrow DER profile. Other requested extensions (including CA/serverAuth), extra names and attributes fail before signing. Issuance never diff --git a/examples/monitoring.toml b/examples/monitoring.toml index 2953326..7a5e7dd 100644 --- a/examples/monitoring.toml +++ b/examples/monitoring.toml @@ -6,7 +6,7 @@ version = 1 ssh_host = "monitoring" # TLS identity; independent of the SSH alias. Required for a fresh station. -[ingress] +[station] hostname = "monitoring.example.com" [grafana] diff --git a/examples/station.toml b/examples/station.toml index f93d6bd..b3787b8 100644 --- a/examples/station.toml +++ b/examples/station.toml @@ -6,7 +6,7 @@ version = 1 ssh_host = "replace-me-monitoring" # TLS identity; independent of the SSH alias. Required for a fresh station. -[ingress] +[station] hostname = "monitoring.example.com" # [grafana] diff --git a/src/agent/diagnostics.zig b/src/agent/diagnostics.zig index b08de5b..6853265 100644 --- a/src/agent/diagnostics.zig +++ b/src/agent/diagnostics.zig @@ -32,9 +32,16 @@ pub fn enrollmentStage(action: []const u8) ?EnrollmentStage { pub const Stage = enum { request, native_initialization, + operation_lock, enrollment, station_registration_prepare, managed_directories, + ingestion_root, + pki_directory, + clients_directory, + registry_directory, + state_directory, + ca_missing_bootstrap_allowed, registry_prepare, ca_state, ca_key_generation, @@ -55,6 +62,10 @@ pub const AgentError = enum { CertificateValidationFailed, FilesystemStateRefused, InvalidManagedState, + UnexpectedManagedFile, + UnexpectedSymlink, + OperationBusy, + OperationLockFailed, AgentInternalError, CaMaintenanceRequired, ClientIdentityInconsistent, @@ -68,6 +79,7 @@ pub const AgentError = enum { }; pub const Detail = enum { agent_internal_error, + operation_busy, ca_maintenance, client_identity_inconsistent, registry_permissions, @@ -90,6 +102,7 @@ pub fn detail(code: u8) ?Detail { 93 => .server_tls_invalid, 94 => .client_certificate_rejected, 95 => .ingestion_rejected, + 96 => .operation_busy, else => null, }; } @@ -103,6 +116,10 @@ pub const Diagnostic = struct { }; pub fn failure(stage: Stage, err: anyerror) Diagnostic { const reason: AgentError = switch (err) { + error.OperationBusy => .OperationBusy, + error.OperationLockFailed => .OperationLockFailed, + error.UnexpectedManagedFile => .UnexpectedManagedFile, + error.UnexpectedManagedSymlink, error.SymLinkLoop => .UnexpectedSymlink, error.CaMaintenanceRequired => .CaMaintenanceRequired, error.ClientIdentityInconsistent => .ClientIdentityInconsistent, error.RegistryPermissions => .RegistryPermissions, @@ -117,7 +134,14 @@ pub fn failure(stage: Stage, err: anyerror) Diagnostic { .ca_certificate_generation, .server_certificate_generation => .CertificateGenerationFailed, .ca_certificate_validation, .server_certificate_validation => .CertificateValidationFailed, .managed_directories, .registry_prepare, .ca_publication, .server_publication => .FilesystemStateRefused, - .ca_state, .server_state, .station_registration_prepare => .InvalidManagedState, + .ca_state, + .server_state, + .station_registration_prepare, + .ingestion_root, + .pki_directory, + .clients_directory, + .state_directory, + => .InvalidManagedState, else => if (err == error.CredentialStateRefused) .InvalidManagedState else .AgentInternalError, }, }; diff --git a/src/agent/files.zig b/src/agent/files.zig index 2f6b207..7564f12 100644 --- a/src/agent/files.zig +++ b/src/agent/files.zig @@ -69,6 +69,7 @@ pub const Store = struct { errdefer current.close(self.io); var parts = std.mem.splitScalar(u8, pathname[1..], '/'); while (parts.next()) |part| { + if ((try current.statFile(self.io, part, .{ .follow_symlinks = false })).kind == .sym_link) return error.UnexpectedManagedSymlink; const next = try current.openDir(self.io, part, .{ .iterate = true, .follow_symlinks = false }); current.close(self.io); current = next; @@ -114,6 +115,9 @@ pub const Store = struct { defer dir.close(self.io); const name = std.fs.path.basename(pathname); var changed = false; + if (dir.statFile(self.io, name, .{ .follow_symlinks = false })) |st| { + if (st.kind == .sym_link) return error.UnexpectedManagedSymlink; + } else |err| if (err != error.FileNotFound) return err; const child = dir.openDir(self.io, name, .{ .iterate = true, .follow_symlinks = false }) catch |err| blk: { if (err != error.FileNotFound or !create) return err; try dir.createDir(self.io, name, .fromMode(0o700)); @@ -150,7 +154,8 @@ pub const Store = struct { try metadata(child.handle, .{ .uid = self.root_owner.uid, .gid = group }, null, std.posix.S.IFDIR); const st = try child.stat(self.io); if (st.permissions.toMode() & 0o7777 != 0o750) { - try j.require(reconcile); + // Only the known previous 0700 generation may be migrated. + try j.require(reconcile and st.permissions.toMode() & 0o7777 == 0o700); try child.setPermissions(self.io, .fromMode(0o750)); try self.syncDir(child); changed = true; @@ -176,6 +181,7 @@ pub const Store = struct { // Refuse FIFOs/devices before opening; opening a FIFO could otherwise // block forever. Ancestors are root-owned and not publicly writable. const before = try dir.statFile(self.io, std.fs.path.basename(pathname), .{ .follow_symlinks = false }); + if (before.kind == .sym_link) return error.UnexpectedManagedSymlink; try j.require(before.kind == .file); const file = try dir.openFile(self.io, std.fs.path.basename(pathname), .{ .follow_symlinks = false, .allow_directory = false }); defer file.close(self.io); @@ -260,8 +266,8 @@ pub const Store = struct { pub fn managed(self: Store, pathname: []const u8, owner: Owner, mode: u16, names_value: []const []const u8, content: []const u8, exact: bool) !Files { _ = try self.directory(pathname, .{ .uid = self.root_owner.uid, .gid = owner.gid }, mode, false); const entries = try self.names(pathname); + for (entries) |name| if (!std.mem.eql(u8, name, ".dragontools-managed") and !j.contains(names_value, name)) return error.UnexpectedManagedFile; if (exact) try j.require(entries.len == names_value.len + 1); - for (entries) |name| try j.require(std.mem.eql(u8, name, ".dragontools-managed") or j.contains(names_value, name)); try j.require(std.mem.eql(u8, try self.read(try self.path(pathname, ".dragontools-managed"), self.root_owner, 0o400, 128), content)); const result = try self.readFiles(pathname, names_value, owner); if (exact) try keys(result, names_value); @@ -287,4 +293,51 @@ pub const Store = struct { try self.rename(stage, pathname, false); published = true; } + pub fn stagedFile(self: Store, pathname: []const u8, owner: Owner) ![]const u8 { + const parent_dir = try self.parent(pathname); + defer parent_dir.close(self.io); + const st = try parent_dir.statFile(self.io, std.fs.path.basename(pathname), .{ .follow_symlinks = false }); + if (st.kind == .sym_link) return error.UnexpectedManagedSymlink; + const mode: u16 = @intCast(st.permissions.toMode() & 0o7777); + try j.require(mode == 0o400 or mode == 0o600); + if (mode == 0o600) { + // write() may stop before or after chown, before its final chmod. + return self.read(pathname, self.root_owner, mode, limit) catch |err| { + if (err != error.CredentialStateRefused) return err; + return self.read(pathname, owner, mode, limit); + }; + } + return self.read(pathname, owner, mode, limit); + } + /// Only station CA/server candidates use this recovery path. Validate every + /// entry before unlinking any: private generated name, exact marker/prefix, + /// no links, known files and metadata from interrupted createBundle writes. + pub fn discardBundle(self: Store, pathname: []const u8, owner: Owner, mode: u16, names_value: []const []const u8) !void { + const dir = try self.walk(pathname); + defer dir.close(self.io); + var st: Metadata = undefined; + try j.require(dragontools_file_metadata(dir.handle, &st) == 0); + try j.require(st.uid == self.root_owner.uid and + (st.gid == self.root_owner.gid and st.mode & 0o7777 == 0o700 or st.gid == owner.gid and (st.mode & 0o7777 == 0o700 or st.mode & 0o7777 == mode))); + const entries = try self.names(pathname); + if (entries.len != 0) { + const marker_path = try self.path(pathname, ".dragontools-managed"); + const marker_stat = dir.statFile(self.io, ".dragontools-managed", .{ .follow_symlinks = false }) catch return error.UnexpectedManagedFile; + if (marker_stat.kind == .sym_link) return error.UnexpectedManagedSymlink; + const marker_mode: u16 = @intCast(marker_stat.permissions.toMode() & 0o7777); + try j.require(marker_mode == 0o400 or marker_mode == 0o600); + const value = try self.read(marker_path, self.root_owner, marker_mode, marker.len); + if (marker_mode == 0o400) try j.require(std.mem.eql(u8, value, marker)) else try j.require(std.mem.startsWith(u8, marker, value)); + // Marker is completed before any bundle contents are created. + if (!std.mem.eql(u8, value, marker)) try j.require(entries.len == 1); + } + for (entries) |name| { + if (std.mem.eql(u8, name, ".dragontools-managed")) continue; + if (!j.contains(names_value, name)) return error.UnexpectedManagedFile; + _ = try self.stagedFile(try self.path(pathname, name), owner); + } + for (entries) |name| if (!std.mem.eql(u8, name, ".dragontools-managed")) try self.unlink(try self.path(pathname, name)); + if (entries.len != 0) try self.unlink(try self.path(pathname, ".dragontools-managed")); + try self.removeEmpty(pathname); + } }; diff --git a/src/agent/protocol.zig b/src/agent/protocol.zig index a8ba940..fe2e592 100644 --- a/src/agent/protocol.zig +++ b/src/agent/protocol.zig @@ -71,6 +71,7 @@ pub fn dispatch(ctx: s.Context, action: []const u8, args: []const []const u8) ![ } pub fn exitCode(err: anyerror) u8 { return switch (err) { + error.OperationBusy => 96, error.DnsUnresolved => 91, error.TcpUnreachable => 92, error.ServerTlsInvalid => 93, diff --git a/src/agent/runtime.zig b/src/agent/runtime.zig index fadf8c9..babe1d6 100644 --- a/src/agent/runtime.zig +++ b/src/agent/runtime.zig @@ -51,8 +51,14 @@ pub fn lock(io: std.Io, root: std.Io.Dir) !std.Io.Dir { // creates no lock file and alters no credential metadata. No unbounded wait. const etc = try root.openDir(io, "etc", .{ .follow_symlinks = false }); defer etc.close(io); - const dir = try etc.openDir(io, "dragontools", .{ .follow_symlinks = false }); + // Zig 0.16 uses O_PATH on Linux unless iterate is enabled. O_PATH handles + // cannot be flock'ed (EBADF), even when no other process holds the lock. + const dir = try etc.openDir(io, "dragontools", .{ .iterate = true, .follow_symlinks = false }); errdefer dir.close(io); - try s.j.require(c.flock(dir.handle, c.LOCK_EX | c.LOCK_NB) == 0); + switch (std.posix.errno(c.flock(dir.handle, c.LOCK_EX | c.LOCK_NB))) { + .SUCCESS => {}, + .AGAIN => return error.OperationBusy, + else => return error.OperationLockFailed, + } return dir; } diff --git a/src/agent/station.zig b/src/agent/station.zig index 47d712a..4c2d425 100644 --- a/src/agent/station.zig +++ b/src/agent/station.zig @@ -9,6 +9,47 @@ const ca_path = f.base ++ "/pki/ca"; const server_path = f.base ++ "/server"; const modern = [_][]const u8{ "certificate_pem", "certificate_identity" }; +fn directories(ctx: Context, create: bool) !bool { + const store = ctx.store; + ctx.track(.ingestion_root); + _ = try store.directory(f.etc, store.root_owner, 0o755, false); + var changed = try store.directory(f.base, store.root_owner, 0o755, create); + ctx.track(.pki_directory); + changed = try store.directory(f.base ++ "/pki", store.root_owner, 0o700, create) or changed; + ctx.track(.clients_directory); + changed = try store.directory(f.base ++ "/clients", store.root_owner, 0o700, create) or changed; + ctx.track(.registry_directory); + changed = try store.registry(ctx.ingestion.gid, create) or changed; + ctx.track(.state_directory); + changed = try store.directory(f.state, store.root_owner, 0o755, create) or changed; + changed = try store.directory(f.state ++ "/ingestion", ctx.ingestion, 0o750, create) or changed; + return changed; +} +fn temporaryName(name: []const u8, prefix: []const u8) bool { + if (name.len != prefix.len + 32 or !std.mem.startsWith(u8, name, prefix)) return false; + for (name[prefix.len..]) |byte| if (!std.ascii.isDigit(byte) and !(byte >= 'a' and byte <= 'f')) return false; + return true; +} +fn candidates(ctx: Context, parent: []const u8, active: []const []const u8, owner: f.Owner, mode: u16, files: []const []const u8) !bool { + var changed = false; + for (try ctx.store.names(parent)) |name| { + if (j.contains(active, name)) continue; + const path = try ctx.store.path(parent, name); + if (temporaryName(name, ".bundle-")) { + try ctx.store.discardBundle(path, owner, mode, files); + } else if (temporaryName(name, ".credential-")) { + // An interrupted single-file atomic publication is never active. + _ = try ctx.store.stagedFile(path, owner); + try ctx.store.unlink(path); + } else return error.UnexpectedManagedFile; + changed = true; + } + return changed; +} +fn cleanParent(ctx: Context, parent: []const u8, active: []const []const u8) !void { + for (try ctx.store.names(parent)) |name| if (!j.contains(active, name)) return error.UnexpectedManagedFile; +} + pub fn loadCa(ctx: Context) !f.Files { const caller_stage = if (ctx.diagnostic_stage) |current| current.* else .enrollment; ctx.track(.ca_state); @@ -28,12 +69,11 @@ pub fn verifyServer(ctx: Context, endpoint: []const u8, allow_renewal: bool) ![] ctx.track(.server_certificate_validation); try s.endpoint(endpoint); const store = ctx.store; - ctx.track(.managed_directories); - _ = try store.directory(f.base, store.root_owner, 0o755, false); - _ = try store.directory(f.base ++ "/pki", store.root_owner, 0o700, false); - _ = try store.directory(f.base ++ "/clients", store.root_owner, 0o700, false); - ctx.track(.registry_prepare); - _ = try store.registry(ctx.ingestion.gid, false); + _ = try directories(ctx, false); + ctx.track(.ingestion_root); + try cleanParent(ctx, f.base, &.{ "pki", "clients", "registry", "server" }); + ctx.track(.pki_directory); + try cleanParent(ctx, f.base ++ "/pki", &.{"ca"}); const root = try loadCa(ctx); ctx.track(.server_state); const values = try store.managed(server_path, ctx.ingestion, 0o750, &.{ "ca.crt", "server.crt", "server.key", "endpoint" }, f.marker, true); @@ -160,16 +200,16 @@ pub fn ensureStation(ctx: Context, hostname: ?[]const u8) !bool { const store = ctx.store; ctx.track(.server_state); const endpoint = try stationEndpoint(ctx, hostname); - ctx.track(.managed_directories); - _ = try store.directory(f.base, store.root_owner, 0o755, false); - ctx.track(.registry_prepare); - var changed = try store.registry(ctx.ingestion.gid, true); - ctx.track(.managed_directories); - changed = try store.directory(f.base ++ "/pki", store.root_owner, 0o700, true) or changed; - changed = try store.directory(f.base ++ "/clients", store.root_owner, 0o700, true) or changed; + var changed = try directories(ctx, true); + ctx.track(.ingestion_root); + changed = try candidates(ctx, f.base, &.{ "pki", "clients", "registry", "server" }, ctx.ingestion, 0o750, &.{ "ca.crt", "server.crt", "server.key", "endpoint" }) or changed; + ctx.track(.pki_directory); + changed = try candidates(ctx, f.base ++ "/pki", &.{"ca"}, store.root_owner, 0o700, &.{ "ca.crt", "ca.key" }) or changed; ctx.track(.ca_state); if (!try store.exists(ca_path)) { if (try store.exists(server_path) or (try store.names(f.base ++ "/clients")).len != 0 or (try store.names(f.base ++ "/registry")).len != 0) return error.CaMaintenanceRequired; + ctx.track(.ca_missing_bootstrap_allowed); + try store.checkpoint("ca_missing_bootstrap_allowed", ca_path); ctx.track(.ca_key_generation); try store.checkpoint("generate_ca_key", ca_path); var key = try pki.Key.generate(); diff --git a/src/agent_main.zig b/src/agent_main.zig index ca405e6..8341e55 100644 --- a/src/agent_main.zig +++ b/src/agent_main.zig @@ -49,8 +49,10 @@ fn entry(init: std.process.Init, enabled: *bool, stage: *diagnostics.Stage) !u8 stage.* = .native_initialization; const root = try std.Io.Dir.openDirAbsolute(init.io, "/", .{}); defer root.close(init.io); + stage.* = .operation_lock; const operation_lock = try runtime.lock(init.io, root); defer operation_lock.close(init.io); + stage.* = .native_initialization; var system: runtime.Runtime = .{ .a = a, .io = init.io }; var ctx = try system.context(root, protocol.stationAction(action)); ctx.diagnostic_stage = stage; diff --git a/src/agent_tests.zig b/src/agent_tests.zig index b76560e..d3047a2 100644 --- a/src/agent_tests.zig +++ b/src/agent_tests.zig @@ -6,6 +6,7 @@ const client_store = @import("agent/client_store.zig"); const j = state.j; const f = state.f; const pki = state.pki; +const runtime = @import("agent/runtime.zig"); const host = "dt-0123456789abcdef0123456789abcdef"; const Services = struct { active: [2]bool = .{ false, false }, @@ -46,6 +47,18 @@ fn registration(a: std.mem.Allocator) !j.Value { fn bootstrap(ctx: state.Context, value: j.Value) !bool { return station.ensureStation(ctx, try j.field(value, "station")); } +test "native operation lock accepts a free directory and serializes independent opens" { + var fixture = try Fixture.init(); + defer fixture.deinit(); + const io = std.testing.io; + const locked = try runtime.lock(io, fixture.temp.dir); + locked.close(io); + const repeated = try runtime.lock(io, fixture.temp.dir); + defer repeated.close(io); + // Separate open descriptions must contend even inside the same process. + try std.testing.expectError(error.OperationBusy, runtime.lock(io, fixture.temp.dir)); + try std.testing.expectEqual(@as(u8, 96), @import("agent/protocol.zig").exitCode(error.OperationBusy)); +} test "native station bootstraps empty parents migrates registry and preserves exact valid PKI" { var fixture = try Fixture.init(); defer fixture.deinit(); @@ -252,7 +265,7 @@ test "native CA corruption missing roots and near-expiry require repair without if (bootstrap(ctx, value)) |_| return error.ExpectedFailure else |_| {} try std.testing.expectEqualStrings("broken certificate", try bytes(ctx, f.base ++ "/pki/ca/ca.crt")); try ctx.store.rename(f.base ++ "/pki/ca", f.base ++ "/pki/saved-ca", false); - try std.testing.expectError(error.CaMaintenanceRequired, bootstrap(ctx, value)); + try std.testing.expectError(error.UnexpectedManagedFile, bootstrap(ctx, value)); try std.testing.expect(!try ctx.store.exists(f.base ++ "/pki/ca")); } test "native registry read-only metadata rejection and owned directory migration" { @@ -622,8 +635,8 @@ test "native managed directory refusal and registry permissions keep distinct di const err = (try ensureFailure(ctx, try registration(ctx.store.a), &stage)).err; try std.testing.expectEqual(@as(u8, if (registry) 89 else 86), @import("agent/protocol.zig").exitCode(err)); const diagnostic = diagnostics.failure(stage, err); - try std.testing.expectEqual(if (registry) diagnostics.Stage.registry_prepare else .managed_directories, diagnostic.stage); - try std.testing.expectEqual(if (registry) diagnostics.AgentError.RegistryPermissions else .FilesystemStateRefused, diagnostic.reason); + try std.testing.expectEqual(if (registry) diagnostics.Stage.registry_directory else .pki_directory, diagnostic.stage); + try std.testing.expectEqual(if (registry) diagnostics.AgentError.RegistryPermissions else .InvalidManagedState, diagnostic.reason); try std.testing.expectEqualStrings("private-sentinel", try bytes(ctx, path)); } } @@ -665,3 +678,65 @@ test "station install bootstraps and verifies with zero clients and enrollment c try std.testing.expect(f.equal(ca, try station.loadCa(ctx))); try std.testing.expectEqualStrings(server, try bytes(ctx, f.base ++ "/server/server.crt")); } + +test "native station bootstrap matrix includes absent partial and exact empty production directories" { + for (0..3) |kind| { + var fixture = try Fixture.init(); + defer fixture.deinit(); + const ctx = fixture.context(); + if (kind == 0) try ctx.store.removeEmpty(f.base); + if (kind >= 1) _ = try ctx.store.directory(f.base ++ "/pki", ctx.store.root_owner, 0o700, true); + if (kind == 2) { + _ = try ctx.store.directory(f.base ++ "/clients", ctx.store.root_owner, 0o700, true); + _ = try ctx.store.registry(ctx.ingestion.gid, true); + _ = try ctx.store.directory(f.state ++ "/ingestion", ctx.ingestion, 0o750, true); + } + const operation_lock = try runtime.lock(std.testing.io, fixture.temp.dir); + defer operation_lock.close(std.testing.io); + try std.testing.expect(try station.ensureStation(ctx, "monitoring.baptizeddragon.com")); + const ca = try station.loadCa(ctx); + const server = try bytes(ctx, f.base ++ "/server/server.crt"); + _ = try station.verifyServer(ctx, "monitoring.baptizeddragon.com", false); + try std.testing.expect(!try station.ensureStation(ctx, "monitoring.baptizeddragon.com")); + try std.testing.expect(f.equal(ca, try station.loadCa(ctx))); + try std.testing.expectEqualStrings(server, try bytes(ctx, f.base ++ "/server/server.crt")); + for ([_][]const u8{ "registry", "clients" }) |name| try std.testing.expectEqual(@as(usize, 0), (try ctx.store.names(try ctx.store.path(f.base, name))).len); + try std.testing.expect(!try ctx.store.exists(f.etc ++ "/apps")); + } +} + +test "native unpublished CA staging recovers while unrecognized files fail closed" { + var fixture = try Fixture.init(); + defer fixture.deinit(); + const ctx = fixture.context(); + const parent = f.base ++ "/pki"; + _ = try ctx.store.directory(parent, ctx.store.root_owner, 0o700, true); + const stage = try ctx.store.temporary(parent, "bundle", true); + try ctx.store.write(try ctx.store.path(stage, ".dragontools-managed"), f.marker, ctx.store.root_owner, 0o400); + try ctx.store.write(try ctx.store.path(stage, "ca.key"), "incomplete private candidate", ctx.store.root_owner, 0o600); + try std.testing.expect(try station.ensureStation(ctx, "station.example")); + try std.testing.expect(!try ctx.store.exists(stage)); + const ca = try station.loadCa(ctx); + try std.testing.expect(!try station.ensureStation(ctx, "station.example")); + try ctx.store.write(parent ++ "/unexpected.key", "private-sentinel", ctx.store.root_owner, 0o400); + try std.testing.expectError(error.UnexpectedManagedFile, station.ensureStation(ctx, "station.example")); + try std.testing.expectError(error.UnexpectedManagedFile, station.verifyServer(ctx, "station.example", false)); + try std.testing.expect(f.equal(ca, try station.loadCa(ctx))); + try std.testing.expectEqualStrings("private-sentinel", try bytes(ctx, parent ++ "/unexpected.key")); +} + +test "native CA and server publication interruptions reuse already published keys" { + for ([_][]const u8{ f.base ++ "/pki/ca", f.base ++ "/server" }) |path| { + var fixture = try Fixture.init(); + defer fixture.deinit(); + const ctx = fixture.context(); + var fault: Fault = .{ .event = "after_publish", .path = path }; + try std.testing.expectError(error.InjectedInterruption, station.ensureStation(fault.context(ctx), "station.example")); + const ca = try station.loadCa(ctx); + const server = if (try ctx.store.exists(f.base ++ "/server")) try bytes(ctx, f.base ++ "/server/server.crt") else null; + try std.testing.expectEqual(server == null, try station.ensureStation(ctx, "station.example")); + try std.testing.expect(f.equal(ca, try station.loadCa(ctx))); + if (server) |saved| try std.testing.expectEqualStrings(saved, try bytes(ctx, f.base ++ "/server/server.crt")); + try std.testing.expect(!try station.ensureStation(ctx, "station.example")); + } +} diff --git a/src/cli/help.zig b/src/cli/help.zig index e53b485..6ee4080 100644 --- a/src/cli/help.zig +++ b/src/cli/help.zig @@ -37,7 +37,7 @@ pub fn render(a: std.mem.Allocator, node: spec.Node) ![]const u8 { try w.print("{s}\n\nUsage:\n ", .{item.description}); try writePath(w, node); if (item.command) |command| { - try w.writeAll(if (command == .version) " [--json]\n" else if (command == .maintenance_check) " [--ssh-host ALIAS | --host HOST] [options]\n" else if (spec.applicationCommand(command)) (if (command == .app_apply) " [--config PATH] [--plan]\n" else " [--config PATH]\n") else if (spec.flagAllowed(spec.flag("--config").?, command)) " (--config PATH | --ssh-host ALIAS | --host HOST) [options]\n" else if (spec.flagAllowed(spec.flag("--station").?, command)) " (--ssh-host ALIAS | --host HOST) --station ALIAS [options]\n" else if (spec.flagAllowed(spec.flag("--ssh-host").?, command)) " (--ssh-host ALIAS | --host HOST) [options]\n" else " --host HOST [options]\n"); + try w.writeAll(if (command == .version) " [--json]\n" else if (command == .maintenance_check) " [--ssh-host ALIAS | --host HOST] [options]\n" else if (spec.applicationCommand(command)) (if (command == .app_apply) " [--config PATH] [--plan]\n" else " [--config PATH]\n") else if (spec.flagAllowed(spec.flag("--config").?, command)) " [--config PATH | --ssh-host ALIAS | --host HOST] [options]\n" else if (spec.flagAllowed(spec.flag("--station").?, command)) " (--ssh-host ALIAS | --host HOST) --station ALIAS [options]\n" else if (spec.flagAllowed(spec.flag("--ssh-host").?, command)) " (--ssh-host ALIAS | --host HOST) [options]\n" else " --host HOST [options]\n"); } else { var has_children = false; for (spec.commands) |child| { @@ -77,7 +77,7 @@ pub fn render(a: std.mem.Allocator, node: spec.Node) ![]const u8 { if (spec.flagAllowed(spec.flag("--ssh-host").?, command)) { try w.writeAll("\nAlias mode: OpenSSH resolves HostName, User, Port, IdentityAgent, IdentityFile\nand ProxyJump through normal SSH configuration. Do not combine --ssh-host\nwith direct connection options. Direct mode defaults: user root, port 22,\nenvironment agent/default identities. Strict host-key checking is always enabled.\n"); } else if (!spec.applicationCommand(command)) try w.writeAll("\nSSH defaults: user root, port 22, environment agent/default identities.\nStrict host-key checking is always enabled. Explicit authentication modes are exclusive.\n"); - if (!spec.applicationCommand(command) and spec.flagAllowed(spec.flag("--config").?, command)) try w.writeAll("\nConfiguration is explicit: no default file is searched. Version 1 supports only\nconnection.ssh_host and Grafana username/password { op = \"op://...\" } references.\nRelative config paths are allowed. Literal credentials and unknown keys fail.\nCLI values override config; --host replaces the configured SSH alias.\nBoth Grafana references are required together after merging. Help, completion,\nstatus and --plan never resolve secrets; install and verify resolve them locally.\nWithout references, Grafana administrator credentials remain unmanaged.\n"); + if (spec.stationCommand(command)) try w.writeAll("\nLoads optional ./station.toml from CWD; --config selects a different file.\nNo parent/home/XDG search. Missing default permits CLI-only usage. Version 1\nsupports connection.ssh_host, station.hostname, Grafana/Telegram secret references\nand named probes. Deprecated [ingress].hostname is accepted unless conflicting.\nRelative config paths are allowed. Literal credentials and unknown keys fail.\nCLI values override config; --host replaces the configured SSH alias and\n--ingress-hostname overrides station.hostname. Credential references are paired.\nHelp, completion, status and --plan never resolve secrets; install and verify\nresolve Grafana references locally; Telegram resolves only during install.\nWithout references, Grafana administrator credentials remain unmanaged.\n"); } try w.writeAll("\n --help\n Show help for this command.\n"); @@ -178,7 +178,7 @@ pub fn render(a: std.mem.Allocator, node: spec.Node) ![]const u8 { \\ServiceProbeFailed alerts after probe_success == 0 for 2m. A down target does not fail station installation. \\Telegram resolves locally during install only; protected files never enter argv or configuration. \\Status reads stored probe metrics. Verify is read-only and sends no test alerts. - \\Send a test explicitly with monitoring notify-test --config monitoring.toml. + \\Send a test explicitly with monitoring notify-test (reads ./station.toml). \\Vector logs/host metrics and optional vmagent app metrics use monitoring agents. \\OTel traces agents, dashboards, firewall, TLS and legacy Telegram flags are unavailable. \\Unavailable options are validated, then rejected before SSH, including with --plan. @@ -205,7 +205,8 @@ test "workflow help has contextual options and explicit availability" { const a = std.testing.allocator; const install = try render(a, .install); defer a.free(install); - try std.testing.expect(std.mem.indexOf(u8, install, "dragontool monitoring install (--config PATH | --ssh-host ALIAS | --host HOST)") != null); + try std.testing.expect(std.mem.indexOf(u8, install, "dragontool monitoring install [--config PATH | --ssh-host ALIAS | --host HOST]") != null); + try std.testing.expect(std.mem.indexOf(u8, install, "./station.toml") != null); try std.testing.expect(std.mem.indexOf(u8, install, "--grafana-user-op") != null); try std.testing.expect(std.mem.indexOf(u8, install, "--grafana-password-op") != null); try std.testing.expect(std.mem.indexOf(u8, install, "status and --plan never resolve secrets") != null); diff --git a/src/cli/parse.zig b/src/cli/parse.zig index d003469..3a97285 100644 --- a/src/cli/parse.zig +++ b/src/cli/parse.zig @@ -204,7 +204,8 @@ pub fn parse(a: std.mem.Allocator, args: []const []const u8) !Options { i += 1; try assign(a, &o, key, args[i]); } - try validateMerged(o, o.config_path == null); + // Station connection/credential completeness follows optional default load. + try validateMerged(o, o.config_path == null and !spec.stationCommand(o.command)); return o; } @@ -262,11 +263,18 @@ pub fn loadAndMerge(a: std.mem.Allocator, io: std.Io, o: *Options) !void { o.application_config = try application.load(a, io, o.config_path orelse application.default_path); return; } - if (o.config_path) |path_value| { - var values = try config.load(a, io, path_value); + if (spec.stationCommand(o.command)) return loadStation(a, io, o, .cwd()); + try validateMerged(o.*, true); +} +fn loadStation(a: std.mem.Allocator, io: std.Io, o: *Options, dir: std.Io.Dir) !void { + const loaded = if (o.config_path) |path_value| try config.loadFrom(a, io, dir, path_value) else try config.loadDefault(a, io, dir); + if (loaded) |loaded_values| { + var values = loaded_values; errdefer values.deinit(); try merge(o, values); - } else try validateMerged(o.*, true); + } else validateMerged(o.*, true) catch |err| { + return if (err == error.HostRequired) error.StationConfigurationRequired else err; + }; } fn mergeText(a: std.mem.Allocator, o: *Options, contents: []const u8) !void { @@ -492,7 +500,11 @@ test "merged config preserves SSH conflicts and validates incomplete credential try std.testing.expectError(error.HostRequired, mergeText(a, &missing, "version = 1")); try std.testing.expectError(error.GrafanaCredentialReferencesRequired, mergeText(a, &missing, "version = 1\n[connection]\nssh_host = 'monitoring'\n[grafana]\nusername = { op = 'op://Example/Grafana/username' }")); try std.testing.expectError(error.InvalidSshHost, mergeText(a, &missing, "version = 1\n[connection]\nssh_host = 'host;id'")); - try std.testing.expectError(error.GrafanaCredentialReferencesRequired, parse(a, &.{ "monitoring", "install", "--ssh-host", "monitoring", "--grafana-user-op", "op://Example/Grafana/username" })); + var partial = try parse(a, &.{ "monitoring", "install", "--ssh-host", "monitoring", "--grafana-user-op", "op://Example/Grafana/username" }); + defer partial.deinit(a); + var empty = std.testing.tmpDir(.{}); + defer empty.cleanup(); + try std.testing.expectError(error.GrafanaCredentialReferencesRequired, loadStation(a, std.testing.io, &partial, empty.dir)); var pair = try parse(a, &.{ "monitoring", "verify", "--ssh-host", "monitoring", "--grafana-user-op", "op://Example/Grafana/username", "--grafana-password-op", "op://Example/Grafana/password" }); defer pair.deinit(a); try std.testing.expect(!pair.unsupported()); @@ -617,3 +629,58 @@ test "station ingress hostname is explicit DNS configuration with CLI precedence try std.testing.expectError(error.InvalidStationHostname, parse(a, &.{ "monitoring", "install", "--ssh-host", "alias", "--ingress-hostname", "https://station.example:9443" })); try std.testing.expectError(error.FlagNotAllowed, parse(a, &.{ "monitoring", "apply", "--ingress-hostname", "station.example" })); } + +test "station commands load only the cwd default and explicit CLI values win" { + const a = std.testing.allocator; + const io = std.testing.io; + var temp = std.testing.tmpDir(.{}); + defer temp.cleanup(); + try temp.dir.writeFile(io, .{ .sub_path = config.default_path, .data = "version=1\n[connection]\nssh_host='monitoring'\n[station]\nhostname='monitoring.baptizeddragon.com'\n" }); + try temp.dir.writeFile(io, .{ .sub_path = "monitoring.toml", .data = "invalid application sentinel" }); + for ([_][]const u8{ "install", "verify", "status", "notify-test" }) |command| { + var o = try parse(a, &.{ "monitoring", command }); + defer o.deinit(a); + try loadStation(a, io, &o, temp.dir); + try std.testing.expectEqualStrings("monitoring", o.ssh_host.?); + try std.testing.expectEqualStrings("monitoring.baptizeddragon.com", o.ingress_hostname.?); + try std.testing.expect(o.application_config == null); + } + var override = try parse(a, &.{ "monitoring", "install", "--ssh-host", "emergency-monitor", "--ingress-hostname", "emergency.example", "--plan" }); + defer override.deinit(a); + try loadStation(a, io, &override, temp.dir); + try std.testing.expectEqualStrings("emergency-monitor", override.ssh_host.?); + try std.testing.expectEqualStrings("emergency.example", override.ingress_hostname.?); + try temp.dir.writeFile(io, .{ .sub_path = "other.toml", .data = "version=1\n[connection]\nssh_host='other-monitor'\n[station]\nhostname='other.example'\n" }); + try temp.dir.writeFile(io, .{ .sub_path = config.default_path, .data = "invalid implicit sentinel" }); + var explicit = try parse(a, &.{ "monitoring", "install", "--config", "other.toml", "--ssh-host", "emergency-monitor" }); + defer explicit.deinit(a); + try loadStation(a, io, &explicit, temp.dir); + try std.testing.expectEqualStrings("emergency-monitor", explicit.ssh_host.?); + try std.testing.expectEqualStrings("other.example", explicit.ingress_hostname.?); +} + +test "missing station default retains CLI-only mode and never falls back to app or parent config" { + const a = std.testing.allocator; + const io = std.testing.io; + var temp = std.testing.tmpDir(.{}); + defer temp.cleanup(); + try temp.dir.writeFile(io, .{ .sub_path = config.default_path, .data = "version=1\n[connection]\nssh_host='parent-must-not-load'\n" }); + try temp.dir.createDir(io, "child", .default_dir); + const child = try temp.dir.openDir(io, "child", .{}); + defer child.close(io); + try child.writeFile(io, .{ .sub_path = "monitoring.toml", .data = application.example }); + for ([_][]const u8{ "install", "verify", "status", "notify-test" }) |command| { + var absent = try parse(a, &.{ "monitoring", command }); + defer absent.deinit(a); + try std.testing.expectError(error.StationConfigurationRequired, loadStation(a, io, &absent, child)); + } + var cli_only = try parse(a, &.{ "monitoring", "install", "--ssh-host", "monitoring", "--ingress-hostname", "monitoring.baptizeddragon.com" }); + defer cli_only.deinit(a); + try loadStation(a, io, &cli_only, child); + try std.testing.expect(cli_only.config_values == null); + var explicit_missing = try parse(a, &.{ "monitoring", "install", "--config", "missing.toml", "--ssh-host", "monitoring" }); + defer explicit_missing.deinit(a); + try std.testing.expectError(error.UnableToReadMonitoringConfig, loadStation(a, io, &explicit_missing, child)); + try child.writeFile(io, .{ .sub_path = config.default_path, .data = "private-invalid-value" }); + try std.testing.expectError(error.InvalidMonitoringConfig, loadStation(a, io, &cli_only, child)); +} diff --git a/src/cli/spec.zig b/src/cli/spec.zig index 7a729fa..403e027 100644 --- a/src/cli/spec.zig +++ b/src/cli/spec.zig @@ -64,7 +64,7 @@ pub const flags = [_]FlagSpec{ .{ .name = "--host", .description = "Direct target host", .metavar = "HOST", .group = "Required", .commands = all }, .{ .name = "--ingress-hostname", .description = "Station mTLS DNS name; required for first install, otherwise reuse managed identity", .metavar = "DNS", .group = "Station ingress", .commands = &.{ .install, .verify, .status } }, .{ .name = "--ssh-host", .description = "OpenSSH host/alias; use normal SSH configuration", .metavar = "ALIAS", .group = "Connection", .commands = native_ssh }, - .{ .name = "--config", .description = "Monitoring TOML path; app commands default to ./monitoring.toml", .metavar = "PATH", .kind = .path, .group = "Configuration", .commands = configured }, + .{ .name = "--config", .description = "Explicit TOML path; station defaults to ./station.toml, app to ./monitoring.toml", .metavar = "PATH", .kind = .path, .group = "Configuration", .commands = configured }, .{ .name = "--user", .description = "SSH user (default: root)", .metavar = "USER", .group = "Connection", .commands = all }, .{ .name = "--port", .description = "SSH port (default: 22)", .metavar = "PORT", .group = "Connection", .commands = all }, .{ .name = "--ssh-sock", .description = "SSH agent socket, including 1Password agent", .metavar = "PATH", .kind = .path, .group = "Connection", .commands = all }, @@ -92,6 +92,9 @@ pub const flags = [_]FlagSpec{ pub fn applicationCommand(command: Command) bool { return command == .app_apply or command == .app_verify or command == .app_status; } +pub fn stationCommand(command: Command) bool { + return command == .install or command == .verify or command == .status or command == .notify_test; +} pub fn flagAllowed(item: FlagSpec, command: Command) bool { return std.mem.indexOfScalar(Command, item.commands, command) != null; } diff --git a/src/config/monitoring.zig b/src/config/monitoring.zig index bdd9780..ad7d74f 100644 --- a/src/config/monitoring.zig +++ b/src/config/monitoring.zig @@ -1,9 +1,10 @@ //! A deliberately narrow TOML v1 document: connection alias and secret references. -//! No resolution, interpolation, include files, implicit discovery or remote I/O. +//! No resolution, interpolation, include files or remote I/O. const std = @import("std"); const probes = @import("../monitoring/probes.zig"); const references = @import("../secrets/reference.zig"); pub const max_bytes = 64 * 1024; +pub const default_path = "./station.toml"; pub const Config = struct { arena: std.heap.ArenaAllocator, @@ -20,7 +21,7 @@ pub const Config = struct { } }; -const Section = enum { root, connection, ingress, grafana, telegram, probe }; +const Section = enum { root, connection, station, ingress, grafana, telegram, probe }; const ProbeTable = struct { name: ?[]const u8 = null, url: ?[]const u8 = null }; @@ -117,6 +118,9 @@ pub fn parse(a: std.mem.Allocator, contents: []const u8) !Config { var version_seen = false; var connection_seen = false; var ingress_seen = false; + var station_seen = false; + var station_hostname: ?[]const u8 = null; + var legacy_hostname: ?[]const u8 = null; var grafana_seen = false; var telegram_seen = false; var probe_tables: std.ArrayList(ProbeTable) = .empty; @@ -148,6 +152,10 @@ pub fn parse(a: std.mem.Allocator, contents: []const u8) !Config { if (connection_seen) return error.DuplicateMonitoringConfigKey; connection_seen = true; section = .connection; + } else if (std.mem.eql(u8, table, "station")) { + if (station_seen) return error.DuplicateMonitoringConfigKey; + station_seen = true; + section = .station; } else if (std.mem.eql(u8, table, "ingress")) { if (ingress_seen) return error.DuplicateMonitoringConfigKey; ingress_seen = true; @@ -179,11 +187,12 @@ pub fn parse(a: std.mem.Allocator, contents: []const u8) !Config { if (config.ssh_host != null) return error.DuplicateMonitoringConfigKey; config.ssh_host = try line.string(storage); }, - .ingress => { + .station, .ingress => { if (!std.mem.eql(u8, key, "hostname")) return error.UnknownMonitoringConfigKey; - if (config.ingress_hostname != null) return error.DuplicateMonitoringConfigKey; - config.ingress_hostname = try line.string(storage); - try @import("application.zig").validateStationHostname(config.ingress_hostname.?); + const target = if (section == .station) &station_hostname else &legacy_hostname; + if (target.* != null) return error.DuplicateMonitoringConfigKey; + target.* = try line.string(storage); + try @import("application.zig").validateStationHostname(target.*.?); }, .grafana => { const target = if (std.mem.eql(u8, key, "username")) &config.grafana_user_op else if (std.mem.eql(u8, key, "password")) &config.grafana_password_op else return error.UnknownMonitoringConfigKey; @@ -206,6 +215,10 @@ pub fn parse(a: std.mem.Allocator, contents: []const u8) !Config { try line.end(); } if (!version_seen) return error.MissingMonitoringConfigVersion; + if (station_hostname) |canonical| if (legacy_hostname) |legacy| { + if (!std.mem.eql(u8, canonical, legacy)) return error.ConflictingStationHostname; + }; + config.ingress_hostname = station_hostname orelse legacy_hostname; if (telegram_seen and (config.telegram_bot_token_op == null or config.telegram_chat_id_op == null)) return error.TelegramCredentialReferencesRequired; const configured = try storage.alloc(probes.Probe, probe_tables.items.len); for (probe_tables.items, configured) |table, *probe| { @@ -218,7 +231,17 @@ pub fn parse(a: std.mem.Allocator, contents: []const u8) !Config { } pub fn load(a: std.mem.Allocator, io: std.Io, path: []const u8) !Config { - const dir = std.Io.Dir.cwd(); + return loadFrom(a, io, .cwd(), path); +} +pub fn loadDefault(a: std.mem.Allocator, io: std.Io, dir: std.Io.Dir) !?Config { + // Only absence is optional. A present invalid/unreadable file fails closed. + _ = dir.statFile(io, default_path, .{ .follow_symlinks = false }) catch |err| switch (err) { + error.FileNotFound => return null, + else => return error.UnableToReadMonitoringConfig, + }; + return try loadFrom(a, io, dir, default_path); +} +pub fn loadFrom(a: std.mem.Allocator, io: std.Io, dir: std.Io.Dir, path: []const u8) !Config { const metadata = dir.statFile(io, path, .{}) catch return error.UnableToReadMonitoringConfig; if (metadata.kind != .file) return error.InvalidMonitoringConfigFile; if (metadata.size > max_bytes) return error.MonitoringConfigTooLarge; @@ -351,3 +374,28 @@ test "Telegram config rejects missing malformed plaintext and unsupported secret try std.testing.expectError(error.DuplicateMonitoringConfigKey, parse(a, "version=1\n[telegram]\n[telegram]")); try std.testing.expectError(error.UnknownMonitoringConfigKey, parse(a, "version=1\n[telegram]\ntoken={op='op://v/i/token'}")); } + +test "station hostname canonical v1 setting and compatible legacy alias" { + const a = std.testing.allocator; + for ([_][]const u8{ + "[station]\nhostname='monitoring.baptizeddragon.com'\n", + "[ingress]\nhostname='monitoring.baptizeddragon.com'\n", + "[station]\nhostname='monitoring.baptizeddragon.com'\n[ingress]\nhostname='monitoring.baptizeddragon.com'\n", + }) |tables| { + const text = try std.fmt.allocPrint(a, "version=1\n{s}", .{tables}); + defer a.free(text); + var value = try parse(a, text); + defer value.deinit(); + try std.testing.expectEqualStrings("monitoring.baptizeddragon.com", value.ingress_hostname.?); + } + for ([_][]const u8{ "https://station.example", "station.example:9443", "station.example/path", "127.0.0.1" }) |hostname| { + const text = try std.fmt.allocPrint(a, "version=1\n[station]\nhostname='{s}'\n", .{hostname}); + defer a.free(text); + try std.testing.expectError(error.InvalidStationHostname, parse(a, text)); + } + for ([_][]const u8{ + "version=1\n[station]\nhostname='one.example'\n[ingress]\nhostname='two.example'\n", + "version=1\n[ingress]\nhostname='one.example'\n[station]\nhostname='two.example'\n", + }) |text| try std.testing.expectError(error.ConflictingStationHostname, parse(a, text)); + try std.testing.expectError(error.DuplicateMonitoringConfigKey, parse(a, "version=1\n[station]\nhostname='one.example'\nhostname='one.example'\n")); +} diff --git a/src/main.zig b/src/main.zig index 00f1941..6ad6e1c 100644 --- a/src/main.zig +++ b/src/main.zig @@ -20,6 +20,9 @@ fn print(io: std.Io, message: []const u8) void { pub fn main(init: std.process.Init) void { run(init) catch |err| { const detail: ?[]const u8 = switch (err) { + error.StationConfigurationRequired => "No station configuration found.\n\nExpected:\n ./station.toml\n\nor provide:\n --config \n or the required CLI options.\n", + error.UnableToReadMonitoringConfig => "Unable to read station config. The default is ./station.toml; use --config for an explicit path. Nothing changed; SSH was not attempted.\n", + error.ConflictingStationHostname => "Conflicting station hostname settings. Use [station].hostname; the v1 [ingress].hostname alias must agree if both are supplied. Nothing changed; SSH was not attempted.\n", error.ApplicationTracesUnsupported => "Traces are declared but not supported by this DragonTools build. Nothing changed; SSH was not attempted.\n", error.ApplicationMetricsAlertsUnsupported => "Custom metrics alerts are not supported by this DragonTools build. Nothing changed; SSH was not attempted.\n", error.UnableToReadApplicationConfig => "Unable to read application config. The default is ./monitoring.toml; use --config for an explicit path. Nothing changed; SSH was not attempted.\n", @@ -102,6 +105,7 @@ fn execute(init: std.process.Init, options: cli.Options) !void { return; } if (options.plan) { + print(init.io, try plan.connection(a, options.ssh_host, options.host, options.ingress_hostname)); print(init.io, try plan.renderStation(a, options.grafana_user_op != null, options.telegram_bot_token_op != null, options.probes.len)); return; } @@ -142,7 +146,7 @@ fn execute(init: std.process.Init, options: cli.Options) !void { "Verification is read-only. Later checks were not attempted. Correct the cause and repeat verification."; print(init.io, try std.fmt.allocPrint(a, "Failed at {s}; {d} steps completed, {d} change steps confirmed. Component: {s}. Check: {s}. {s} Check SSH/prerequisites for detection failures, or inspect the managed unit and journal for later failures.\n", .{ @tagName(report.phase), report.completed, report.changes, if (report.component) |component| component.name() else "host", if (report.check) |check| @tagName(check) else @tagName(report.phase), advice })); print(init.io, try report.credentialDiagnostics(a)); - if (err == error.IngressHostnameRequired) print(init.io, "Fresh station ingress requires --ingress-hostname or [ingress].hostname in the station config. The TLS identity is never inferred from SSH.\n"); + if (err == error.IngressHostnameRequired) print(init.io, "Fresh station ingress requires --ingress-hostname or [station].hostname in the station config. The TLS identity is never inferred from SSH.\n"); return err; }; if (options.command == .install and report.changes == 0) print(init.io, "No changes required.\n"); diff --git a/src/monitoring/agents/tests.zig b/src/monitoring/agents/tests.zig index 6dbba1e..278d59f 100644 --- a/src/monitoring/agents/tests.zig +++ b/src/monitoring/agents/tests.zig @@ -717,6 +717,7 @@ test "station ensure failure reports safe substage and preserves native semantic .{ .code = 93, .failure = error.RemoteOperationFailed, .detail = .server_tls_invalid, .check = .server_tls_invalid }, .{ .code = 94, .failure = error.RemoteOperationFailed, .detail = .client_certificate_rejected, .check = .client_certificate_rejected }, .{ .code = 95, .failure = error.RemoteOperationFailed, .detail = .ingestion_rejected, .check = .ingestion_rejected }, + .{ .code = 96, .failure = error.OperationBusy, .detail = .operation_busy, .check = .operation_busy }, }) |case| { var arena = std.heap.ArenaAllocator.init(std.testing.allocator); defer arena.deinit(); @@ -758,6 +759,42 @@ test "silent or rejected native diagnostics still identify station ensure and ge try std.testing.expectEqualStrings("Stage: station_ensure\nDetail: agent_internal_error\n", try report.state.credentialDiagnostics(a)); } +test "station ensure diagnostics identify fixed managed invariants without exposing remote output" { + const Case = struct { stage: remote.diagnostics.Stage, reason: remote.diagnostics.AgentError = .InvalidManagedState, check: readiness.Check, code: u8 = 86 }; + for ([_]Case{ + .{ .stage = .ingestion_root, .check = .ingestion_root_invalid }, + .{ .stage = .pki_directory, .check = .pki_directory_invalid }, + .{ .stage = .clients_directory, .check = .clients_directory_invalid }, + .{ .stage = .registry_directory, .reason = .RegistryPermissions, .check = .registry_directory_invalid, .code = 89 }, + .{ .stage = .state_directory, .check = .state_directory_invalid }, + .{ .stage = .ca_state, .check = .ca_bundle_invalid }, + .{ .stage = .ca_certificate_validation, .reason = .CertificateValidationFailed, .check = .ca_bundle_invalid }, + .{ .stage = .server_state, .check = .server_identity_invalid }, + .{ .stage = .server_certificate_validation, .reason = .CertificateValidationFailed, .check = .server_identity_invalid }, + .{ .stage = .pki_directory, .reason = .UnexpectedManagedFile, .check = .unexpected_managed_file }, + .{ .stage = .clients_directory, .reason = .UnexpectedSymlink, .check = .unexpected_symlink }, + .{ .stage = .operation_lock, .reason = .OperationBusy, .check = .operation_busy, .code = 96 }, + .{ .stage = .operation_lock, .reason = .OperationLockFailed, .check = .operation_lock_failed }, + }) |case| { + var arena = std.heap.ArenaAllocator.init(std.testing.allocator); + defer arena.deinit(); + const a = arena.allocator(); + var report: model.Report = .{}; + var fake: Fake = .{ .allocator = a, .report = &report, .ensure_failure = .{ + .code = case.code, + .output = "PRIVATE KEY sentinel", + .diagnostic = .{ .stage = case.stage, .reason = case.reason }, + } }; + try std.testing.expectError(if (case.code == 89) error.RegistryPermissionsConflict else if (case.code == 96) error.OperationBusy else error.RemoteOperationFailed, install.install(a, fake.asRemote(), fake.asRemote(), &report, registration)); + try std.testing.expectEqual(case.check, report.state.check.?); + try std.testing.expectEqual(remote.diagnostics.EnrollmentStage.station_ensure, report.state.enrollment_stage.?); + const output = try report.state.credentialDiagnostics(a); + try std.testing.expect(std.mem.indexOf(u8, output, "PRIVATE KEY") == null); + try std.testing.expect(std.mem.indexOf(u8, output, "sentinel") == null); + try std.testing.expectEqual(@as(usize, 0), fake.enrollments); + } +} + test "Caddy listener retries precede enrollment and preserve restart intent on timeout" { for ([_]bool{ false, true }) |timeout| { var arena = std.heap.ArenaAllocator.init(std.testing.allocator); diff --git a/src/monitoring/install.zig b/src/monitoring/install.zig index ebc1425..3b1c996 100644 --- a/src/monitoring/install.zig +++ b/src/monitoring/install.zig @@ -49,6 +49,26 @@ pub const Report = struct { pub fn captureAgentFailure(self: *Report, result: remote.Result) void { self.agent_detail = if (self.enrollment_stage != null) remote.diagnostics.detail(result.agent_exit_code orelse result.code) else null; self.agent_diagnostic = if (self.agent_detail != null) result.diagnostic else null; + if (self.agent_diagnostic) |diagnostic| { + // Fixed invariants only; never forward helper input or raw stderr. + const mapped: ?@import("readiness.zig").Check = switch (diagnostic.reason) { + .OperationBusy => .operation_busy, + .OperationLockFailed => .operation_lock_failed, + .UnexpectedManagedFile => .unexpected_managed_file, + .UnexpectedSymlink => .unexpected_symlink, + else => switch (diagnostic.stage) { + .ingestion_root => .ingestion_root_invalid, + .pki_directory => .pki_directory_invalid, + .clients_directory => .clients_directory_invalid, + .registry_directory => .registry_directory_invalid, + .state_directory => .state_directory_invalid, + .ca_state, .ca_certificate_validation => .ca_bundle_invalid, + .server_state, .server_certificate_validation => .server_identity_invalid, + else => null, + }, + }; + if (mapped) |check| self.check = check; + } } pub fn emit(self: *Report, phase: progress.Phase) void { if (self.progress) |sink| if (self.component) |component| sink.emit(.{ .component = component, .phase = phase, .station_enabled = self.station_enabled }); @@ -103,9 +123,13 @@ pub const Report = struct { return error.ClientIdentityInconsistent; }, 89 => { - self.check = .registry_permissions; + if (self.check != .registry_directory_invalid) self.check = .registry_permissions; return error.RegistryPermissionsConflict; }, + 96 => { + self.check = .operation_busy; + return error.OperationBusy; + }, 90 => { self.check = .ingress_hostname_required; return error.IngressHostnameRequired; diff --git a/src/monitoring/plan.zig b/src/monitoring/plan.zig index a235d3c..008b6a6 100644 --- a/src/monitoring/plan.zig +++ b/src/monitoring/plan.zig @@ -9,6 +9,14 @@ const logs_plugin = @import("../components/grafana_victorialogs_plugin.zig"); pub const unavailable = "Not yet available: dashboards, OTel Collector/traces agents, HostDown and systemd-service state alerts, firewall, public Grafana TLS, automatic OS upgrades.\n"; +pub fn connection(a: std.mem.Allocator, alias: ?[]const u8, host: []const u8, hostname: ?[]const u8) ![]const u8 { + return std.fmt.allocPrint(a, "Connection:\n {s}: {s}\n\nStation:\n hostname: {s}\n metrics ingress: :9443\n logs ingress: :9444\n\n", .{ + if (alias != null) "SSH alias" else "direct host", + alias orelse host, + hostname orelse "reuse managed identity (explicit hostname required for first install)", + }); +} + pub fn renderStation(a: std.mem.Allocator, grafana_configured: bool, telegram_configured: bool, probe_count: usize) ![]const u8 { const core = try renderWithCredentials(a, grafana_configured); defer a.free(core); @@ -65,7 +73,7 @@ pub fn renderWithCredentials(a: std.mem.Allocator, configured: bool) ![]const u8 "Grafana access: SSH port forwarding only; public Grafana HTTPS/TLS and firewall management are unavailable.\n\n" ++ "Safe rerun: inspect actual state, resume pending activation, and verify before finalization. Healthy unchanged services are not restarted; unchanged valid binaries are not downloaded. No controller state database.\n" ++ "Host metric rules use verified Vector contracts; systemd-service state alerts remain deferred.\n" ++ - "Station install owns CA/server PKI, Caddy v2.11.4 mTLS 0.0.0.0:9443 metrics / 0.0.0.0:9444 logs and private ingress authorization. Use --ingress-hostname or [ingress].hostname for a fresh station; reruns may reuse the managed server identity. Zero registered clients is healthy; :9445 stays closed. Application apply enrolls clients and installs agents only after station ingress verification.\n" ++ + "Station install owns CA/server PKI, Caddy v2.11.4 mTLS 0.0.0.0:9443 metrics / 0.0.0.0:9444 logs and private ingress authorization. Use --ingress-hostname or [station].hostname for a fresh station; reruns may reuse the managed server identity. Zero registered clients is healthy; :9445 stays closed. Application apply enrolls clients and installs agents only after station ingress verification.\n" ++ unavailable ++ "No remote operations performed.\n", .{ vm.version, diff --git a/src/monitoring/readiness.zig b/src/monitoring/readiness.zig index e0700f0..913d0c0 100644 --- a/src/monitoring/readiness.zig +++ b/src/monitoring/readiness.zig @@ -41,6 +41,17 @@ pub const Check = enum { ingestion_rejected, ca_maintenance, registry_permissions, + ingestion_root_invalid, + pki_directory_invalid, + clients_directory_invalid, + registry_directory_invalid, + state_directory_invalid, + unexpected_managed_file, + unexpected_symlink, + ca_bundle_invalid, + server_identity_invalid, + operation_busy, + operation_lock_failed, credential_recovery, client_identity_inconsistent, host_metrics_ready, diff --git a/tests/cli_smoke.py b/tests/cli_smoke.py index 5ebb146..4c09434 100644 --- a/tests/cli_smoke.py +++ b/tests/cli_smoke.py @@ -26,11 +26,18 @@ DRAGONTOOLS_TEST_MARKER=str(marker), DRAGONTOOLS_PROVIDER_MARKER=str(provider_marker), NO_COLOR="1", TERM="dumb") checked = 0 + empty_repository = directory / "empty-repository" + empty_repository.mkdir() + + def run_process(*args, **kwargs): + # Never consume a developer's station.toml from the checkout running tests. + kwargs["cwd"] = kwargs.get("cwd") or empty_repository + return subprocess.run(*args, **kwargs) def local_run(args, code=0, expected=None, cwd=None): """Pipe stdin explicitly so no-argument/wizard checks cannot read the terminal.""" global checked - result = subprocess.run([str(binary), *args], env=env, input="", cwd=cwd, + result = run_process([str(binary), *args], env=env, input="", cwd=cwd, capture_output=True, text=True, timeout=15) output = result.stdout + result.stderr assert result.returncode == code, (args, result.returncode, output) @@ -66,6 +73,45 @@ def local_run(args, code=0, expected=None, cwd=None): name = "orders" url = "https://orders.example.com/healthz" ''') + # Station defaults use only this CWD; explicit config replaces, never merges it. + station_repository = directory / "station-repository" + station_repository.mkdir() + implicit_station = station_repository / "station.toml" + station_text = station_config.read_text() + '\n[station]\nhostname="monitoring.baptizeddragon.com"\n' + implicit_station.write_text(station_text) + implicit = local_run(["monitoring", "install", "--plan"], cwd=station_repository).stdout + explicit = local_run(["monitoring", "install", "--config", str(implicit_station), "--plan"]).stdout + assert implicit == explicit + for value in ("SSH alias: monitoring", "hostname: monitoring.baptizeddragon.com", + "metrics ingress: :9443", "logs ingress: :9444"): + assert value in implicit, implicit + override = local_run(["monitoring", "install", "--plan", "--ssh-host", "emergency-monitor", + "--ingress-hostname", "alternate.example"], cwd=station_repository).stdout + assert "SSH alias: emergency-monitor" in override and "hostname: alternate.example" in override + assert "monitoring.baptizeddragon.com" not in override + for command in ("install", "verify", "status", "notify-test"): + local_run(["monitoring", command], 1, "StationConfigurationRequired") + implicit_station.write_text('version=1\n[station]\nhostname="https://REDACTION-SENTINEL"\n') + local_run(["monitoring", command], 1, "InvalidStationHostname", cwd=station_repository) + # Invalid implicit config is ignored when an explicit file was selected. + local_run(config_args, expected="No remote operations performed", cwd=station_repository) + implicit_station.write_text(station_text) + for command in ("apply", "app-verify", "app-status"): + local_run(["monitoring", command], 1, "UnableToReadApplicationConfig", cwd=station_repository) + child_repository = station_repository / "child" + child_repository.mkdir() + local_run(["monitoring", "install", "--plan"], 1, "StationConfigurationRequired", cwd=child_repository) + local_run([*plan_args, "--ingress-hostname", "station.example"], cwd=child_repository) + (child_repository / "monitoring.toml").write_text('version=1\n[connection]\nssh_host="monitoring"\n') + local_run(["monitoring", "install", "--plan"], 1, "StationConfigurationRequired", cwd=child_repository) + for bad in ("https://station.example", "station.example:9443", "station.example/path", "127.0.0.1"): + implicit_station.write_text('version=1\n[connection]\nssh_host="monitoring"\n[station]\nhostname="' + bad + '"\n') + local_run(["monitoring", "install", "--plan"], 1, "InvalidStationHostname", cwd=station_repository) + implicit_station.write_text(station_text + '\n[ingress]\nhostname="different.example"\n') + local_run(["monitoring", "install", "--plan"], 1, "ConflictingStationHostname", cwd=station_repository) + implicit_station.write_text(station_text + '\n[ingress]\nhostname="monitoring.baptizeddragon.com"\n') + local_run(["monitoring", "install", "--plan"], cwd=station_repository) + implicit_station.write_text(station_text) # Application repository defaults are one local file, never station config or # secret resolution. Plans and invalid configs must never spawn SSH. app_repository = directory / "application-repository" @@ -181,10 +227,10 @@ def local_run(args, code=0, expected=None, cwd=None): 1, "FlagNotAllowed"), (["host", "install-oh-my-zsh", "--ssh-host", "monitoring", "--ssh-op-path", "op://vault/item/key"], 1, "FlagNotAllowed"), - (["monitoring", "install", "--ssh-host", "REDACTION-SENTINEL", "--plan"], 0, "Grafana: loopback:3000"), + (["monitoring", "install", "--ssh-host", "monitoring", "--plan"], 0, "Grafana: loopback:3000"), (config_args, 0, configured_credentials), (["monitoring", "install", "--config", str(station_config), "--plan"], 0, "External HTTP probes: 2 configured"), - (["monitoring", "install", "--config", os.path.relpath(config), "--plan"], 0, configured_credentials), + (["monitoring", "install", "--config", os.path.relpath(config, empty_repository), "--plan"], 0, configured_credentials), ([*config_args, "--ssh-host", "other"], 0, configured_credentials), ([*config_args, "--host", "example.com", "--user", "ops", "--port", "2222"], 0, configured_credentials), ([*config_args, "--user", "root"], 1, "ConflictingSshMode"), @@ -374,7 +420,7 @@ def local_run(args, code=0, expected=None, cwd=None): if shell_binary is None: print(f"SKIP: {shell} completion syntax (shell not installed)") continue - syntax = subprocess.run([shell_binary, "-n", str(script)], env=env, + syntax = run_process([shell_binary, "-n", str(script)], env=env, input="", capture_output=True, text=True, timeout=15) assert syntax.returncode == 0, (shell, syntax.stderr) checked += 1 @@ -382,7 +428,7 @@ def local_run(args, code=0, expected=None, cwd=None): bash = shutil.which("bash") if bash: def bash_complete(words): - result = subprocess.run( + result = run_process( [bash, "--noprofile", "--norc", "-c", 'source "$1"\nshift\nCOMP_WORDS=("$@")\n' 'COMP_CWORD=$((${#COMP_WORDS[@]} - 1))\n' @@ -438,7 +484,7 @@ def bash_complete(words): if zsh: def zsh_candidates(words): # Capture the candidates handed to Zsh's native UI. No terminal is needed. - result = subprocess.run( + result = run_process( [zsh, "-f", "-c", 'script=$1\nshift\nwords=("$@")\nCURRENT=${#words[@]}\n' '_describe() { print -rl -- "${candidates[@]}"; }\n' @@ -478,7 +524,7 @@ def zsh_candidates(words): fish = shutil.which("fish") if fish: def fish_complete(command): - result = subprocess.run( + result = run_process( [fish, "--no-config", "-c", 'source "$argv[1]"\ncomplete -C "$argv[2]"', str(scripts["fish"]), command], env=env, input="", capture_output=True, text=True, timeout=15) @@ -525,7 +571,7 @@ def fish_complete(command): for command in ("install", "verify", "status"): for connection in (["--host", "example.com"], ["--ssh-host", "monitoring"]): marker.unlink(missing_ok=True) - result = subprocess.run([str(binary), "monitoring", command, *connection], + result = run_process([str(binary), "monitoring", command, *connection], env=env, input="", capture_output=True, text=True, timeout=15) assert marker.exists(), (command, connection, "The supported workflow did not invoke SSH") assert result.returncode == 1, (command, result.stdout, result.stderr) @@ -541,7 +587,7 @@ def fish_complete(command): # A normal status command reads references from config but must never resolve # them. It only reaches the intentionally failing fake SSH transport. marker.unlink(missing_ok=True) - status = subprocess.run([str(binary), "monitoring", "status", "--config", str(config)], + status = run_process([str(binary), "monitoring", "status", "--config", str(config)], env=env, input="", capture_output=True, text=True, timeout=15) assert marker.exists() and status.returncode == 1, (status.stdout, status.stderr) assert "Failed at status;" in status.stdout, status.stdout @@ -564,7 +610,7 @@ def fish_complete(command): esac """) marker.unlink(missing_ok=True) - status = subprocess.run([str(binary), "monitoring", "status", "--config", str(config)], + status = run_process([str(binary), "monitoring", "status", "--config", str(config)], env=env, input="", capture_output=True, text=True, timeout=15) assert status.returncode == 0, (status.stdout, status.stderr) assert marker.read_text() == "status\n" * 10 + "probes\n" @@ -588,7 +634,7 @@ def fish_complete(command): op.chmod(0o755) for command in ("install", "verify"): provider_marker.unlink(missing_ok=True) - result = subprocess.run([str(binary), "monitoring", command, "--config", str(config)], + result = run_process([str(binary), "monitoring", command, "--config", str(config)], env=env, input="", capture_output=True, text=True, timeout=15) assert result.returncode == 1 and provider_marker.exists(), (result.stdout, result.stderr) assert not marker.exists(), "Credential resolution failure reached SSH" @@ -605,7 +651,7 @@ def fish_complete(command): + "sys.stdout.write(values[sys.argv[-1].rsplit('/', 1)[1]])\n") for command in ("install", "verify"): provider_marker.unlink(missing_ok=True) - result = subprocess.run([str(binary), "monitoring", command, "--config", str(config)], + result = run_process([str(binary), "monitoring", command, "--config", str(config)], env=env, input="", capture_output=True, text=True, timeout=15) output = result.stdout + result.stderr assert result.returncode == 1 and expected_error in output, output @@ -619,7 +665,7 @@ def fish_complete(command): shutil.copy(ssh, missing_provider_bin / "ssh") without_op = dict(env, PATH=str(missing_provider_bin)) for command in ("install", "verify"): - result = subprocess.run([str(binary), "monitoring", command, "--config", str(config)], + result = run_process([str(binary), "monitoring", command, "--config", str(config)], env=without_op, input="", capture_output=True, text=True, timeout=15) output = result.stdout + result.stderr assert result.returncode == 1 and "GrafanaUsernameResolutionFailed" in output, output @@ -630,7 +676,7 @@ def fish_complete(command): for connection in (["--ssh-host", "monitoring"], ["--host", "example.com", "--user", "root"]): marker.unlink(missing_ok=True) - result = subprocess.run([str(binary), "host", "install-oh-my-zsh", *connection], + result = run_process([str(binary), "host", "install-oh-my-zsh", *connection], env=env, input="", capture_output=True, text=True, timeout=15) assert marker.exists(), (connection, "Host workflow did not invoke SSH") assert result.returncode == 1, (connection, result.stdout, result.stderr) @@ -659,7 +705,7 @@ def fish_complete(command): ''') for connection in (["--host", "example.com"], ["--ssh-host", "monitoring"]): marker.unlink(missing_ok=True) - result = subprocess.run([str(binary), "monitoring", "verify", *connection], + result = run_process([str(binary), "monitoring", "verify", *connection], env=env, input="", capture_output=True, text=True, timeout=15) assert result.returncode == 1, (result.stdout, result.stderr) assert "Component: VictoriaMetrics. Check: self_scrape_ready." in result.stdout, result.stdout @@ -793,7 +839,7 @@ def python_arguments(text, needle, depth=0): for command in ("install", "verify", "install"): marker.unlink(missing_ok=True) provider_marker.unlink(missing_ok=True) - result = subprocess.run([str(binary), "monitoring", command, "--config", str(config)], + result = run_process([str(binary), "monitoring", command, "--config", str(config)], env=env, input="", capture_output=True, text=True, timeout=30) assert result.returncode == 0, (command, result.stdout, result.stderr) output = result.stdout + result.stderr @@ -827,7 +873,7 @@ def python_arguments(text, needle, depth=0): for command in ("install", "verify"): marker.unlink(missing_ok=True) provider_marker.unlink(missing_ok=True) - result = subprocess.run([str(binary), "monitoring", command, "--ssh-host", "monitoring"], + result = run_process([str(binary), "monitoring", command, "--ssh-host", "monitoring"], env=env, input="", capture_output=True, text=True, timeout=30) assert result.returncode == 0, (command, result.stdout, result.stderr) assert not marker.exists() and not provider_marker.exists() @@ -845,7 +891,7 @@ def python_arguments(text, needle, depth=0): marker.unlink(missing_ok=True) provider_marker.unlink(missing_ok=True) command_env = dict(station_env, DRAGONTOOLS_ASSERT_READONLY="1") if command == "verify" else station_env - result = subprocess.run([str(binary), "monitoring", command, "--config", str(station_config)], + result = run_process([str(binary), "monitoring", command, "--config", str(station_config)], env=command_env, input="", capture_output=True, text=True, timeout=30) output = result.stdout + result.stderr assert result.returncode == 0, output @@ -865,7 +911,7 @@ def python_arguments(text, needle, depth=0): marker.unlink(missing_ok=True) provider_marker.unlink(missing_ok=True) - result = subprocess.run([str(binary), "monitoring", "status", "--config", str(station_config)], + result = run_process([str(binary), "monitoring", "status", "--config", str(station_config)], env=station_env, input="", capture_output=True, text=True, timeout=30) output = result.stdout + result.stderr assert result.returncode == 0, output @@ -875,7 +921,7 @@ def python_arguments(text, needle, depth=0): assert "REDACTION-SENTINEL" not in output checked += 1 - result = subprocess.run([str(binary), "monitoring", "notify-test", "--config", str(station_config)], + result = run_process([str(binary), "monitoring", "notify-test", "--config", str(station_config)], env=dict(station_env, DRAGONTOOLS_ALLOW_NOTIFY="1"), input="", capture_output=True, text=True, timeout=30) output = result.stdout + result.stderr assert result.returncode == 0, output @@ -891,7 +937,7 @@ def python_arguments(text, needle, depth=0): # without leaking commands, credentials, upstream response or stderr. marker.unlink(missing_ok=True) provider_marker.unlink(missing_ok=True) - result = subprocess.run([str(binary), "monitoring", "verify", "--config", str(config)], + result = run_process([str(binary), "monitoring", "verify", "--config", str(config)], env=dict(env, DRAGONTOOLS_LOGS_FAIL="1"), input="", capture_output=True, text=True, timeout=30) output = result.stdout + result.stderr @@ -907,7 +953,7 @@ def python_arguments(text, needle, depth=0): # Fresh station identity is explicit. Reusing a saved identity is allowed; # an absent one produces a safe actionable station error, never alias inference. for flags, expected in (([], 1), (["--ingress-hostname", "station.example"], 0)): - result = subprocess.run([str(binary), "monitoring", "install", "--ssh-host", "management-alias", *flags], + result = run_process([str(binary), "monitoring", "install", "--ssh-host", "management-alias", *flags], env=dict(env, DRAGONTOOLS_FRESH_STATION="1"), input="", capture_output=True, text=True, timeout=30) assert result.returncode == expected, result.stdout + result.stderr if expected: @@ -978,7 +1024,7 @@ def python_arguments(text, needle, depth=0): ): marker.unlink(missing_ok=True) provider_marker.unlink(missing_ok=True) - result = subprocess.run([str(binary), 'monitoring', 'apply', '--config', str(app_config)], + result = run_process([str(binary), 'monitoring', 'apply', '--config', str(app_config)], env=dict(env, DRAGONTOOLS_AGENT_CODE=str(code), DRAGONTOOLS_AGENT_DIAGNOSTIC=diagnostic), input='', capture_output=True, text=True, timeout=30) output = result.stdout + result.stderr diff --git a/tests/station_bootstrap_test.py b/tests/station_bootstrap_test.py new file mode 100644 index 0000000..f3bf9fa --- /dev/null +++ b/tests/station_bootstrap_test.py @@ -0,0 +1,174 @@ +#!/usr/bin/env python3 +"""Real native helper in an isolated Linux filesystem; no SSH/systemd/network. + +Requires root for chroot and distinct root/dt-ingest ownership. Portable native +PKI and lock tests run independently on both Linux and macOS in zig build test. +""" +import argparse +import fcntl +import hashlib +import json +import os +from pathlib import Path +import shutil +import stat +import subprocess +import sys +import tempfile +import unittest + +parser = argparse.ArgumentParser() +parser.add_argument("--agent", type=Path, required=True) +args = parser.parse_args() +AGENT = args.agent.resolve() +HOSTNAME = "monitoring.baptizeddragon.com" +INGEST = 1001 + + +@unittest.skipUnless(sys.platform == "linux" and os.geteuid() == 0, + "Linux root required for isolated ownership/chroot fixture") +class StationBootstrap(unittest.TestCase): + def setUp(self): + self.temporary = tempfile.TemporaryDirectory(prefix="dragontools-station-") + self.addCleanup(self.temporary.cleanup) + self.root = Path(self.temporary.name) + self.directory("etc/dragontools", 0o755) + self.directory("var/lib/dragontools", 0o755) + self.directory("dev", 0o755) + # The embedded crypto entropy provider opens the OS random device. + os.mknod(self.root / "dev/random", stat.S_IFCHR | 0o600, os.makedev(1, 8)) + (self.root / "etc/passwd").write_text( + "root:x:0:0:root:/root:/bin/false\n" + f"dt-ingest:x:{INGEST}:{INGEST}:ingress:/nonexistent:/bin/false\n") + shutil.copyfile(AGENT, self.root / "dragontool-agent") + (self.root / "dragontool-agent").chmod(0o755) + self.base = self.root / "etc/dragontools/ingestion" + + def directory(self, path, mode, uid=0, gid=0): + value = self.root / path + value.mkdir(parents=True, exist_ok=True) + value.chmod(mode) + os.chown(value, uid, gid) + return value + + def skeleton(self, registry_mode=0o750): + self.directory("etc/dragontools/ingestion", 0o755) + for name in ("pki", "clients"): + self.directory(f"etc/dragontools/ingestion/{name}", 0o700) + self.directory("etc/dragontools/ingestion/registry", registry_mode, 0, INGEST) + self.directory("var/lib/dragontools/ingestion", 0o750, INGEST, INGEST) + + def call(self, action="station-ensure", hostname=HOSTNAME, code=0, + output=b"changed", stage=None, reason=None): + def isolate(): + os.chroot(self.root) + os.chdir("/") + result = subprocess.run( + ["/dragontool-agent", "internal", "--stdin", "--diagnostics"], + input=json.dumps({"action": action, "args": [hostname]}).encode(), + capture_output=True, timeout=30, preexec_fn=isolate, + env={"PATH": "/nonexistent"}) + # Assertions never include captured private contents in failure output. + safe_stage = next((name for name in ( + "native_initialization", "operation_lock", "ingestion_root", "pki_directory", + "clients_directory", "registry_directory", "state_directory", "ca_state", + "ca_key_generation", "ca_certificate_generation", "ca_certificate_validation", + "ca_key_serialization", "ca_publication", "server_state", "server_key_generation", + "server_certificate_generation", "server_certificate_validation", + "server_key_serialization", "server_publication") + if f"AgentStage: {name}\n".encode() in result.stderr), "unrecognized") + self.assertEqual(result.returncode, code, f"safe failure stage: {safe_stage}") + self.assertTrue(result.stdout == output, "unexpected native response") + expected = b"" if stage is None else f"AgentStage: {stage}\nAgentError: {reason}\n".encode() + self.assertTrue(result.stderr == expected, "unexpected safe diagnostic") + self.assertNotIn(b"PRIVATE KEY", result.stdout + result.stderr) + return result + + def snapshot(self): + return {str(p.relative_to(self.root)): + (hashlib.sha256(p.read_bytes()).digest(), p.stat().st_mtime_ns, + p.stat().st_mode, p.stat().st_uid, p.stat().st_gid) + for p in self.base.rglob("*") if p.is_file()} + + def converged(self): + self.call() + self.assertTrue((self.base / "pki/ca/ca.key").is_file()) + self.assertTrue((self.base / "server/server.key").is_file()) + self.assertEqual((self.base / "server/endpoint").read_text(), HOSTNAME) + self.assertFalse((self.root / "etc/dragontools/apps").exists()) + for name in ("clients", "registry"): + self.assertEqual(list((self.base / name).iterdir()), []) + before = self.snapshot() + self.call("station-verify", output=b"unchanged") + self.call(output=b"unchanged") + self.assertEqual(before, self.snapshot()) + + def test_exact_empty_production_tree_then_noop(self): + self.skeleton() + self.converged() + + def test_absent_ingestion_tree(self): + self.converged() + + def test_interrupted_parents(self): + self.directory("etc/dragontools/ingestion", 0o755) + self.directory("etc/dragontools/ingestion/pki", 0o700) + self.converged() + + def test_registry_previous_generation_migrates(self): + self.skeleton(0o700) + self.converged() + self.assertEqual((self.base / "registry").stat().st_mode & 0o7777, 0o750) + + def test_unknown_registry_mode_refused(self): + self.skeleton(0o710) + self.call(code=89, output=b"", stage="registry_directory", reason="RegistryPermissions") + self.assertFalse((self.base / "pki/ca").exists()) + self.assertEqual((self.base / "registry").stat().st_mode & 0o7777, 0o710) + + def test_free_lock_and_contention_are_distinct(self): + self.skeleton() + fd = os.open(self.root / "etc/dragontools", os.O_RDONLY | os.O_DIRECTORY) + try: + fcntl.flock(fd, fcntl.LOCK_EX | fcntl.LOCK_NB) + self.call(code=96, output=b"", stage="operation_lock", reason="OperationBusy") + self.assertEqual(list((self.base / "pki").iterdir()), []) + finally: + os.close(fd) + self.converged() + + def test_wrong_owner_is_not_empty_bootstrap(self): + self.skeleton() + os.chown(self.base / "pki", INGEST, INGEST) + self.call(code=86, output=b"", stage="pki_directory", reason="InvalidManagedState") + self.assertEqual((self.base / "pki").stat().st_uid, INGEST) + + def test_symlink_is_not_empty_bootstrap(self): + self.skeleton() + (self.base / "pki").rmdir() + (self.base / "pki").symlink_to("clients") + self.call(code=86, output=b"", stage="pki_directory", reason="UnexpectedSymlink") + self.assertTrue((self.base / "pki").is_symlink()) + self.assertEqual(list((self.base / "clients").iterdir()), []) + + def test_invalid_existing_ca_preserved_and_redacted(self): + self.skeleton() + self.converged() + certificate = self.base / "pki/ca/ca.crt" + certificate.write_bytes((self.base / "pki/ca/ca.key").read_bytes()) + before = self.snapshot() + self.call(code=86, output=b"", stage="ca_certificate_validation", + reason="CertificateValidationFailed") + self.assertEqual(before, self.snapshot()) + + def test_configured_hostname_is_verified(self): + self.skeleton() + self.converged() + before = self.snapshot() + self.call("station-verify", hostname="other.example", code=86, output=b"", + stage="server_certificate_validation", reason="CertificateValidationFailed") + self.assertEqual(before, self.snapshot()) + + +if __name__ == "__main__": + unittest.main(argv=[sys.argv[0]]) From 2341d4e6ac91d7c567aabc21896ba5e2d24e8331 Mon Sep 17 00:00:00 2001 From: Vasyl Osypchuk Date: Sat, 19 Sep 2026 10:20:35 +0300 Subject: [PATCH 7/7] add station hostname --- station.toml | 15 +++++++++++++++ 1 file changed, 15 insertions(+) create mode 100644 station.toml diff --git a/station.toml b/station.toml new file mode 100644 index 0000000..71fe15c --- /dev/null +++ b/station.toml @@ -0,0 +1,15 @@ +version = 1 + +[connection] +ssh_host = "monitoring" + +[station] +hostname = "monitoring.baptizeddragon.com" + +[grafana] +username = { op = "op://BaptizedDragon/Grafana/username" } +password = { op = "op://BaptizedDragon/Grafana/password" } + +[telegram] +bot_token = { op = "op://BaptizedDragon/DragonTools/alarms-telegram-bot-token" } +chat_id = { op = "op://BaptizedDragon/DragonTools/alarms-telegram-chat-id" }