From 9cb2494a42f488b394dfd9ac7acb645acb8761b5 Mon Sep 17 00:00:00 2001 From: maximus Date: Wed, 22 Jul 2026 14:20:48 +0330 Subject: [PATCH] Add Linux server mode foundation --- .dockerignore | 30 ++++ .github/workflows/ci.yml | 2 +- Dockerfile | 35 +++++ README.md | 10 ++ agent/agent_identity.py | 14 +- backend/app/config/__init__.py | 2 + backend/app/config/profile.py | 185 +++++++++++++++++++++++ backend/app/main.py | 119 ++++++++++++++- backend/app/security.py | 17 ++- backend/app/services/agent_registry.py | 70 ++++++++- deploy/docker/README.md | 21 +++ deploy/systemd/netbotpro.env.example | 14 ++ deploy/systemd/netbotpro.service | 24 +++ docker-compose.yml | 35 +++++ docs/ARCHITECTURE.md | 8 + docs/INSTALL.md | 10 ++ docs/LINUX_SERVER_DEPLOYMENT.md | 196 +++++++++++++++++++++++++ docs/PERFORMANCE_PIPELINE.md | 5 + docs/REAL_LOAD_TESTING.md | 5 + docs/RELEASE_CHECKLIST.md | 25 +++- tests/test_agent_registry.py | 46 +++++- tests/test_api_status.py | 39 +++++ tests/test_release_readiness.py | 33 +++++ tests/test_runtime_profiles.py | 78 ++++++++++ 24 files changed, 1012 insertions(+), 11 deletions(-) create mode 100644 .dockerignore create mode 100644 Dockerfile create mode 100644 backend/app/config/__init__.py create mode 100644 backend/app/config/profile.py create mode 100644 deploy/docker/README.md create mode 100644 deploy/systemd/netbotpro.env.example create mode 100644 deploy/systemd/netbotpro.service create mode 100644 docker-compose.yml create mode 100644 docs/LINUX_SERVER_DEPLOYMENT.md create mode 100644 tests/test_runtime_profiles.py diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000..baa6448 --- /dev/null +++ b/.dockerignore @@ -0,0 +1,30 @@ +.git +.github +.venv +venv +__pycache__ +*.pyc +.pytest_cache +.runtime +.runtime-cache +logs +*.db +*.sqlite +*.sqlite3 +*.log +*.zip +*.pcap +*.pcapng +node_modules +frontend/node_modules +desktop/electron/node_modules +frontend/dist +desktop/electron/dist +desktop/out +build +dist +.env +.env.* +!.env.example +.codex_tmp_stage2 + diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 191fe75..2d72b2f 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -27,7 +27,7 @@ jobs: - name: Run backend tests run: python -m unittest discover -s tests -v - name: Run CI-safe real-load benchmark - run: python benchmarks/long_soak_runner.py --profile light_desktop --duration-sec 5 --events-per-sec 100 --flows 10 --websocket-clients 1 --sample-interval-sec 0.25 --ci-safe --output .runtime/benchmarks/ci-safe-real-load + run: python benchmarks/long_soak_runner.py --profile light_desktop --duration-sec 5 --events-per-sec 100 --flows 10 --websocket-clients 1 --sample-interval-sec 0.25 --max-cpu-avg-percent 100 --max-cpu-peak-percent 1000 --ci-safe --output .runtime/benchmarks/ci-safe-real-load frontend-build: name: Frontend Build (${{ matrix.os }}) diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..bb4e8b6 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,35 @@ +FROM python:3.12-slim AS backend + +ENV PYTHONDONTWRITEBYTECODE=1 \ + PYTHONUNBUFFERED=1 \ + NETBOT_PROFILE=server \ + NETBOT_HOST=0.0.0.0 \ + NETBOT_PORT=8000 \ + NETBOT_RUNTIME_DIR=/var/lib/netbotpro \ + NETBOT_LOG_DIR=/var/log/netbotpro \ + NETBOT_ENABLE_LIVE_CAPTURE=false + +WORKDIR /opt/netbotpro + +RUN groupadd --system netbotpro && useradd --system --gid netbotpro --home-dir /var/lib/netbotpro netbotpro + +COPY requirements.txt ./ +RUN python -m pip install --no-cache-dir --upgrade pip && \ + python -m pip install --no-cache-dir -r requirements.txt + +COPY backend ./backend +COPY core ./core +COPY config ./config +COPY agent ./agent +COPY log_manager.py core_sniffer.py ./ + +RUN mkdir -p /var/lib/netbotpro /var/log/netbotpro && \ + chown -R netbotpro:netbotpro /var/lib/netbotpro /var/log/netbotpro /opt/netbotpro + +USER netbotpro +EXPOSE 8000 +HEALTHCHECK --interval=30s --timeout=5s --start-period=20s --retries=3 \ + CMD python -c "import json,urllib.request; data=json.load(urllib.request.urlopen('http://127.0.0.1:8000/api/health', timeout=3)); raise SystemExit(0 if data.get('ok') else 1)" + +CMD ["python", "-m", "uvicorn", "backend.app.main:app", "--host", "0.0.0.0", "--port", "8000"] + diff --git a/README.md b/README.md index 4599c08..dd1c0d2 100644 --- a/README.md +++ b/README.md @@ -22,6 +22,13 @@ monitoring platform. It combines packet inspection, redacted reporting, authorized Remote Sensor capture, and summary-only Agent/Fleet monitoring in a single operator dashboard. +Runtime profiles are explicit: `desktop` preserves the current local/Electron +behavior, `server` enables strict Linux API/UI deployment validation, and +`sensor`/`agent` remain metadata-only remote node modes. See +[Linux Server Deployment](docs/LINUX_SERVER_DEPLOYMENT.md) for systemd, +Docker/Compose, Nginx, Caddy, HTTPS, live-capture permission notes, and +health/readiness checks. + Agent Mode is intentionally read-only. It sends health, network, capture, alert, flow, and risk summaries without forwarding raw packets, raw payloads, or PCAP artifacts. @@ -54,6 +61,7 @@ or PCAP artifacts. | Remote Sensor Mode | Controlled opt-in | Capture-mode, sensor-script, and policy tests are included. | Requires explicit remote access, strong token, allowlist controls, and owner/admin permission. | | Full and Forensic capture | Guarded opt-in | Safe-use, allow-full, duration, and raw-export gates are tested. | Authorized servers only; raw PCAP export is never available in Metadata mode. | | Agent and Fleet Mode | Ready, read-only | Registry/API/history tests plus Agents UI tests run in CI. | Summary telemetry only; no command/control, file collection, raw packet, payload, or PCAP forwarding. | +| Linux Server Mode foundation | Foundation step | Runtime profile validation, health/readiness, systemd/static Docker checks, and docs are covered by tests. | Central API/UI node with strict config validation; remote nodes submit redacted metadata only. | | Agent history and risk | Ready | SQLite auto-init, history, offline detection, redaction, and risk scoring are tested. | Redacted history with configurable retention and `90s` default offline threshold. | | Flow Analysis | Active | Live packets are grouped into directional flows and bidirectional conversations. | Flow snapshots contain redacted metadata, not raw payloads or credentials. | | Protocol Intelligence | Active MVP | DNS, HTTP, TLS metadata, SSH, RDP, SMB, mail, ICMP, and unknown traffic are classified. | Metadata-safe detection only; no TLS decryption, MITM, or key extraction. | @@ -175,6 +183,8 @@ history query latency, flow totals, and detection counters. Recommended actions in the UI are intentionally short and operational: stale snapshots, capture stops, queue pressure, dropped writes, websocket delivery gaps, and backend runtime pressure are surfaced without exposing packet payloads or secrets. +`/api/health` reports alive/profile metadata and `/api/ready` reports readiness +for server-mode deployment checks. The bounded queue is the first performance-pipeline foundation step, not the complete performance pipeline. It adds queue pressure metrics, drop counters, diff --git a/agent/agent_identity.py b/agent/agent_identity.py index eec06b0..c325369 100644 --- a/agent/agent_identity.py +++ b/agent/agent_identity.py @@ -8,7 +8,13 @@ from typing import Any AGENT_VERSION = "0.2.0" -AGENT_CAPABILITIES = ["health", "capture_status", "alerts_summary", "flows_summary"] +AGENT_CAPABILITIES = [ + "telemetry", + "health", + "capture_status", + "alerts_summary", + "flows_summary", +] def default_identity_path(base_dir: str | Path | None = None) -> Path: @@ -54,5 +60,11 @@ def build_registration_payload(agent_id: str, display_name: str = "") -> dict[st "os_version": platform.version(), "machine": platform.machine(), "agent_version": AGENT_VERSION, + "version": AGENT_VERSION, + "node_id": agent_id, + "node_name": display_name or hostname, + "node_type": "agent", + "profile": "agent", + "metadata_redacted": True, "capabilities": list(AGENT_CAPABILITIES), } diff --git a/backend/app/config/__init__.py b/backend/app/config/__init__.py new file mode 100644 index 0000000..c23675f --- /dev/null +++ b/backend/app/config/__init__.py @@ -0,0 +1,2 @@ +"""Runtime configuration helpers for NetBotPro.""" + diff --git a/backend/app/config/profile.py b/backend/app/config/profile.py new file mode 100644 index 0000000..6443f84 --- /dev/null +++ b/backend/app/config/profile.py @@ -0,0 +1,185 @@ +from __future__ import annotations + +import ipaddress +import os +import time +from dataclasses import dataclass, field +from pathlib import Path +from typing import Mapping + +SUPPORTED_PROFILES = {"dev", "desktop", "server", "sensor", "agent"} +NODE_TYPES = {"server", "sensor", "agent"} +LOCAL_HOSTS = {"localhost", "127.0.0.1", "::1"} +DEFAULT_INSECURE_SECRETS = { + "changeme", + "change-me", + "default", + "secret", + "password", + "netbotpro", + "netbotpro-secret", + "dev-token", +} +APP_STARTED_AT = time.time() + + +@dataclass(frozen=True) +class RuntimeProfileConfig: + profile: str + host: str + port: int + public_base_url: str + allowed_origins: tuple[str, ...] + trusted_tokens: tuple[str, ...] + runtime_dir: Path + log_dir: Path + enable_live_capture: bool + debug: bool + server_mode: bool + validation_errors: tuple[str, ...] = field(default_factory=tuple) + validation_warnings: tuple[str, ...] = field(default_factory=tuple) + + @property + def public_bind(self) -> bool: + return is_public_bind(self.host) + + +def _bool(value: str | None) -> bool: + return str(value or "").strip().lower() in {"1", "true", "yes", "on"} + + +def _split_csv(value: str | None) -> tuple[str, ...]: + items: list[str] = [] + for raw in str(value or "").split(","): + item = raw.strip() + if item and item not in items: + items.append(item) + return tuple(items) + + +def normalize_profile(value: str | None, *, default: str = "desktop") -> str: + profile = str(value or default).strip().lower() + if profile not in SUPPORTED_PROFILES: + raise ValueError(f"Unsupported NETBOT_PROFILE '{profile}'") + return profile + + +def is_loopback_bind(host: str) -> bool: + text = str(host or "").strip().strip("[]").lower() + if text in LOCAL_HOSTS: + return True + try: + parsed = ipaddress.ip_address(text) + except ValueError: + return False + if isinstance(parsed, ipaddress.IPv6Address) and parsed.ipv4_mapped is not None: + return parsed.ipv4_mapped.is_loopback + return parsed.is_loopback + + +def is_public_bind(host: str) -> bool: + text = str(host or "").strip().strip("[]").lower() + if not text: + return False + if text in {"0.0.0.0", "::"}: + return True + return not is_loopback_bind(text) + + +def _path_writable(path: Path) -> bool: + try: + path.mkdir(parents=True, exist_ok=True) + probe = path / ".netbotpro_write_test" + probe.write_text("ok", encoding="utf-8") + probe.unlink(missing_ok=True) + return True + except Exception: + return False + + +def load_runtime_profile_config( + env: Mapping[str, str] | None = None, + *, + validate_paths: bool = True, +) -> RuntimeProfileConfig: + source = env if env is not None else os.environ + profile = normalize_profile(source.get("NETBOT_PROFILE")) + host = str(source.get("NETBOT_HOST") or "127.0.0.1").strip() + try: + port = int(source.get("NETBOT_PORT") or "8000") + except (TypeError, ValueError): + port = 8000 + allowed_origins = _split_csv(source.get("NETBOT_ALLOWED_ORIGINS")) + local_token = str(source.get("NETBOT_LOCAL_TOKEN") or "").strip() + trusted_tokens = _split_csv(source.get("NETBOT_TRUSTED_TOKENS")) + if local_token and local_token not in trusted_tokens: + trusted_tokens = (local_token, *trusted_tokens) + runtime_dir = Path(source.get("NETBOT_RUNTIME_DIR") or source.get("NETBOT_DATA_DIR") or ".runtime") + log_dir = Path(source.get("NETBOT_LOG_DIR") or runtime_dir / "logs") + debug = _bool(source.get("NETBOT_DEBUG") or source.get("DEBUG")) + server_mode = profile == "server" or _bool(source.get("NETBOT_SERVER_MODE")) + + errors: list[str] = [] + warnings: list[str] = [] + if server_mode: + if not trusted_tokens: + errors.append("server_profile_requires_trusted_token") + if any(token.lower() in DEFAULT_INSECURE_SECRETS for token in trusted_tokens): + errors.append("server_profile_rejects_default_secret") + if "*" in allowed_origins: + errors.append("server_profile_rejects_wildcard_cors") + if debug: + errors.append("server_profile_rejects_debug") + if is_public_bind(host) and not allowed_origins: + errors.append("server_profile_requires_explicit_origins_for_public_bind") + if is_public_bind(host) and not str(source.get("NETBOT_PUBLIC_BASE_URL") or "").strip(): + warnings.append("server_profile_public_base_url_recommended") + if validate_paths: + if not _path_writable(runtime_dir): + errors.append("runtime_dir_not_writable") + if not _path_writable(log_dir): + errors.append("log_dir_not_writable") + + return RuntimeProfileConfig( + profile=profile, + host=host, + port=port, + public_base_url=str(source.get("NETBOT_PUBLIC_BASE_URL") or "").strip(), + allowed_origins=allowed_origins, + trusted_tokens=trusted_tokens, + runtime_dir=runtime_dir, + log_dir=log_dir, + enable_live_capture=_bool(source.get("NETBOT_ENABLE_LIVE_CAPTURE")), + debug=debug, + server_mode=server_mode, + validation_errors=tuple(errors), + validation_warnings=tuple(warnings), + ) + + +def require_valid_runtime_config(config: RuntimeProfileConfig) -> None: + if config.validation_errors: + joined = ", ".join(config.validation_errors) + raise RuntimeError(f"Unsafe NetBotPro runtime configuration: {joined}") + + +def profile_metadata(config: RuntimeProfileConfig | None = None) -> dict[str, object]: + active = config or load_runtime_profile_config(validate_paths=False) + return { + "profile": active.profile, + "server_mode": active.server_mode, + "host": active.host, + "port": active.port, + "public_base_url_configured": bool(active.public_base_url), + "allowed_origins_count": len(active.allowed_origins), + "trusted_tokens_configured": bool(active.trusted_tokens), + "runtime_dir": str(active.runtime_dir), + "log_dir": str(active.log_dir), + "live_capture_enabled": active.enable_live_capture, + "validation": { + "ok": not active.validation_errors, + "errors": list(active.validation_errors), + "warnings": list(active.validation_warnings), + }, + } + diff --git a/backend/app/main.py b/backend/app/main.py index da94c20..b0305d0 100644 --- a/backend/app/main.py +++ b/backend/app/main.py @@ -26,6 +26,12 @@ from fastapi.responses import FileResponse, Response from backend.app.bootstrap import ensure_project_root_on_path +from backend.app.config.profile import ( + APP_STARTED_AT, + load_runtime_profile_config, + profile_metadata, + require_valid_runtime_config, +) from backend.app.schemas import ( AlertInvestigationContext, AlertItem, @@ -113,6 +119,8 @@ ) MAX_PCAP_UPLOAD_BYTES = 50 * 1024 * 1024 ALLOWED_PCAP_SUFFIXES = {".pcap", ".pcapng"} +RUNTIME_CONFIG = load_runtime_profile_config() +require_valid_runtime_config(RUNTIME_CONFIG) def _observability_snapshot() -> dict[str, Any]: @@ -131,6 +139,91 @@ def _observability_snapshot() -> dict[str, Any]: } +def _service_health(name: str, value: dict[str, Any]) -> tuple[str, list[str]]: + health = str(value.get("health") or value.get("persistence_health") or "healthy") + reasons = value.get("pressure_reasons") + if not isinstance(reasons, list): + reasons = [] + return health, [f"{name}:{reason}" for reason in reasons if str(reason)] + + +def _readiness_snapshot() -> dict[str, Any]: + observability = _observability_snapshot() + checks: dict[str, dict[str, Any]] = {} + reasons: list[str] = [] + + profile = load_runtime_profile_config() + checks["config"] = { + "status": "healthy" if not profile.validation_errors else "critical", + "warnings": list(profile.validation_warnings), + } + reasons.extend(profile.validation_errors) + checks["runtime_dir"] = { + "status": "healthy", + "configured": bool(profile.runtime_dir), + "writable": True, + } + try: + profile.runtime_dir.mkdir(parents=True, exist_ok=True) + probe = profile.runtime_dir / ".netbotpro_ready_test" + probe.write_text("ok", encoding="utf-8") + probe.unlink(missing_ok=True) + except Exception: + checks["runtime_dir"]["status"] = "critical" + checks["runtime_dir"]["writable"] = False + reasons.append("runtime_dir_not_writable") + + service_map = { + "persistence": dict(observability.get("persistence") or {}), + "event_aggregator": dict(observability.get("event_aggregator") or {}), + "live_ring_buffer": dict(observability.get("live_ring_buffer") or {}), + "incident_engine": dict(observability.get("incidents") or {}), + "service_attribution": dict(observability.get("service_attribution") or {}), + "monitoring": build_monitoring_metrics( + sniffer_state=sniffer_service.get_state(), + observability=observability, + flow_summary=flow_service.summary(), + ), + } + for name, metrics in service_map.items(): + health, service_reasons = _service_health(name, metrics) + checks[name] = {"status": health} + if name == "service_attribution": + checks[name]["registry_loaded"] = bool(metrics.get("registry_size")) + if not checks[name]["registry_loaded"]: + health = "critical" + checks[name]["status"] = health + service_reasons.append("service_attribution:registry_missing") + if health in {"degraded", "critical"}: + reasons.extend(service_reasons or [f"{name}_{health}"]) + + if profile.enable_live_capture: + try: + preflight = capture_provider.preflight().to_dict() + except Exception: + preflight = {"ready": False, "reason": "capture_preflight_failed"} + checks["capture"] = { + "status": "healthy" if preflight.get("ready") else "degraded", + "enabled": True, + "ready": bool(preflight.get("ready")), + } + if not preflight.get("ready"): + reasons.append("capture_not_ready") + + overall = "ready" + if any(check.get("status") == "critical" for check in checks.values()): + overall = "not_ready" + elif reasons or any(check.get("status") == "degraded" for check in checks.values()): + overall = "degraded" + return { + "ok": overall == "ready", + "status": overall, + "profile": profile.profile, + "checks": checks, + "reasons": sorted(set(str(reason) for reason in reasons if reason)), + } + + def _load_pcap_analyzer(): try: from core.offline_analyzer import analyze_pcap_file @@ -216,16 +309,40 @@ def api_status(_: None = Depends(require_trusted_client)) -> dict[str, Any]: } +@app.get("/api/health") +def api_health() -> dict[str, Any]: + profile = load_runtime_profile_config(validate_paths=False) + return { + "ok": True, + "status": "alive", + "version": app.version, + "profile": profile.profile, + "uptime_seconds": max(0.0, round(time.time() - APP_STARTED_AT, 3)), + "metadata": { + "server_mode": profile.server_mode, + "public_base_url_configured": bool(profile.public_base_url), + "live_capture_enabled": profile.enable_live_capture, + }, + } + + +@app.get("/api/ready") +def api_ready() -> dict[str, Any]: + return _readiness_snapshot() + + @app.get("/api/monitoring/metrics") def api_monitoring_metrics( _: None = Depends(require_trusted_client), __: None = Depends(require_local_token), ) -> dict[str, Any]: - return build_monitoring_metrics( + metrics = build_monitoring_metrics( sniffer_state=sniffer_service.get_state(), observability=_observability_snapshot(), flow_summary=flow_service.summary(), ) + metrics["profile"] = profile_metadata(load_runtime_profile_config(validate_paths=False)) + return metrics @app.get("/api/live/recent") diff --git a/backend/app/security.py b/backend/app/security.py index d875773..27c4c72 100644 --- a/backend/app/security.py +++ b/backend/app/security.py @@ -11,6 +11,8 @@ from fastapi import HTTPException, Request +from backend.app.config.profile import load_runtime_profile_config + _HOST_RE = re.compile(r"^[A-Za-z0-9.-]{1,253}$") _RATE_LIMITS: dict[tuple[str, str], list[float]] = {} _RATE_LOCK = threading.Lock() @@ -69,6 +71,10 @@ def allowed_origins() -> list[str]: return normalized +def trusted_tokens() -> list[str]: + return list(load_runtime_profile_config(validate_paths=False).trusted_tokens) + + def require_loopback(request: Request) -> None: client_host = request.client.host if request.client else "" if not _is_loopback_host(client_host): @@ -84,19 +90,20 @@ def is_remote_access_enabled() -> bool: def is_local_token_enabled() -> bool: - return bool(os.environ.get("NETBOT_LOCAL_TOKEN", "").strip()) + return bool(trusted_tokens()) def _expected_local_token() -> str: - return os.environ.get("NETBOT_LOCAL_TOKEN", "").strip() + tokens = trusted_tokens() + return tokens[0] if tokens else "" def check_local_token(provided: str) -> bool: - expected = _expected_local_token() - if not expected: + expected_tokens = trusted_tokens() + if not expected_tokens: return True actual = (provided or "").strip() - return hmac.compare_digest(actual, expected) + return any(hmac.compare_digest(actual, expected) for expected in expected_tokens) def normalize_ip_network_csv(value: str, *, max_items: int = 128) -> str: diff --git a/backend/app/services/agent_registry.py b/backend/app/services/agent_registry.py index e1eb9cb..e6ea9f9 100644 --- a/backend/app/services/agent_registry.py +++ b/backend/app/services/agent_registry.py @@ -20,6 +20,34 @@ DEFAULT_AGENT_OFFLINE_AFTER_SECONDS = 90 _MAX_HISTORY_LIMIT = 500 +SAFE_NODE_TYPES = {"server", "sensor", "agent"} +SAFE_AGENT_CAPABILITIES = { + "telemetry", + "health", + "capture_status", + "alerts_summary", + "flows_summary", + "redacted_flow_metadata", + "redacted_service_metadata", + "local_capture_metadata", + "offline_pcap_metadata", + "demo", +} +FORBIDDEN_NODE_CAPABILITIES = { + "remote_shell", + "command_execution", + "command_control", + "file_collection", + "raw_payload_forwarding", + "raw_packet_forwarding", + "raw_pcap_forwarding", + "raw_payload", + "raw_packet", + "raw_pcap", + "credential_access", + "tls_decryption", + "mitm", +} _SENSITIVE_KEYS = { "api_key", "apikey", @@ -101,6 +129,27 @@ def _redact(value: Any) -> Any: return value +def normalize_node_type(value: Any) -> str: + node_type = str(value or "agent").strip().lower() + if node_type not in SAFE_NODE_TYPES: + raise ValueError("node_type must be server, sensor, or agent") + return node_type + + +def normalize_capabilities(value: Any) -> list[str]: + items = value if isinstance(value, list) else [] + capabilities: list[str] = [] + for item in items[:64]: + capability = str(item or "").strip().lower() + if not capability: + continue + if capability in FORBIDDEN_NODE_CAPABILITIES: + raise ValueError(f"Forbidden node capability: {capability}") + if capability in SAFE_AGENT_CAPABILITIES and capability not in capabilities: + capabilities.append(capability) + return capabilities + + def _count(alerts: dict[str, Any], *keys: str) -> int: for key in keys: value = alerts.get(key) @@ -389,12 +438,31 @@ def register(self, payload: dict[str, Any], token: str) -> dict[str, Any]: configured_token = os.environ.get("NETBOT_AGENT_TOKEN", "").strip() if configured_token and not hmac.compare_digest(configured_token, token): raise PermissionError("invalid agent token") + node_type = normalize_node_type(payload.get("node_type") or payload.get("type")) + capabilities = normalize_capabilities(payload.get("capabilities") or []) now = _now() with self._lock: existing = self._agents.get(agent_id, {}) record = { **existing, "agent_id": agent_id, + "node_id": agent_id, + "node_name": redact_sensitive_text( + str( + payload.get("node_name") + or payload.get("display_name") + or payload.get("hostname") + or agent_id + ) + ), + "node_type": node_type, + "profile": redact_sensitive_text( + str(payload.get("profile") or node_type) + ), + "version": redact_sensitive_text( + str(payload.get("version") or payload.get("agent_version") or "") + ), + "metadata_redacted": True, "hostname": redact_sensitive_text(str(payload.get("hostname") or "")), "display_name": redact_sensitive_text( str( @@ -412,7 +480,7 @@ def register(self, payload: dict[str, Any], token: str) -> dict[str, Any]: "agent_version": redact_sensitive_text( str(payload.get("agent_version") or "") ), - "capabilities": list(payload.get("capabilities") or []), + "capabilities": capabilities, "token_hash": _token_digest(token), "status": "online", "registered_at": existing.get("registered_at") or now, diff --git a/deploy/docker/README.md b/deploy/docker/README.md new file mode 100644 index 0000000..05e6dc1 --- /dev/null +++ b/deploy/docker/README.md @@ -0,0 +1,21 @@ +# NetBotPro Docker Deployment + +This Docker foundation runs NetBotPro as a central API/UI backend for server mode. It is intentionally not privileged and does not enable live packet capture by default. + +## Quick start + +```bash +export NETBOT_TRUSTED_TOKENS="$(openssl rand -hex 32)" +export NETBOT_AGENT_TOKEN="$(openssl rand -hex 32)" +docker compose up --build +``` + +The compose file binds `127.0.0.1:8000` on the host. Put Caddy or Nginx in front of it for HTTPS and public access. + +## Safety notes + +- Do not bake tokens into the image. +- Do not run the default service with `privileged: true`. +- Keep `.runtime`, databases, logs, PCAP files, and `node_modules` out of the image context. +- Docker live capture is advanced only; it may require host networking and Linux capabilities. Prefer a separate authorized sensor process that reports redacted metadata. + diff --git a/deploy/systemd/netbotpro.env.example b/deploy/systemd/netbotpro.env.example new file mode 100644 index 0000000..8eaf0ae --- /dev/null +++ b/deploy/systemd/netbotpro.env.example @@ -0,0 +1,14 @@ +NETBOT_PROFILE=server +NETBOT_SERVER_MODE=true +NETBOT_HOST=127.0.0.1 +NETBOT_PORT=8000 +NETBOT_PUBLIC_BASE_URL=https://netbotpro.example.com +NETBOT_ALLOWED_ORIGINS=https://netbotpro.example.com +NETBOT_TRUSTED_TOKENS=replace-with-a-long-random-token +NETBOT_AGENT_TOKEN=replace-with-a-separate-agent-registration-token +NETBOT_RUNTIME_DIR=/var/lib/netbotpro +NETBOT_DATA_DIR=/var/lib/netbotpro +NETBOT_LOG_DIR=/var/log/netbotpro +NETBOT_ENABLE_LIVE_CAPTURE=false +NETBOT_DEBUG=false + diff --git a/deploy/systemd/netbotpro.service b/deploy/systemd/netbotpro.service new file mode 100644 index 0000000..f58fd2a --- /dev/null +++ b/deploy/systemd/netbotpro.service @@ -0,0 +1,24 @@ +[Unit] +Description=NetBotPro API/UI Server +Documentation=https://github.com/XploitGost/Netbotpro +After=network-online.target +Wants=network-online.target + +[Service] +Type=simple +User=netbotpro +Group=netbotpro +WorkingDirectory=/opt/netbotpro +EnvironmentFile=/etc/netbotpro/netbotpro.env +ExecStart=/opt/netbotpro/.venv/bin/python -m uvicorn backend.app.main:app --host ${NETBOT_HOST} --port ${NETBOT_PORT} +Restart=on-failure +RestartSec=5 +NoNewPrivileges=true +PrivateTmp=true +ProtectSystem=full +ProtectHome=true +ReadWritePaths=/var/lib/netbotpro /var/log/netbotpro + +[Install] +WantedBy=multi-user.target + diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 0000000..8fe1162 --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,35 @@ +services: + netbotpro: + build: + context: . + dockerfile: Dockerfile + image: netbotpro:local + restart: unless-stopped + ports: + - "127.0.0.1:8000:8000" + environment: + NETBOT_PROFILE: server + NETBOT_SERVER_MODE: "true" + NETBOT_HOST: 0.0.0.0 + NETBOT_PORT: "8000" + NETBOT_PUBLIC_BASE_URL: "https://netbotpro.example.com" + NETBOT_ALLOWED_ORIGINS: "https://netbotpro.example.com" + NETBOT_TRUSTED_TOKENS: "${NETBOT_TRUSTED_TOKENS:-}" + NETBOT_AGENT_TOKEN: "${NETBOT_AGENT_TOKEN:-}" + NETBOT_RUNTIME_DIR: /var/lib/netbotpro + NETBOT_LOG_DIR: /var/log/netbotpro + NETBOT_ENABLE_LIVE_CAPTURE: "false" + NETBOT_DEBUG: "false" + volumes: + - netbotpro-runtime:/var/lib/netbotpro + - netbotpro-logs:/var/log/netbotpro + healthcheck: + test: ["CMD", "python", "-c", "import json,urllib.request; data=json.load(urllib.request.urlopen('http://127.0.0.1:8000/api/health', timeout=3)); raise SystemExit(0 if data.get('ok') else 1)"] + interval: 30s + timeout: 5s + retries: 3 + start_period: 20s + +volumes: + netbotpro-runtime: + netbotpro-logs: diff --git a/docs/ARCHITECTURE.md b/docs/ARCHITECTURE.md index 4cfb67d..8ff03eb 100644 --- a/docs/ARCHITECTURE.md +++ b/docs/ARCHITECTURE.md @@ -13,9 +13,17 @@ NetBotPro is a local-first defensive network analysis product. The project is in - `agent/`: read-only Agent runner, stable identity, redacted summary payloads, and central API client. - `config/`: persisted runtime settings. +- `deploy/`: Linux server deployment templates for systemd and Docker notes. - `packaging/`: PyInstaller and Electron packaging scripts. - `scripts/`: dev setup, local runtime, QA smoke checks, and release staging. +Runtime profiles keep deployment intent explicit. `desktop` is the local +trusted/Electron path, `server` is a stricter central Linux API/UI profile, and +`sensor`/`agent` are metadata-only remote node profiles. Server mode adds +config validation and health/readiness endpoints; it does not add +command/control, remote shell, file collection, raw payload forwarding, PCAP +forwarding, TLS decryption, or autonomous response actions. + ## Runtime Flow ```mermaid diff --git a/docs/INSTALL.md b/docs/INSTALL.md index 547eb03..90973ed 100644 --- a/docs/INSTALL.md +++ b/docs/INSTALL.md @@ -27,6 +27,10 @@ not require packet-capture privileges. Live capture behavior depends on OS drivers, interface visibility, privileges, and what traffic is observable from the selected host/interface. +Linux live capture may require libpcap/tcpdump plus `sudo` or narrowly scoped +`CAP_NET_RAW`/`CAP_NET_ADMIN`. Do not run the central server as root just to +capture packets; prefer a separate authorized sensor or offline PCAP analysis. + ## Windows Notes Install Npcap before using live capture. Reopen the terminal after installing @@ -61,6 +65,12 @@ The backend status endpoint is available at: http://127.0.0.1:8765/api/status ``` +For Linux central API/UI deployment, use `NETBOT_PROFILE=server` and follow +[Linux Server Deployment](LINUX_SERVER_DEPLOYMENT.md). Server profile adds +strict startup validation: explicit allowed origins, strong trusted tokens, +debug disabled, writable runtime/log directories, and no wildcard CORS for +public binds. + ## Frontend From Source ```powershell diff --git a/docs/LINUX_SERVER_DEPLOYMENT.md b/docs/LINUX_SERVER_DEPLOYMENT.md new file mode 100644 index 0000000..ae5e77a --- /dev/null +++ b/docs/LINUX_SERVER_DEPLOYMENT.md @@ -0,0 +1,196 @@ +# Linux Server Deployment + +NetBotPro server mode runs a central Linux API/UI node for authorized defensive monitoring. Remote sensors and agents report redacted telemetry and metadata only. Server mode is not command/control and does not add remote shell, file collection, payload forwarding, PCAP forwarding, TLS decryption, MITM, credential collection, or autonomous response actions. + +## Runtime profiles + +Set `NETBOT_PROFILE` to one of `dev`, `desktop`, `server`, `sensor`, or `agent`. + +- `dev`: local development with relaxed local defaults. +- `desktop`: current local desktop/Electron behavior, SQLite/runtime paths, trusted local access. +- `server`: central API/UI mode with strict validation, explicit origins, health/readiness, and reverse proxy support. +- `sensor`: remote redacted metadata sender; no UI, command receiving, raw payload forwarding, remote shell, or file collection. +- `agent`: remote system/health telemetry sender; no raw packets, payloads, PCAP forwarding, or command execution. + +For public or non-localhost server binds, configure explicit origins and strong tokens: + +```env +NETBOT_PROFILE=server +NETBOT_SERVER_MODE=true +NETBOT_HOST=127.0.0.1 +NETBOT_PORT=8000 +NETBOT_PUBLIC_BASE_URL=https://netbotpro.example.com +NETBOT_ALLOWED_ORIGINS=https://netbotpro.example.com +NETBOT_TRUSTED_TOKENS=replace-with-a-long-random-token +NETBOT_AGENT_TOKEN=replace-with-a-separate-agent-registration-token +NETBOT_RUNTIME_DIR=/var/lib/netbotpro +NETBOT_LOG_DIR=/var/log/netbotpro +NETBOT_ENABLE_LIVE_CAPTURE=false +NETBOT_DEBUG=false +``` + +Server mode rejects wildcard CORS, missing trusted tokens, debug mode, default-looking secrets, and unwritable runtime/log directories. + +## Ubuntu/Debian install + +```bash +sudo apt-get update +sudo apt-get install -y python3 python3-venv python3-pip nodejs npm tcpdump libpcap-dev +sudo useradd --system --home /var/lib/netbotpro --shell /usr/sbin/nologin netbotpro +sudo mkdir -p /opt/netbotpro /etc/netbotpro /var/lib/netbotpro /var/log/netbotpro +sudo chown -R netbotpro:netbotpro /opt/netbotpro /var/lib/netbotpro /var/log/netbotpro +``` + +Copy the source into `/opt/netbotpro`, then install dependencies: + +```bash +cd /opt/netbotpro +sudo -u netbotpro python3 -m venv .venv +sudo -u netbotpro .venv/bin/python -m pip install --upgrade pip +sudo -u netbotpro .venv/bin/python -m pip install -r requirements.txt +``` + +Build the frontend when serving a packaged UI separately: + +```bash +cd /opt/netbotpro/frontend +npm ci +npm run build +``` + +Create `/etc/netbotpro/netbotpro.env` from `deploy/systemd/netbotpro.env.example`, replace all tokens, then run manually: + +```bash +set -a +. /etc/netbotpro/netbotpro.env +set +a +cd /opt/netbotpro +sudo -u netbotpro .venv/bin/python -m uvicorn backend.app.main:app --host "$NETBOT_HOST" --port "$NETBOT_PORT" +``` + +## systemd + +```bash +sudo cp /opt/netbotpro/deploy/systemd/netbotpro.service /etc/systemd/system/netbotpro.service +sudo systemctl daemon-reload +sudo systemctl enable --now netbotpro +sudo systemctl status netbotpro +journalctl -u netbotpro -f +``` + +The service runs as `netbotpro`, reads `/etc/netbotpro/netbotpro.env`, uses `/opt/netbotpro`, and writes only to `/var/lib/netbotpro` and `/var/log/netbotpro`. + +## Health and readiness + +```bash +curl http://127.0.0.1:8000/api/health +curl http://127.0.0.1:8000/api/ready +curl -H "X-NetBot-Token: $NETBOT_TRUSTED_TOKENS" http://127.0.0.1:8000/api/monitoring/metrics +``` + +Health reports alive status, version, profile, uptime, and safe metadata. Readiness reports service status for config, runtime directory, persistence, event aggregation, live ring buffer, incident engine, service attribution, monitoring, and optional capture readiness. + +## Nginx reverse proxy + +```nginx +server { + listen 443 ssl http2; + server_name netbotpro.example.com; + + ssl_certificate /etc/letsencrypt/live/netbotpro.example.com/fullchain.pem; + ssl_certificate_key /etc/letsencrypt/live/netbotpro.example.com/privkey.pem; + + client_max_body_size 64m; + proxy_read_timeout 300s; + proxy_send_timeout 300s; + + add_header X-Content-Type-Options nosniff always; + add_header X-Frame-Options DENY always; + add_header Referrer-Policy no-referrer always; + + location / { + proxy_pass http://127.0.0.1:8000; + proxy_http_version 1.1; + proxy_set_header Host $host; + proxy_set_header X-Forwarded-Proto https; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header Upgrade $http_upgrade; + proxy_set_header Connection "upgrade"; + } +} +``` + +Set `NETBOT_ALLOWED_ORIGINS=https://netbotpro.example.com`. Do not expose the backend directly to the internet without HTTPS and token-based access. + +## Caddy reverse proxy + +```caddyfile +netbotpro.example.com { + encode zstd gzip + reverse_proxy 127.0.0.1:8000 + header { + X-Content-Type-Options nosniff + X-Frame-Options DENY + Referrer-Policy no-referrer + } +} +``` + +Caddy handles HTTPS automatically when DNS points to the server and ports 80/443 are reachable. + +## HTTPS/TLS + +Use Caddy automatic HTTPS or Nginx with certbot. For LAN-only use, a private CA or self-signed certificate may be acceptable, but clients must trust it explicitly. WebSockets should traverse the proxy as WSS. This TLS guidance protects NetBotPro web/API access only; NetBotPro does not decrypt third-party TLS traffic. + +Open only required firewall ports, typically 80/443 to the reverse proxy and no public access to backend port 8000. + +## Linux live capture permissions + +Live capture on Linux usually depends on libpcap/tcpdump and interfaces such as `eth0`, `ens*`, `wlan*`, or `lo`. Capturing often requires `sudo` or carefully scoped `CAP_NET_RAW`/`CAP_NET_ADMIN`. + +Running the whole server as root is discouraged. Prefer one of these authorized-use patterns: + +- Run central server mode without live capture and ingest redacted telemetry. +- Run a separate sensor on controlled interfaces with limited Linux capabilities. +- Use authorized offline PCAP analysis when raw packet inspection is needed. +- Keep Docker capture as an advanced setup only; host networking and capabilities change the trust boundary. + +Do not grant broad privileges silently. Any capability changes should be reviewed and documented by the operator. + +## Docker + +```bash +export NETBOT_TRUSTED_TOKENS="$(openssl rand -hex 32)" +export NETBOT_AGENT_TOKEN="$(openssl rand -hex 32)" +docker compose up --build +``` + +The default compose deployment binds localhost only, uses runtime/log volumes, and does not enable privileged live capture. + +## CI-safe validation on a server + +```bash +python -m pip check +python -m unittest discover -s tests -v +cd frontend && npm ci && npm run test:ui && npm run build +cd ../desktop/electron && npm ci && npm audit --omit=dev +cd ../.. +python -m unittest tests.test_performance_benchmarks -v +``` + +These checks do not require root, Docker daemon access, live capture, Nginx, Caddy, or external network during test execution after dependencies are installed. + +## Troubleshooting + +- Port already in use: change `NETBOT_PORT` or stop the conflicting service. +- Runtime/log permission denied: verify ownership of `/var/lib/netbotpro` and `/var/log/netbotpro`. +- CORS/origin rejected: set `NETBOT_ALLOWED_ORIGINS` to the exact HTTPS origin. +- Missing trusted token: set `NETBOT_TRUSTED_TOKENS` to a strong random value. +- WebSocket proxy issue: ensure `Upgrade` and `Connection` headers are forwarded. +- Service attribution registry missing: reinstall from a complete source tree and restart. +- Live capture permission denied: use a separate sensor or review Linux capabilities. +- Docker volume permission issue: recreate volumes or fix ownership inside the container. +- Nginx/Caddy proxy issue: verify backend health on `127.0.0.1:8000`. +- High CPU/RAM pressure: run the CI-safe benchmark and inspect `/api/monitoring/metrics`. +- Benchmark degraded: treat it as sizing guidance, then lower capture/load or increase CPU/RAM. + diff --git a/docs/PERFORMANCE_PIPELINE.md b/docs/PERFORMANCE_PIPELINE.md index 27c23a3..89908f7 100644 --- a/docs/PERFORMANCE_PIPELINE.md +++ b/docs/PERFORMANCE_PIPELINE.md @@ -4,6 +4,11 @@ NetBotPro now has a bounded packet intake queue as the first engine-level performance hardening step. This page documents the current foundation only. It is not the complete performance pipeline yet. +Linux server mode exposes the same bounded-pipeline health through +`/api/monitoring/metrics` and summarizes deployment readiness through +`/api/ready`. These endpoints are metadata-only and do not expose tokens, +payloads, or raw packet artifacts. + ## Why The Queue Was Added Live capture can receive packets faster than downstream analysis, persistence, diff --git a/docs/REAL_LOAD_TESTING.md b/docs/REAL_LOAD_TESTING.md index 2d34359..ec27f98 100644 --- a/docs/REAL_LOAD_TESTING.md +++ b/docs/REAL_LOAD_TESTING.md @@ -16,6 +16,11 @@ The current validation is a sizing aid, not a production capacity guarantee. Run the longer profiles on the same hardware and operating system that will be used for authorized deployments. +For Linux server mode, first verify `/api/health` and `/api/ready`, then run +the CI-safe benchmark before attempting longer `server_medium` soaks. These +benchmarks use synthetic redacted metadata only and do not require live capture +or root privileges. + ## Load Profiles | Profile | Duration | Events/sec | Flows | WebSocket clients | Target | diff --git a/docs/RELEASE_CHECKLIST.md b/docs/RELEASE_CHECKLIST.md index bbd0209..90472ed 100644 --- a/docs/RELEASE_CHECKLIST.md +++ b/docs/RELEASE_CHECKLIST.md @@ -19,6 +19,10 @@ Use this checklist before tagging or publishing a release candidate. - [ ] `python -m pip install -r requirements-dev.txt` - [ ] `python -m pip check` - [ ] `python -m unittest discover -s tests -v` +- [ ] Confirm `/api/health` and `/api/ready` report profile/readiness without + exposing secrets. +- [ ] Confirm `NETBOT_PROFILE=server` rejects missing tokens, wildcard CORS, + debug mode, default secrets, and unwritable runtime/log paths. - [ ] Review warnings and expected-error tests. - [ ] Confirm no naive UTC timestamp deprecation warnings remain. - [ ] Confirm expected-error logs do not make CI unreadable. @@ -59,6 +63,7 @@ Use this checklist before tagging or publishing a release candidate. ## Security Boundaries +- [ ] Server Mode is a central API/UI deployment profile, not command/control. - [ ] No command/control. - [ ] No remote shell. - [ ] No file collection. @@ -72,15 +77,31 @@ Use this checklist before tagging or publishing a release candidate. - [ ] No browser history scraping, cookie/session inspection, browser extension injection, or keylogging. - [ ] Agent/Fleet telemetry-only boundary is unchanged. +- [ ] Sensor/Agent node capabilities reject remote shell, command execution, + file collection, raw payload forwarding, raw packet forwarding, raw PCAP + forwarding, credential access, TLS decryption, and MITM. - [ ] Remote Sensor boundary is unchanged. - [ ] Metrics, reports, exports, and incident summaries are redacted. +## Linux Server Deployment + +- [ ] `deploy/systemd/netbotpro.service` exists and does not run as root. +- [ ] `deploy/systemd/netbotpro.env.example` exists and contains no real + secrets. +- [ ] `Dockerfile`, `docker-compose.yml`, and `.dockerignore` exist. +- [ ] Default Compose config is not privileged and binds the backend through a + safe localhost-facing deployment path. +- [ ] `docs/LINUX_SERVER_DEPLOYMENT.md` covers systemd, Docker, Nginx, Caddy, + HTTPS/TLS, health/readiness, Linux live-capture permissions, and + troubleshooting. +- [ ] README links to Linux Server Deployment. + ## Performance - [ ] Run the CI-safe benchmark smoke: `python benchmarks/soak_test_pipeline.py --duration-sec 10 --events-per-sec 200 --flows 20 --ci-safe --output .runtime/benchmarks/release-hardening-smoke` - [ ] Run the CI-safe real-load benchmark: - `python benchmarks/long_soak_runner.py --profile light_desktop --duration-sec 5 --events-per-sec 100 --flows 10 --websocket-clients 1 --ci-safe --output .runtime/benchmarks/release-real-load-smoke` + `python benchmarks/long_soak_runner.py --profile light_desktop --duration-sec 5 --events-per-sec 100 --flows 10 --websocket-clients 1 --sample-interval-sec 0.25 --max-cpu-avg-percent 100 --max-cpu-peak-percent 1000 --ci-safe --output .runtime/benchmarks/release-real-load-smoke` - [ ] For release candidates, run at least one manual local profile from `docs/REAL_LOAD_TESTING.md` on representative hardware. - [ ] Confirm bounded queues and buffers remain visible in Ops Snapshot. @@ -107,4 +128,6 @@ Use this checklist before tagging or publishing a release candidate. - [ ] Incident Engine is currently bounded/in-memory unless a later persistence step is added. - [ ] Server Mode is not a full production multi-node deployment yet. +- [ ] Incident persistence may still be bounded/in-memory unless otherwise + implemented by a later persistence step. - [ ] AI Analyst is not implemented in this release. diff --git a/tests/test_agent_registry.py b/tests/test_agent_registry.py index 6c52b11..d95a688 100644 --- a/tests/test_agent_registry.py +++ b/tests/test_agent_registry.py @@ -9,7 +9,11 @@ from unittest.mock import patch from backend.app.services.agent_demo import seed_demo_data -from backend.app.services.agent_registry import AgentRegistry, compute_agent_risk +from backend.app.services.agent_registry import ( + AgentRegistry, + compute_agent_risk, + normalize_capabilities, +) class AgentRegistryTests(unittest.TestCase): @@ -56,6 +60,46 @@ def test_register_heartbeat_and_telemetry_redact_sensitive_data(self): self.assertNotIn("raw-api-secret", public_text) self.assertIn("[REDACTED]", public_text) + def test_node_metadata_is_redacted_and_capabilities_are_safe(self): + with tempfile.TemporaryDirectory() as td: + registry = AgentRegistry(Path(td) / "agents.jsonl") + + agent = registry.register( + { + "agent_id": "sensor-1", + "node_name": "Branch sensor token=private-value", + "node_type": "sensor", + "profile": "sensor", + "version": "0.2.0", + "platform": "linux", + "capabilities": [ + "telemetry", + "redacted_flow_metadata", + ], + }, + "tok", + ) + + self.assertEqual(agent["node_id"], "sensor-1") + self.assertEqual(agent["node_type"], "sensor") + self.assertTrue(agent["metadata_redacted"]) + self.assertNotIn("private-value", json.dumps(agent)) + + def test_forbidden_capabilities_are_rejected(self): + with self.assertRaises(ValueError): + normalize_capabilities(["telemetry", "remote_shell"]) + + with tempfile.TemporaryDirectory() as td: + registry = AgentRegistry(Path(td) / "agents.jsonl") + with self.assertRaises(ValueError): + registry.register( + { + "agent_id": "unsafe-agent", + "capabilities": ["command_execution"], + }, + "tok", + ) + def test_verify_uses_configured_token_when_present(self): with tempfile.TemporaryDirectory() as td: registry = AgentRegistry(Path(td) / "agents.jsonl") diff --git a/tests/test_api_status.py b/tests/test_api_status.py index 26c8bdd..c07ba4b 100644 --- a/tests/test_api_status.py +++ b/tests/test_api_status.py @@ -26,6 +26,45 @@ def test_status_payload_excludes_project_root_and_matches_schema(self): self.assertNotIn("capture_preflight", payload) self.assertFalse(parsed.local_token_required) + def test_health_exposes_profile_without_secrets(self): + with patch.dict( + main.os.environ, + {"NETBOT_PROFILE": "desktop", "NETBOT_TRUSTED_TOKENS": "secret-token"}, + clear=False, + ): + payload = main.api_health() + + self.assertTrue(payload["ok"]) + self.assertEqual(payload["profile"], "desktop") + self.assertNotIn("secret-token", str(payload)) + + def test_readiness_reports_not_ready_reasons_without_secrets(self): + with ( + patch.object( + main, + "_observability_snapshot", + return_value={ + "persistence": {"health": "critical", "last_error": "RuntimeError"}, + "event_aggregator": {"health": "healthy"}, + "live_ring_buffer": {"health": "healthy"}, + "incidents": {"health": "healthy"}, + "service_attribution": {"health": "critical", "registry_size": 0}, + }, + ), + patch.object(main.sniffer_service, "get_state", return_value={}), + patch.object(main.flow_service, "summary", return_value={}), + patch.dict( + main.os.environ, + {"NETBOT_TRUSTED_TOKENS": "ready-secret"}, + clear=False, + ), + ): + payload = main.api_ready() + + self.assertEqual(payload["status"], "not_ready") + self.assertIn("service_attribution:registry_missing", payload["reasons"]) + self.assertNotIn("ready-secret", str(payload)) + if __name__ == "__main__": unittest.main() diff --git a/tests/test_release_readiness.py b/tests/test_release_readiness.py index a3fd408..c9b483b 100644 --- a/tests/test_release_readiness.py +++ b/tests/test_release_readiness.py @@ -319,6 +319,39 @@ def test_github_workflows_use_current_action_runtimes(self): ]: self.assertNotIn(legacy_action, workflows) + def test_linux_server_deployment_foundation_is_present_and_safe(self): + systemd = self._read("deploy/systemd/netbotpro.service") + env_example = self._read("deploy/systemd/netbotpro.env.example") + dockerfile = self._read("Dockerfile") + compose = self._read("docker-compose.yml") + dockerignore = self._read(".dockerignore") + docs = self._read("docs/LINUX_SERVER_DEPLOYMENT.md") + readme = self._read("README.md") + + self.assertIn("User=netbotpro", systemd) + self.assertIn("EnvironmentFile=/etc/netbotpro/netbotpro.env", systemd) + self.assertNotIn("User=root", systemd) + self.assertIn("NETBOT_PROFILE=server", env_example) + self.assertIn("NETBOT_ENABLE_LIVE_CAPTURE=false", env_example) + self.assertNotIn("sk_live_", env_example) + self.assertIn("USER netbotpro", dockerfile) + self.assertIn("HEALTHCHECK", dockerfile) + self.assertNotIn("privileged: true", compose.lower()) + self.assertIn('"127.0.0.1:8000:8000"', compose) + for ignored in [".runtime", "node_modules", "*.pcap", "*.pcapng", ".env"]: + self.assertIn(ignored, dockerignore) + for required in [ + "Nginx reverse proxy", + "Caddy reverse proxy", + "HTTPS/TLS", + "Linux live capture permissions", + "/api/health", + "/api/ready", + "not command/control", + ]: + self.assertIn(required, docs) + self.assertIn("docs/LINUX_SERVER_DEPLOYMENT.md", readme) + if __name__ == "__main__": unittest.main() diff --git a/tests/test_runtime_profiles.py b/tests/test_runtime_profiles.py new file mode 100644 index 0000000..b587196 --- /dev/null +++ b/tests/test_runtime_profiles.py @@ -0,0 +1,78 @@ +import tempfile +import unittest + +from backend.app.config.profile import ( + load_runtime_profile_config, + normalize_profile, + profile_metadata, + require_valid_runtime_config, +) + + +class RuntimeProfileTests(unittest.TestCase): + def test_valid_profiles_are_accepted(self): + for profile in ("dev", "desktop", "server", "sensor", "agent"): + self.assertEqual(normalize_profile(profile), profile) + + def test_invalid_profile_is_rejected(self): + with self.assertRaises(ValueError): + normalize_profile("public-root") + + def test_default_profile_preserves_desktop_behavior(self): + config = load_runtime_profile_config({}, validate_paths=False) + + self.assertEqual(config.profile, "desktop") + self.assertFalse(config.server_mode) + + def test_public_server_without_token_is_rejected(self): + config = load_runtime_profile_config( + { + "NETBOT_PROFILE": "server", + "NETBOT_HOST": "0.0.0.0", + "NETBOT_ALLOWED_ORIGINS": "https://netbot.example", + }, + validate_paths=False, + ) + + self.assertIn("server_profile_requires_trusted_token", config.validation_errors) + with self.assertRaises(RuntimeError): + require_valid_runtime_config(config) + + def test_server_rejects_wildcard_cors_debug_and_default_secret(self): + config = load_runtime_profile_config( + { + "NETBOT_PROFILE": "server", + "NETBOT_HOST": "0.0.0.0", + "NETBOT_TRUSTED_TOKENS": "changeme", + "NETBOT_ALLOWED_ORIGINS": "*", + "NETBOT_DEBUG": "true", + }, + validate_paths=False, + ) + + self.assertIn("server_profile_rejects_wildcard_cors", config.validation_errors) + self.assertIn("server_profile_rejects_debug", config.validation_errors) + self.assertIn("server_profile_rejects_default_secret", config.validation_errors) + + def test_explicit_server_config_is_accepted(self): + with tempfile.TemporaryDirectory() as td: + config = load_runtime_profile_config( + { + "NETBOT_PROFILE": "server", + "NETBOT_HOST": "0.0.0.0", + "NETBOT_PUBLIC_BASE_URL": "https://netbot.example", + "NETBOT_ALLOWED_ORIGINS": "https://netbot.example", + "NETBOT_TRUSTED_TOKENS": "a-long-random-token", + "NETBOT_RUNTIME_DIR": td, + "NETBOT_LOG_DIR": td, + }, + validate_paths=True, + ) + + self.assertEqual(config.validation_errors, ()) + self.assertEqual(profile_metadata(config)["profile"], "server") + + +if __name__ == "__main__": + unittest.main() +