External sites: signed in scans, one user agent, a default scan per site

A site can carry a scan sign in: a header such as Authorization: Bearer, a
session cookie copied from a signed in browser, or a form login.  It is saved
encrypted and never shown again.  A form login is signed in once before the
engines start, keeping hidden fields such as a CSRF token, and its session is
handed on.  nuclei, ffuf, dalfox, sqlmap, nikto and wapiti sign in; the
baseline checks and testssl stay signed out.  Anything an engine echoed back
of the secret is taken out of the findings, the engine notes and errors
before they are stored.  A form login or HTTP Basic goes to nextgen-dast in
its own fields; a token or cookie, which its API has no field for, is refused
rather than run signed out.  A signed in scan past baseline needs the
confirmation, and the scan page and the report say whether it signed in.

Every scan, screenshot, telemetry read and login now sends one current
desktop Chrome user agent, which Settings can replace with another browser's.
sqlmap no longer picks a random one and testssl no longer uses its own.

The scan dialog started on baseline, a check of a few seconds that runs no
engines, so a real scan looked like it had finished too early.  It now starts
on quick, says what each profile does and how long it takes, and uses the
site's own default profile, which Add and Edit can set.  Bulk scans can use
each site's default.  A baseline result offers a deeper scan.

A Cloudflare 502 for a site whose origin is down was taken for a firewall
block page and its screenshot hidden.  It is now shown as the site being down
behind its CDN.  Words that also appear on ordinary pages, such as please
enable cookies, only count as a block on a refusal status.

Author: hackrange

app/core/api_reference.py

@@ -459,6 +459,11 @@ def _f(name, kind, required, description):
     _f("external_auth_needed", "boolean", False, "Signing in goes through something outside the site."),
     _f("whitelist_needed", "boolean", False, "The scanner has to be let through a firewall."),
     _f("change_frequency_days", "integer", False, "How often the site changes, in days."),
+    _f("default_profile", "string", False, "The scan profile used when none is given: "
+                                           "baseline, quick, standard, thorough or premium."),
+    _f("scan_auth", "object", False, "How a scan signs in: kind header (header_name, "
+                                     "header_value), cookie (cookie) or form (login_url, "
+                                     "username, password).  null removes it.  Never returned."),
     _f("repositories", "array", False, "Objects with repository (org/name) and an optional "
                                        "branch.  Replaces what was picked before."),
     _f("telemetry_fields", "object", False, "Telemetry filled in by hand, by field name: "
@@ -549,7 +554,11 @@ def _external_endpoints() -> list:
              "and up, and any DAST server, send attack payloads and need confirm.",
              [asset_id],
              body=[_f("profile", "string", False, "baseline, quick, standard, "
-                                                  "thorough or premium."),
+                                                  "thorough or premium.  The site's default "
+                                                  "when left out."),
+                   _f("authenticated", "boolean", False, "Sign in with the site's scan "
+                                                         "sign in.  On by default when it "
+                                                         "has one."),
                    _f("server", "string", False, "builtin, or a nextgen-dast server's name."),
                    _f("confirm", "boolean", False, "True to say you may scan it."),
                    _f("ai", "boolean", False, "True to have the model triage the "

app/core/dast_engines.py

@@ -38,17 +38,19 @@
 log = logging.getLogger("gcr.dast")
 
 PROFILES = {
-    "baseline": "Read only checks: headers, cookies, TLS, exposed files, open "
-                "ports.  A few dozen requests, about a minute.",
-    "quick": "Baseline, then testssl and nuclei for exposures, "
-             "misconfigurations, known CVEs and default logins.",
-    "standard": "Quick, then nikto and wapiti.  Sends attack payloads.  "
-                "Can take an hour or more.",
-    "thorough": "Standard, with ffuf content discovery first and a wider "
-                "nuclei pass.  Sends attack payloads.",
-    "premium": "Thorough, then sqlmap and dalfox.  The most invasive.  Can run "
-               "for hours.",
+    "baseline": "A light, read only check: headers, cookies, TLS, open ports and "
+                "files that should never be served.  About 20 requests, done in "
+                "seconds.  Runs none of the scanning engines.",
+    "quick": "Baseline, then testssl and nuclei for exposures, misconfigurations, "
+             "known CVEs and default logins.  Usually 10 to 30 minutes.",
+    "standard": "Quick, then nikto and wapiti, which crawl the site and try attack "
+                "payloads.  An hour or more.",
+    "thorough": "Standard, with ffuf content discovery first and a wider nuclei "
+                "pass.  Sends attack payloads.  Several hours on a large site.",
+    "premium": "Thorough, then sqlmap and dalfox.  The most invasive.  Can run for "
+               "many hours.",
 }
+DEFAULT_PROFILE = "quick"
 ACTIVE_PROFILES = {"standard", "thorough", "premium"}
 
 PLANS = {
@@ -78,8 +80,11 @@
 TEMPLATES = Path(os.environ.get("GCR_NUCLEI_TEMPLATES",
                                 str(Path.home() / ".cache" / "nuclei-templates")))
 
-UA = ("Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 "
-      "(KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36")
+def _ua() -> str:
+    """The one user agent every engine sends, from Settings."""
+    from app.core.external_probe import user_agent
+
+    return user_agent()
 
 
 def binary(tool: str) -> str:
@@ -176,7 +181,7 @@ def command(tool: str, url: str, host: str, profile: str, workdir: Path,
     if tool == "testssl":
         report = workdir / "testssl.json"
         return ([exe, "--jsonfile", str(report), "--quiet", "--color", "0",
-                 "--warnings", "off", "--sneaky", host], report)
+                 "--warnings", "off", "--user-agent", _ua(), host], report)
     if tool == "nuclei":
         tags = {"quick": "exposure,misconfig,cve,default-login",
                 "thorough": "exposure,misconfig,cve,default-login,token-spray,"
@@ -191,7 +196,7 @@ def command(tool: str, url: str, host: str, profile: str, workdir: Path,
                 "-disable-update-check", "-no-color", "-silent",
                 "-severity", "info,low,medium,high,critical", "-tags", tags,
                 "-rl", "50", "-c", "25", "-timeout", "10", "-retries", "1",
-                "-H", f"User-Agent: {UA}"]
+                "-H", f"User-Agent: {_ua()}"]
         templates = _nuclei_templates(workdir, -1)
         if templates:
             argv += ["-t", templates]
@@ -201,7 +206,7 @@ def command(tool: str, url: str, host: str, profile: str, workdir: Path,
         flags = "x6" if profile == "quick" else ("x6789ab" if profile in ("thorough", "premium") else "x69")
         return ([exe, "-h", url, "-output", str(report), "-Format", "json",
                  "-ask", "no", "-nointeractive", "-Tuning", flags,
-                 "-maxtime", "3000s", "-useragent", UA], report)
+                 "-maxtime", "3000s", "-useragent", _ua()], report)
     if tool == "wapiti":
         report = workdir / "wapiti.json"
         modules = ("sql,timesql,xss,permanentxss,ssrf,xxe,exec,file,csrf,"
@@ -211,7 +216,7 @@ def command(tool: str, url: str, host: str, profile: str, workdir: Path,
         argv = [exe, "-u", url, "-f", "json", "-o", str(report),
                 "--flush-session", "--verbose", "0", "--scope", "subdomain",
                 "-m", modules, "--max-attack-time", "1800",
-                "--max-scan-time", str(TIMEOUTS["wapiti"] - 300), "-A", UA,
+                "--max-scan-time", str(TIMEOUTS["wapiti"] - 300), "-A", _ua(),
                 "--store-session", str(workdir / "wapiti-session"),
                 "--store-config", str(workdir / "wapiti-config")]
         # Routes from the code and paths found by ffuf, as extra places to
@@ -221,7 +226,7 @@ def command(tool: str, url: str, host: str, profile: str, workdir: Path,
         return argv, report
     if tool == "sqlmap":
         out = workdir / "sqlmap"
-        return ([exe, "-u", url, "--batch", "--random-agent", "--crawl", "2",
+        return ([exe, "-u", url, "--batch", "-A", _ua(), "--crawl", "2",
                  "--forms", "--smart", "--level", "2", "--risk", "2",
                  "--output-dir", str(out), "--disable-coloring"], out)
     if tool == "dalfox":
@@ -233,13 +238,13 @@ def command(tool: str, url: str, host: str, profile: str, workdir: Path,
         targets.write_text("\n".join([url] + with_params[:200]) + "\n")
         return ([exe, "scan", str(targets), "-f", "json", "-o", str(report), "-S",
                  "--workers", "10", "--timeout", "10", "--scan-timeout", "1500",
-                 "--skip-mining-dom", "--user-agent", UA], report)
+                 "--skip-mining-dom", "--user-agent", _ua()], report)
     if tool == "ffuf":
         report = workdir / "ffuf.json"
         return ([exe, "-u", url.rstrip("/") + "/FUZZ", "-w", str(WORDLIST),
                  "-of", "json", "-o", str(report), "-mc", "200,301,302,401,403",
                  "-fs", "0", "-ac", "-t", "20", "-rate", "40", "-timeout", "10",
-                 "-maxtime", "240", "-s", "-H", f"User-Agent: {UA}"], report)
+                 "-maxtime", "240", "-s", "-H", f"User-Agent: {_ua()}"], report)
     raise ValueError(f"unknown engine {tool}")
 
 
@@ -270,7 +275,7 @@ def links_in(url: str, body: str) -> list:
 
 
 def run_engine(tool: str, url: str, host: str, profile: str, workdir: Path,
-               scan_id: int, paths: list = None) -> dict:
+               scan_id: int, paths: list = None, auth_args: list = None) -> dict:
     """Run one engine and read its report.
 
     -> {tool, status, seconds, findings, detail, paths}
@@ -289,6 +294,7 @@ def run_engine(tool: str, url: str, host: str, profile: str, workdir: Path,
     engine_dir.mkdir(parents=True, exist_ok=True)
     try:
         argv, report = command(tool, url, host, profile, engine_dir, paths)
+        argv = list(argv) + list(auth_args or [])
     except Exception as exc:  # noqa: BLE001
         out.update(status="failed", detail=f"{type(exc).__name__}: {exc}"[:300])
         return out

app/core/dast_report.py

@@ -282,9 +282,26 @@ def build(db, asset, scan, user=None, organization: str = "") -> dict:
     for e in skipped:
         limitations.append(f"{e.get('name') or e.get('tool')} did not complete: "
                            f"{e.get('detail') or e.get('status')}.")
-    if asset.external_auth_needed:
-        limitations.append("Signing in to this site goes through an outside service, so the "
-                           "scan saw only what is public.")
+    if scan.authenticated:
+        kinds = {"header": "a header", "cookie": "a session cookie", "form": "a form login"}
+        signin = next((e for e in engines if e.get("tool") == "signin"), None)
+        limitations.append(f"The engines signed in with {kinds.get(scan.auth_kind, 'a sign in')}, "
+                           "so they saw what that user sees.  The baseline checks and testssl "
+                           "ran signed out.")
+        if signin and signin.get("status") != "ok":
+            limitations.append(f"Signing in did not work: {signin.get('detail')}.")
+        unsigned = [e.get("name") for e in engines
+                    if e.get("signed_in") is False and e.get("status") == "ok"
+                    and e.get("tool") not in ("baseline", "testssl")]
+        if unsigned:
+            limitations.append("These ran signed out: " + ", ".join(unsigned) + ".")
+    else:
+        limitations.append("The scan did not sign in, so it saw only the public parts of "
+                           "the site.")
+    if asset.external_auth_needed and not scan.authenticated:
+        limitations.append("Signing in to this site goes through an outside service.  A "
+                           "session cookie set in the site's sign in settings lets a scan "
+                           "get past it.")
     if asset.whitelist_needed:
         limitations.append("The site needs the scanner's address let through a firewall.  If "
                            "it was not, parts of the site were blocked from the scan.")