Many external sites can be edited at once

The list's ticked sites can now be changed together with Edit the ticked
sites: application, executive and technical owner, environment, status,
external auth and whitelist needed, change frequency, default scan profile,
no scan, tags, comments, proxy, scan sign in and repositories.  Each field has
a Change box and only the ticked ones are applied, so everything else stays
as each site has it; typing in a field ticks it.  Tags are added, removed or
replaced, comments replaced or added to, repositories added, replaced or
cleared.  Every change is checked for every site with the same rules as Edit
before any is written, and if one site will not take it, nothing changes and
the message names that site.  The API does the same through a PATCH of the
assets list with ids.

Author: hackrange

app/core/api_reference.py

@@ -513,6 +513,22 @@ def _external_endpoints() -> list:
                    _f("no_scan_reason", "string", False, "Why not.")] + _DETAILS,
              body_example={"no_scan": True, "no_scan_reason": "Vendor hosted"},
              example=_ASSET),
+        _ext("ext-bulk", "PATCH", "/external/assets", "Change many sites",
+             "The same changes on every site in ids, all or nothing: if any site will not "
+             "take a change, nothing is changed and the message names it.  Takes the same "
+             "fields as changing one site.  tags_mode is add, remove or replace; "
+             "comments_mode is replace or append; repositories_mode is add, replace or clear.",
+             body=[_f("ids", "array", True, "The sites to change, up to 500.")] + _DETAILS + [
+                 _f("tags", "array", False, "Tags, as tags_mode says."),
+                 _f("tags_mode", "string", False, "add, remove or replace."),
+                 _f("comments_mode", "string", False, "replace or append."),
+                 _f("repositories_mode", "string", False, "add, replace or clear."),
+                 _f("no_scan", "boolean", False, "Never scan them."),
+                 _f("no_scan_reason", "string", False, "Why not.")],
+             body_example={"ids": [12, 14, 15], "environment": "dev",
+                           "technical_owner": "team@example.com",
+                           "tags": ["ecommerce"], "tags_mode": "add", "whitelist_needed": True},
+             example={"changed": 3, "hosts": ["a.example.com", "b.example.com", "c.example.com"]}),
         _ext("ext-delete", "DELETE", "/external/assets/{asset_id}", "Remove a site",
              "Removes the site and its scan history.  409 while a scan runs.",
              [asset_id], example={}),

app/core/external.py

@@ -333,6 +333,93 @@ def set_proxy(asset, value) -> None:
     asset.proxy = store._encrypt(text)
 
 
+BULK_FIELDS = ("application", "executive_owner", "technical_owner", "environment",
+               "status", "external_auth_needed", "whitelist_needed",
+               "change_frequency_days", "default_profile")
+
+
+def bulk_update(db, assets: list, changes: dict, actor: str, user=None) -> list:
+    """Apply the same changes to many sites, all or nothing.
+
+    changes holds only what is to change:
+        details      {field: value} for BULK_FIELDS, checked like Edit
+        tags         (add | remove | replace, [tags])
+        comments     (replace | append, text)
+        no_scan      (bool, reason)
+        proxy        a proxy address, or "" to remove it
+        auth         (kind, {values}) as Edit takes it; kind "" removes it
+        repositories (add | replace | clear, [(repository id, branch)])
+
+    Every site is checked before any is written; the first one that will not
+    take a change raises ValueError naming it, and the caller rolls back.
+    -> the hosts changed
+    """
+    from app.core import external_auth
+
+    envs = environments(db)
+    details = dict(changes.get("details") or {})
+    application = details.pop("application", None)
+    cleaned = clean_details(details, partial=True, envs=envs) if details else {}
+    tags = changes.get("tags")
+    if tags and tags[0] not in ("add", "remove", "replace"):
+        raise ValueError("Tags are added, removed or replaced.")
+    comments = changes.get("comments")
+    if comments and comments[0] not in ("replace", "append"):
+        raise ValueError("Comments are replaced or added to.")
+    repos = changes.get("repositories")
+    if repos and repos[0] not in ("add", "replace", "clear"):
+        raise ValueError("Repositories are added, replaced or cleared.")
+    changed = []
+    for asset in assets:
+        try:
+            if application is not None:
+                asset.application = str(application)[:200].strip()
+            for key, value in cleaned.items():
+                setattr(asset, key, value)
+            if tags:
+                mode, values = tags
+                have = asset.tag_list()
+                wanted = clean_tags(values)
+                if mode == "add":
+                    set_tags(asset, have + wanted)
+                elif mode == "remove":
+                    drop = {t.casefold() for t in wanted}
+                    set_tags(asset, [t for t in have if t.casefold() not in drop])
+                else:
+                    set_tags(asset, wanted)
+            if comments:
+                mode, text = comments
+                text = str(text or "").strip()
+                if mode == "append" and asset.notes and text:
+                    text = (asset.notes.rstrip() + "\n" + text)
+                asset.notes = text[:4000] if (mode == "replace" or text) else asset.notes
+            if changes.get("no_scan") is not None:
+                flag, reason = changes["no_scan"]
+                asset.no_scan = bool(flag)
+                asset.no_scan_reason = (str(reason or "")[:300] if flag else "")
+            if "proxy" in changes:
+                set_proxy(asset, changes["proxy"] or "")
+            if changes.get("auth") is not None:
+                kind, values = changes["auth"]
+                external_auth.save(asset, kind, values, actor, asset.parent_domain)
+            if repos:
+                mode, picks = repos
+                if mode == "add":
+                    current = [tuple(p.split("|", 1)) for p in picks_of(db, asset)]
+                    merged = [(int(r), b) for r, b in current]
+                    merged += [p for p in picks if p not in merged]
+                    set_picks(db, asset, merged, actor, user)
+                elif mode == "replace":
+                    set_picks(db, asset, picks, actor, user)
+                else:
+                    set_picks(db, asset, [], actor, user)
+        except ValueError as exc:
+            raise ValueError(f"{asset.host}: {exc}")
+        changed.append(asset.host)
+    db.flush()
+    return changed
+
+
 def profile_for(asset: ExternalAsset) -> str:
     """The profile a scan of this site starts with when nobody picks one."""
     from app.core import dast_engines

app/web/routes_external.py

@@ -227,6 +227,7 @@ def list_page(request: Request, q: str = "", tag: str = "", app: str = "",
 
         context.update(active_page="external", rows=rows, totals=totals,
                        profiles=dast_engines.PROFILES, default_profile=dast_engines.DEFAULT_PROFILE,
+                       auth_kinds=external_auth_mod.KINDS,
                        q=q, tag=tag, app=app, grade=grade, dast=dast, risk=risk,
                        scope=scope, all_tags=all_tags, apps=apps, opts=opts,
                        env=env, status=status, environments=envs,
@@ -378,6 +379,69 @@ def _ticked(form) -> list:
     return list(dict.fromkeys(ids))[:500]
 
 
+def _bulk_changes(form) -> dict:
+    """What the bulk edit dialog asks to change: only the ticked fields."""
+    def on(name):
+        return _checked(form, f"apply_{name}")
+
+    changes, details = {}, {}
+    for key in external.BULK_FIELDS:
+        if on(key):
+            value = form.get(key)
+            if key in ("external_auth_needed", "whitelist_needed"):
+                value = str(value or "").lower() in ("1", "yes", "true", "on")
+            details[key] = value if value is not None else ""
+    if details:
+        changes["details"] = details
+    if on("tags"):
+        changes["tags"] = (str(form.get("tags_mode") or "add"),
+                           str(form.get("tags") or "").replace(";", ",").split(","))
+    if on("comments"):
+        changes["comments"] = (str(form.get("comments_mode") or "replace"),
+                               clean_text(form.get("comments"), "Comments", limit=4000, required=False))
+    if on("no_scan"):
+        changes["no_scan"] = (str(form.get("no_scan") or "").lower() in ("1", "yes"),
+                              clean_text(form.get("no_scan_reason"), "Reason", limit=300, required=False))
+    if on("proxy"):
+        changes["proxy"] = "" if form.get("proxy_mode") == "clear" else str(form.get("proxy") or "")
+        if changes["proxy"] == "" and form.get("proxy_mode") != "clear":
+            raise ValueError("Give the proxy address, or choose to remove it.")
+    if on("auth"):
+        changes["auth"] = (str(form.get("auth_kind") or ""), {
+            key: form.get(f"auth_{key}") or "" for key in (
+                "header_name", "header_value", "cookie", "login_url", "username",
+                "password", "username_field", "password_field")})
+    if on("repositories"):
+        changes["repositories"] = (str(form.get("repositories_mode") or "add"),
+                                   external.parse_picks(form.getlist("pick")))
+    return changes
+
+
+@router.post("/external/bulk-edit")
+async def bulk_edit(request: Request, user: User = Depends(current_user)):
+    """Change the same things on every ticked site, all or nothing."""
+    await check_csrf(request)
+    form = await request.form()
+    with session_scope() as db:
+        _guard(db, user)
+        ids = _ticked(form)
+        if not ids:
+            return redirect(_back("/external", "Tick the sites to change first.", "error"))
+        try:
+            changes = _bulk_changes(form)
+            if not changes:
+                return redirect(_back("/external", "Tick Change beside what to change.", "error"))
+            assets = db.query(ExternalAsset).filter(ExternalAsset.id.in_(ids)).all()
+            changed = external.bulk_update(db, assets, changes, user.username, user)
+        except ValueError as exc:
+            db.rollback()
+            return redirect(_back("/external", f"Nothing was changed.  {exc}", "error"))
+        record(db, "external.bulk_edit", actor=user.username, target=f"{len(changed)} sites",
+               detail=(", ".join(sorted(changes)) + ": " + ", ".join(changed))[:500],
+               source_ip=client_ip(request))
+    return redirect(_back("/external", f"Changed {len(changed)} site{'s' if len(changed) != 1 else ''}."))
+
+
 @router.post("/external/delete-many")
 async def delete_many(request: Request, user: User = Depends(current_user)):
     """Remove the ticked sites and their scan history.  One with a scan
@@ -1258,6 +1322,73 @@ async def api_patch(asset_id: int, request: Request, caller=Depends(_api_caller(
         return _view(db, asset, me, request)
 
 
+@api.patch("/assets")
+async def api_bulk_patch(request: Request, caller=Depends(_api_caller())):
+    """Change the same things on many sites, all or nothing."""
+    body = await _body(request)
+    ids = body.get("ids")
+    if not isinstance(ids, list) or not ids or not all(isinstance(i, int) for i in ids) or len(ids) > 500:
+        raise HTTPException(status_code=400, detail="ids is a list of up to 500 site ids.")
+    with session_scope() as db:
+        me = _api_me(db, caller)
+        assets = db.query(ExternalAsset).filter(ExternalAsset.id.in_(ids)).all()
+        missing = set(ids) - {a.id for a in assets}
+        if missing:
+            raise HTTPException(status_code=404, detail=f"No such sites: {sorted(missing)[:10]}")
+        changes, details = {}, {}
+        try:
+            for key in external.BULK_FIELDS:
+                if key in body:
+                    details[key] = body[key]
+            src = _api_details({k: v for k, v in body.items() if k in external.BULK_FIELDS})
+            details.update(src)
+            if "application" in body:
+                if not isinstance(body["application"], str):
+                    raise ValueError("application has to be a string.")
+                details["application"] = body["application"]
+            if details:
+                changes["details"] = details
+            for key, modes in (("tags", ("add", "remove", "replace")),
+                               ("comments", ("replace", "append")),
+                               ("repositories", ("add", "replace", "clear"))):
+                if key in body:
+                    mode = body.get(f"{key}_mode") or modes[0]
+                    if mode not in modes:
+                        raise ValueError(f"{key}_mode is one of {', '.join(modes)}.")
+                    value = body[key]
+                    if key == "repositories":
+                        value = external.picks_from_names(db, value or [])
+                    elif key == "tags" and (not isinstance(value, list)
+                                            or not all(isinstance(t, str) for t in value)):
+                        raise ValueError("tags has to be a list of strings.")
+                    elif key == "comments" and not isinstance(value, str):
+                        raise ValueError("comments has to be a string.")
+                    changes[key] = (mode, value)
+            if "no_scan" in body:
+                if not isinstance(body["no_scan"], bool):
+                    raise ValueError("no_scan has to be true or false.")
+                changes["no_scan"] = (body["no_scan"], str(body.get("no_scan_reason") or ""))
+            if "proxy" in body:
+                if body["proxy"] is not None and not isinstance(body["proxy"], str):
+                    raise ValueError("proxy has to be a string, or null to remove it.")
+                changes["proxy"] = body["proxy"] or ""
+            if "scan_auth" in body:
+                auth = body["scan_auth"] or {}
+                if not isinstance(auth, dict) or not all(isinstance(v, str) for v in auth.values()):
+                    raise ValueError("scan_auth has to be an object of strings, or null.")
+                changes["auth"] = (auth.get("kind", ""), auth)
+            if not changes:
+                raise ValueError("Name at least one thing to change.")
+            changed = external.bulk_update(db, assets, changes, me.username, me)
+        except ValueError as exc:
+            db.rollback()
+            raise HTTPException(status_code=400, detail=f"Nothing was changed.  {exc}")
+        record(db, "external.bulk_edit", actor=me.username, target=f"{len(changed)} sites",
+               detail=(", ".join(sorted(changes)) + f", api token {caller.token.prefix}")[:500],
+               source_ip=client_ip(request))
+        return {"changed": len(changed), "hosts": changed}
+
+
 @api.delete("/assets/{asset_id}")
 def api_delete(asset_id: int, request: Request, caller=Depends(_api_caller())):
     with session_scope() as db:

app/web/static/app.css

@@ -1158,3 +1158,11 @@ details.code-behind > summary::before { content: "\25B8"; color: var(--text-fain
 details.code-behind[open] > summary::before { content: "\25BE"; }
 .code-summary { font-weight: 400; font-size: .9rem; display: flex; gap: .25rem; align-items: center;
   margin-left: auto; }
+/* Bulk edit: a Change box beside each field. */
+.bulk-row { display: grid; grid-template-columns: 6.5rem 1fr; gap: .6rem; align-items: start;
+  padding: .45rem 0; border-bottom: 1px dashed var(--border); }
+.bulk-row .bulk-apply { margin: .2rem 0 0; }
+.bulk-label { font-weight: 600; font-size: .9rem; margin-bottom: .2rem; }
+.bulk-body input[type=text], .bulk-body input[type=password], .bulk-body input[type=url],
+.bulk-body input[type=number], .bulk-body select, .bulk-body textarea { width: 100%; }
+.bulk-mode { display: grid; grid-template-columns: 12rem 1fr; gap: .4rem; }

app/web/static/external.js

@@ -370,3 +370,43 @@
   window.addEventListener("hashchange", open);
   window.addEventListener("load", open);
 })();
+
+/* Bulk edit: a field that is touched is a field to change, and the ticked
+ * sites in the list go with the form. */
+(function () {
+  "use strict";
+  var form = document.querySelector("[data-bulk-form]");
+  if (!form) { return; }
+  form.querySelectorAll("[data-bulk-field]").forEach(function (row) {
+    var apply = row.querySelector('input[name^="apply_"]');
+    function mark(event) {
+      if (event.target !== apply) { apply.checked = true; apply.defaultChecked = true; }
+    }
+    row.addEventListener("input", mark);
+    row.addEventListener("change", mark);
+  });
+  var count = form.querySelector("[data-bulk-count]");
+  function ticked() {
+    return Array.prototype.filter.call(
+      document.querySelectorAll('input[name="id"][form="pick"]'), function (b) { return b.checked; });
+  }
+  document.addEventListener("click", function (event) {
+    if (event.target.closest && event.target.closest('[data-modal="bulk-edit-modal"]') && count) {
+      setTimeout(function () {
+        var n = ticked().length;
+        count.textContent = "(" + n + " site" + (n === 1 ? "" : "s") + ")";
+        form.querySelectorAll('input[name^="apply_"]').forEach(function (b) { b.defaultChecked = false; b.checked = false; });
+      }, 0);
+    }
+  });
+  form.addEventListener("submit", function (event) {
+    form.querySelectorAll('input[name="id"][type="hidden"]').forEach(function (n) { n.remove(); });
+    var picks = ticked();
+    if (!picks.length) { event.preventDefault(); return; }
+    picks.forEach(function (box) {
+      var hidden = document.createElement("input");
+      hidden.type = "hidden"; hidden.name = "id"; hidden.value = box.value;
+      form.appendChild(hidden);
+    });
+  });
+})();