External sites take their environments from Settings

The environments a site can be in were fixed at Prod, Dev, Stage and Clone.
They are now a list under Settings, Web and external sites, that starts with
those four.  The first one is the default for a new site.  Names are kept in
lower case on the site, so every existing site stays where it was, and
renaming Prod to PROD moves nothing.  A site in an environment taken off the
list keeps it, says it is no longer listed, and can still be saved, until
somebody picks another.  The column is widened to 40 characters on start.

Author: hackrange

app/core/api_reference.py

@@ -455,7 +455,8 @@ def _f(name, kind, required, description):
     _f("technical_owner", "string", False, "Email addresses, comma separated."),
     _f("comments", "string", False, "Anything else about the site."),
     _f("status", "string", False, "active or obsolete."),
-    _f("environment", "string", False, "prod, dev, stage or clone."),
+    _f("environment", "string", False, "One of the environments listed in Settings, "
+                                       "which start as Prod, Dev, Stage and Clone."),
     _f("external_auth_needed", "boolean", False, "Signing in goes through something outside the site."),
     _f("whitelist_needed", "boolean", False, "The scanner has to be let through a firewall."),
     _f("change_frequency_days", "integer", False, "How often the site changes, in days."),

app/core/dast_report.py

@@ -328,7 +328,7 @@ def build(db, asset, scan, user=None, organization: str = "") -> dict:
         "profile": profile, "profile_text": dast_engines.PROFILES.get(profile, ""),
         "limitations": limitations, "links": links, "sast": counts,
         "tele": tele, "fields": fields,
-        "environment": external.ENVIRONMENTS.get(asset.environment, asset.environment or ""),
+        "environment": external.environments(db).get(asset.environment, asset.environment or ""),
         "started": started, "finished": finished,
         "duration": int((finished - started).total_seconds()) if started and finished else None,
         "organization_name": organization,

app/core/external.py

@@ -50,7 +50,52 @@
 SERVER_KINDS = {"nextgen-dast": "nextgen-dast (started and read from here)",
                 "other": "Other (scans are recorded by hand or through the API)"}
 BUILTIN_SERVER = "Git Code Review"
+# The starting list.  The real one is the external.environments setting; see
+# environments().  A site stores the name in lower case, so renaming Prod to
+# PROD keeps every site in it, and these four match what sites always held.
 ENVIRONMENTS = {"prod": "Prod", "dev": "Dev", "stage": "Stage", "clone": "Clone"}
+ENV_NAME = re.compile(r"^[A-Za-z0-9][A-Za-z0-9 ._()/&+-]{0,39}$")
+
+
+def env_key(name) -> str:
+    return re.sub(r"\s+", " ", str(name or "")).strip().casefold()[:40]
+
+
+def clean_environments(raw) -> list:
+    """The Settings list, checked.  Comma or line separated.  Raises ValueError."""
+    if isinstance(raw, str):
+        raw = raw.replace("\r", "\n").replace("\n", ",").split(",")
+    out, seen = [], set()
+    for item in raw or []:
+        name = re.sub(r"\s+", " ", str(item or "")).strip()
+        if not name:
+            continue
+        if not ENV_NAME.fullmatch(name):
+            raise ValueError(f"Environment {name[:60]!r} has to be up to 40 letters, numbers, "
+                             "spaces or . _ ( ) / & + - and start with a letter or number.")
+        if env_key(name) not in seen:
+            seen.add(env_key(name))
+            out.append(name)
+    if not out:
+        raise ValueError("Keep at least one environment.")
+    if len(out) > 40:
+        raise ValueError("At most 40 environments.")
+    return out
+
+
+def environments(db) -> dict:
+    """{key: name} from Settings, in its order.  The first is the default."""
+    from app.core import settings_store
+
+    try:
+        names = clean_environments(settings_store.get(db, "external.environments") or [])
+    except ValueError:
+        names = list(ENVIRONMENTS.values())
+    return {env_key(n): n for n in names}
+
+
+def default_environment(db) -> str:
+    return next(iter(environments(db)))
 STATUSES = {"active": "Active", "obsolete": "Obsolete"}
 MAX_PICKS = 200
 MAX_BRANCHES = 300
@@ -168,7 +213,8 @@ def clean_owners(value, what: str) -> str:
     return text
 
 
-def clean_details(data: dict, partial: bool = False) -> dict:
+def clean_details(data: dict, partial: bool = False, envs: dict = None,
+                  keep_env: str = "") -> dict:
     """The fields a person fills in about a site, checked.
 
     data holds strings as a form sends them, or JSON values from the API.
@@ -197,9 +243,13 @@ def present(key):
             raise ValueError("Status is Active or Obsolete.")
         out["status"] = value
     if present("environment") or not partial:
-        value = str(data.get("environment") or "prod").strip().lower()
-        if value not in ENVIRONMENTS:
-            raise ValueError("Environment is Prod, Dev, Stage or Clone.")
+        envs = envs or ENVIRONMENTS
+        value = env_key(data.get("environment")) or next(iter(envs))
+        # A site may keep an environment that was taken off the list since.
+        if value not in envs and value != env_key(keep_env):
+            names = list(envs.values())
+            raise ValueError("Environment is " + (", ".join(names[:-1]) + " or " + names[-1]
+                                                  if len(names) > 1 else names[0]) + ".")
         out["environment"] = value
     for key in ("external_auth_needed", "whitelist_needed"):
         if present(key) or not partial:
@@ -457,6 +507,7 @@ def add_assets(db, rows: list, actor: str, no_scan: bool = False,
                                   no_scan_reason=(row.get("no_scan_reason")
                                                   or no_scan_reason or "")[:300])
             db.add(asset)
+            asset.environment = default_environment(db)
             apply_details(asset, details or {})
             db.flush()
             if picks:
@@ -969,7 +1020,7 @@ def asset_dict(asset: ExternalAsset, links: list = None, counts: dict = None,
         "executive_owner": asset.executive_owner or "",
         "technical_owner": asset.technical_owner or "",
         "status": asset.status or "active",
-        "environment": asset.environment or "prod",
+        "environment": asset.environment or "",
         "external_auth_needed": bool(asset.external_auth_needed),
         "whitelist_needed": bool(asset.whitelist_needed),
         "change_frequency_days": asset.change_frequency_days,

app/core/models.py

@@ -2278,8 +2278,9 @@ class ExternalAsset(Base):
     # active or obsolete.  An obsolete site is kept for its history and left
     # out of the nightly telemetry.
     status = Column(String(16), default="active", nullable=False)
-    # prod, dev, stage or clone.
-    environment = Column(String(16), default="prod", nullable=False)
+    # One of the environments listed in Settings, by its name in lower case.
+    # The list starts as Prod, Dev, Stage and Clone.
+    environment = Column(String(40), default="prod", nullable=False)
     # Whether a scan needs to sign in through something outside the site, and
     # whether the scanner's address has to be let through a firewall first.
     external_auth_needed = Column(Boolean, default=False, nullable=False)

app/core/settings_store.py

@@ -338,6 +338,10 @@
                                 "Before an engine scan, read the default branch of "
                                 "the repositories mapped to the site for routes to "
                                 "add to the scan and for the site's name in the code."),
+    "external.environments":   (["Prod", "Dev", "Stage", "Clone"], False,
+                                "The environments a site can be in. The first is the "
+                                "default for a new site. A site keeps one that is "
+                                "taken off the list until somebody picks another."),
     "external.user_agent":     ("", False,
                                 "The browser every scan and telemetry read says it is. "
                                 "Empty uses a current desktop Chrome. It has to start "
@@ -498,7 +502,8 @@
     ("Web testing", ["dast.max_age_days"]),
     ("External sites", ["external.enabled", "external.screenshots",
                         "external.refresh_days", "external.whitebox",
-                        "external.user_agent", "external.allow_private"]),
+                        "external.environments", "external.user_agent",
+                        "external.allow_private"]),
     ("Limits", ["github.parallel_repos", "scan.max_concurrent",
                 "secrets.history_window_days",
                 "scan.timeout_per_repo", "scan.max_hours",

app/web/routes_api.py

@@ -1899,6 +1899,13 @@ def put(key, value):
                 "The user agent for scans has to be one line starting Mozilla/5.0 (, "
                 "the way a real browser's does."))
         put("external.user_agent", agent)
+        from app.core import external as _external
+
+        try:
+            put("external.environments", _external.clean_environments(
+                text("external.environments") or "Prod, Dev, Stage, Clone"))
+        except ValueError as exc:
+            raise HTTPException(status_code=400, detail=f"Environments: {exc}")
         put("external.refresh_days", clean_int(form.get("external.refresh_days", 7) or 0,
                                                "Read telemetry again after",
                                                low=0, high=365))

app/web/routes_external.py

@@ -144,6 +144,7 @@ def list_page(request: Request, q: str = "", tag: str = "", app: str = "",
         app = clean_text(app, "Application", limit=200, required=False)
         opts = external.options(db)
         mapper = external.Mapper(db, user)
+        envs = external.environments(db)
         assets = db.query(ExternalAsset).order_by(ExternalAsset.host).all()
         blocked = external.blocked_hosts(db)
         every_link = {a.id: mapper.links(a) for a in assets}
@@ -166,7 +167,7 @@ def list_page(request: Request, q: str = "", tag: str = "", app: str = "",
                 "tele": external.telemetry_of(a),
                 "scan": scanning.get(a.id),
             }
-            if env and (a.environment or "prod") != env:
+            if env and (a.environment or "") != env:
                 continue
             if status and (a.status or "active") != status:
                 continue
@@ -217,7 +218,7 @@ def list_page(request: Request, q: str = "", tag: str = "", app: str = "",
                        profiles=dast_engines.PROFILES, default_profile=dast_engines.DEFAULT_PROFILE,
                        q=q, tag=tag, app=app, grade=grade, dast=dast, risk=risk,
                        scope=scope, all_tags=all_tags, apps=apps, opts=opts,
-                       env=env, status=status, environments=external.ENVIRONMENTS,
+                       env=env, status=status, environments=envs,
                        statuses=external.STATUSES,
                        servers=servers,
                        running=sum(1 for a in assets if a.telemetry_status == "running"),
@@ -235,6 +236,7 @@ def export_csv(request: Request, user: User = Depends(current_user)):
     with session_scope() as db:
         _guard(db, user)
         mapper = external.Mapper(db, user)
+        envs = external.environments(db)
         out = io.StringIO()
         writer = csv.writer(out)
         writer.writerow(["host", "kind", "parent_domain", "application", "tags",
@@ -253,7 +255,7 @@ def export_csv(request: Request, user: User = Depends(current_user)):
                         or [l["full_name"]]
             writer.writerow([_quote_formula(str(c)) for c in [
                 a.host, a.kind, a.parent_domain, a.application, "; ".join(a.tag_list()),
-                external.ENVIRONMENTS.get(a.environment, a.environment or ""),
+                envs.get(a.environment, a.environment or ""),
                 external.STATUSES.get(a.status, a.status or ""),
                 a.executive_owner or "", a.technical_owner or "",
                 "yes" if a.external_auth_needed else "no",
@@ -288,7 +290,7 @@ async def add(request: Request, user: User = Depends(current_user)):
         no_scan = _checked(form, "no_scan")
         reason = clean_text(form.get("no_scan_reason"), "Reason", limit=300, required=False)
         try:
-            details = external.clean_details(_detail_form(form))
+            details = external.clean_details(_detail_form(form), envs=external.environments(db))
             picks = external.parse_picks(form.getlist("pick"))
             got = external.add_assets(db, rows, user.username, no_scan, reason, tags,
                                       details=details, picks=picks, user=user)
@@ -502,7 +504,7 @@ def detail_page(asset_id: int, request: Request, scans: int = 1,
             auth=external_auth_mod.describe(asset), auth_kinds=external_auth_mod.KINDS,
             site_profile=external.profile_for(asset),
             picks_json=json.dumps(external.picks_of(db, asset)),
-            environments=external.ENVIRONMENTS, statuses=external.STATUSES,
+            environments=external.environments(db), statuses=external.STATUSES,
             scan_views=[external.scan_dict(s) for s in page_scans],
             servers=db.query(DastServer).order_by(DastServer.name).all(),
             profiles=dast_engines.PROFILES, active=dast_engines.ACTIVE_PROFILES,
@@ -541,7 +543,8 @@ async def edit(asset_id: int, request: Request, user: User = Depends(current_use
         asset.application = clean_text(form.get("application"), "Application",
                                        limit=200, required=False)
         try:
-            external.apply_details(asset, external.clean_details(_detail_form(form)))
+            external.apply_details(asset, external.clean_details(
+                _detail_form(form), envs=external.environments(db), keep_env=asset.environment))
             if form.get("with_picks"):
                 external.set_picks(db, asset, external.parse_picks(form.getlist("pick")),
                                    user.username, user)
@@ -1053,7 +1056,7 @@ def api_list(request: Request, caller=Depends(_api_caller())):
         if params.get("severity"):
             query = query.filter(ExternalAsset.last_dast_severity == params["severity"][:16])
         if params.get("environment"):
-            query = query.filter(ExternalAsset.environment == params["environment"].lower()[:16])
+            query = query.filter(ExternalAsset.environment == external.env_key(params["environment"]))
         if params.get("status"):
             query = query.filter(ExternalAsset.status == params["status"].lower()[:16])
         total = query.count()
@@ -1079,7 +1082,7 @@ async def api_add(request: Request, caller=Depends(_api_caller())):
         me = _api_me(db, caller)
         existed = db.query(ExternalAsset).filter(ExternalAsset.host == host).one_or_none()
         try:
-            details = external.clean_details(_api_details(body))
+            details = external.clean_details(_api_details(body), envs=external.environments(db))
             picks = external.picks_from_names(db, body.get("repositories") or [])
             got = external.add_assets(db, [{
                 "host": host, "application": _str_field(body, "application", 200) or "",
@@ -1117,8 +1120,9 @@ async def api_patch(asset_id: int, request: Request, caller=Depends(_api_caller(
         if value is not None:
             asset.application = value
         try:
-            external.apply_details(asset, external.clean_details(_api_details(body),
-                                                                 partial=True))
+            external.apply_details(asset, external.clean_details(
+                _api_details(body), partial=True, envs=external.environments(db),
+                keep_env=asset.environment))
             if "repositories" in body:
                 external.set_picks(db, asset,
                                    external.picks_from_names(db, body["repositories"] or []),

app/web/templates/_external_fields.html

@@ -17,10 +17,15 @@
     <span class="faint small">Email only.</span></label>
   <label>Environment
     <select name="environment">
+      {% set current = a.environment if a else (environments | list | first) %}
       {% for key, label in environments.items() %}
-      <option value="{{ key }}" {{ 'selected' if (a.environment if a else 'prod') == key }}>{{ label }}</option>
+      <option value="{{ key }}" {{ 'selected' if current == key }}>{{ label }}</option>
       {% endfor %}
-    </select></label>
+      {% if a and a.environment and a.environment not in environments %}
+      <option value="{{ a.environment }}" selected>{{ a.environment }} (no longer in the list)</option>
+      {% endif %}
+    </select>
+    <span class="faint small">The list is in Settings, under Web and external sites.</span></label>
   <label>Status
     <select name="status">
       {% for key, label in statuses.items() %}

app/web/templates/external.html

@@ -98,7 +98,7 @@
             {% if a.application %}<div class="small">{{ a.application }}</div>{% endif %}
             <div style="margin-top:.2rem">{% for t in r.tags %}<a class="badge" href="{{ here }}?tag={{ t | urlencode }}">{{ t }}</a> {% endfor %}</div>
             {% if r.blocked %}<span class="badge waived" title="{{ a.no_scan_reason or 'under a domain marked no scan' }}">no scan</span>{% endif %}
-            <div style="margin-top:.2rem"><span class="badge {{ 'high' if a.environment == 'prod' else 'info' }}">{{ environments.get(a.environment, a.environment) }}</span>
+            <div style="margin-top:.2rem"><span class="badge {{ 'high' if (a.environment or '').startswith('prod') else 'info' }}" {% if a.environment not in environments %}title="No longer in the environments list"{% endif %}>{{ environments.get(a.environment, a.environment) }}</span>
               {% if a.status == 'obsolete' %}<span class="badge waived">obsolete</span>{% endif %}
               {% if a.external_auth_needed %}<span class="badge info" title="External auth needed">ext auth</span>{% endif %}
               {% if a.whitelist_needed %}<span class="badge pending" title="Whitelist needed">whitelist</span>{% endif %}</div>

app/web/templates/external_site.html

@@ -10,7 +10,7 @@
       {%- if asset.application %} · {{ asset.application }}{% endif %}
       &middot; added {{ asset.added_at | dt('%Y-%m-%d') }}{% if asset.added_by %} by {{ asset.added_by }}{% endif %}</div>
     <div style="margin-top:.3rem">
-      <span class="badge {{ 'high' if asset.environment == 'prod' else 'info' }}">{{ environments.get(asset.environment, asset.environment) }}</span>
+      <span class="badge {{ 'high' if (asset.environment or '').startswith('prod') else 'info' }}">{{ environments.get(asset.environment, asset.environment) }}</span>
       <span class="badge {{ 'waived' if asset.status == 'obsolete' else 'ok' }}">{{ statuses.get(asset.status, asset.status) }}</span>
       {% for t in tags %}<a class="badge" href="{{ base }}/external?tag={{ t | urlencode }}">{{ t }}</a> {% endfor %}
       {% if blocked %}<span class="badge waived">no scan</span> <span class="small faint">{{ blocked }}</span>{% endif %}</div>
@@ -77,7 +77,7 @@
         <tr><th>Application</th><td>{{ asset.application or '' }}{% if not asset.application %}<span class="faint">nobody has said</span>{% endif %}</td></tr>
         <tr><th>Executive owner</th><td>{% for o in (asset.executive_owner or '').split(', ') if o %}<div><a href="mailto:{{ o }}">{{ o }}</a></div>{% else %}<span class="faint">nobody has said</span>{% endfor %}</td></tr>
         <tr><th>Technical owner</th><td>{% for o in (asset.technical_owner or '').split(', ') if o %}<div><a href="mailto:{{ o }}">{{ o }}</a></div>{% else %}<span class="faint">nobody has said</span>{% endfor %}</td></tr>
-        <tr><th>Environment</th><td>{{ environments.get(asset.environment, asset.environment) }}</td></tr>
+        <tr><th>Environment</th><td>{{ environments.get(asset.environment, asset.environment) }}{% if asset.environment not in environments %} <span class="faint small">(no longer in the list in Settings)</span>{% endif %}</td></tr>
         <tr><th>Status</th><td>{{ statuses.get(asset.status, asset.status) }}</td></tr>
         <tr><th>External auth needed</th><td>{{ 'Yes' if asset.external_auth_needed else 'No' }}</td></tr>
         <tr><th>Whitelist needed</th><td>{{ 'Yes' if asset.whitelist_needed else 'No' }}</td></tr>