Expand OWASP resource mappings for issue #471

application/cmd/cre_main.py

@@ -36,6 +36,51 @@
 app = None
 
 
+def fetch_upstream_json(
+    path: str,
+    timeout: Optional[float] = None,
+    max_attempts: Optional[int] = None,
+    backoff_seconds: Optional[float] = None,
+) -> Dict[str, Any]:
+    base_url = os.environ.get("CRE_UPSTREAM_API_URL", "https://opencre.org/rest/v1")
+    timeout = timeout or float(os.environ.get("CRE_UPSTREAM_TIMEOUT_SECONDS", "30"))
+    max_attempts = max_attempts or int(os.environ.get("CRE_UPSTREAM_MAX_ATTEMPTS", "4"))
+    backoff_seconds = backoff_seconds or float(
+        os.environ.get("CRE_UPSTREAM_RETRY_BACKOFF_SECONDS", "2")
+    )
+    url = f"{base_url}{path}"
+    last_error: Optional[Exception] = None
+
+    for attempt in range(1, max_attempts + 1):
+        try:
+            response = requests.get(url, timeout=timeout)
+            if response.status_code == 200:
+                return response.json()
+
+            status_error = RuntimeError(
+                f"cannot connect to upstream status code {response.status_code}"
+            )
+            # Retry only on transient upstream failures.
+            if response.status_code < 500 and response.status_code != 429:
+                raise status_error
+            last_error = status_error
+        except requests.exceptions.RequestException as exc:
+            last_error = exc
+
+        if attempt < max_attempts:
+            logger.warning(
+                "upstream fetch failed for %s on attempt %s/%s, retrying",
+                url,
+                attempt,
+                max_attempts,
+            )
+            time.sleep(backoff_seconds * attempt)
+
+    if last_error:
+        raise RuntimeError(f"upstream fetch failed for {url}") from last_error
+    raise RuntimeError(f"upstream fetch failed for {url}")
+
+
 def register_node(node: defs.Node, collection: db.Node_collection) -> db.Node:
     """
     for each link find if either the root node or the link have a CRE,
@@ -353,6 +398,8 @@ def register_standard(
 ):
     if os.environ.get("CRE_NO_GEN_EMBEDDINGS") == "1":
         generate_embeddings = False
+    if os.environ.get("CRE_NO_CALCULATE_GAP_ANALYSIS"):
+        calculate_gap_analysis = False
 
     if not standard_entries:
         logger.warning("register_standard() called with no standard_entries")
@@ -589,15 +636,7 @@ def download_graph_from_upstream(cache: str) -> None:
     collection = db_connect(path=cache).with_graph()
 
     def download_cre_from_upstream(creid: str):
-        cre_response = requests.get(
-            os.environ.get("CRE_UPSTREAM_API_URL", "https://opencre.org/rest/v1")
-            + f"/id/{creid}"
-        )
-        if cre_response.status_code != 200:
-            raise RuntimeError(
-                f"cannot connect to upstream status code {cre_response.status_code}"
-            )
-        data = cre_response.json()
+        data = fetch_upstream_json(f"/id/{creid}")
         credict = data["data"]
         cre = defs.Document.from_dict(credict)
         if cre.id in imported_cres:
@@ -609,15 +648,7 @@ def download_cre_from_upstream(creid: str):
             if link.document.doctype == defs.Credoctypes.CRE:
                 download_cre_from_upstream(link.document.id)
 
-    root_cres_response = requests.get(
-        os.environ.get("CRE_UPSTREAM_API_URL", "https://opencre.org/rest/v1")
-        + "/root_cres"
-    )
-    if root_cres_response.status_code != 200:
-        raise RuntimeError(
-            f"cannot connect to upstream status code {root_cres_response.status_code}"
-        )
-    data = root_cres_response.json()
+    data = fetch_upstream_json("/root_cres")
     for root_cre in data["data"]:
         cre = defs.Document.from_dict(root_cre)
         register_cre(cre, collection)
@@ -880,6 +911,54 @@ def run(args: argparse.Namespace) -> None:  # pragma: no cover
         BaseParser().register_resource(
             secure_headers.SecureHeaders, db_connection_str=args.cache_file
         )
+    if args.owasp_top10_2025_in:
+        from application.utils.external_project_parsers.parsers import owasp_top10_2025
+
+        BaseParser().register_resource(
+            owasp_top10_2025.OwaspTop10_2025, db_connection_str=args.cache_file
+        )
+    if args.owasp_api_top10_2023_in:
+        from application.utils.external_project_parsers.parsers import (
+            owasp_api_top10_2023,
+        )
+
+        BaseParser().register_resource(
+            owasp_api_top10_2023.OwaspApiTop10_2023,
+            db_connection_str=args.cache_file,
+        )
+    if args.owasp_kubernetes_top10_2022_in:
+        from application.utils.external_project_parsers.parsers import (
+            owasp_kubernetes_top10_2022,
+        )
+
+        BaseParser().register_resource(
+            owasp_kubernetes_top10_2022.OwaspKubernetesTop10_2022,
+            db_connection_str=args.cache_file,
+        )
+    if args.owasp_kubernetes_top10_2025_in:
+        from application.utils.external_project_parsers.parsers import (
+            owasp_kubernetes_top10_2025,
+        )
+
+        BaseParser().register_resource(
+            owasp_kubernetes_top10_2025.OwaspKubernetesTop10_2025,
+            db_connection_str=args.cache_file,
+        )
+    if args.owasp_llm_top10_2025_in:
+        from application.utils.external_project_parsers.parsers import (
+            owasp_llm_top10_2025,
+        )
+
+        BaseParser().register_resource(
+            owasp_llm_top10_2025.OwaspLlmTop10_2025,
+            db_connection_str=args.cache_file,
+        )
+    if args.owasp_aisvs_in:
+        from application.utils.external_project_parsers.parsers import owasp_aisvs
+
+        BaseParser().register_resource(
+            owasp_aisvs.OwaspAisvs, db_connection_str=args.cache_file
+        )
     if args.pci_dss_4_in:
         from application.utils.external_project_parsers.parsers import pci_dss
 

application/frontend/src/pages/Explorer/explorer.scss

@@ -21,19 +21,40 @@ main#explorer-content {
     color: #0056b3;
   }
 
-  .search-field {
-    input {
-      font-size: 16px;
-      height: 32px;
-      width: 320px;
+  .search-field {
+    input {
+      font-size: 16px;
+      height: 32px;
+      width: 320px;
       margin-bottom: 10px;
       border-radius: 3px;
       border: 1px solid #858585;
       padding: 0 5px;
       color: #333333;
-      background-color: #ffffff;
-    }
-  }
+      background-color: #ffffff;
+    }
+  }
+
+  .tree-actions {
+    display: flex;
+    gap: 10px;
+    margin-bottom: 14px;
+
+    button {
+      height: 34px;
+      padding: 0 12px;
+      border: 1px solid #b1b0b0;
+      border-radius: 4px;
+      background: #ffffff;
+      color: #1a202c;
+      cursor: pointer;
+      font-size: 14px;
+
+      &:hover {
+        background: #f4f6f8;
+      }
+    }
+  }
 
   #graphs-menu {
     display: flex;
@@ -150,16 +171,20 @@ main#explorer-content {
     padding: 1rem;
     /* Reduce from 30px to prevent content touching edges */
 
-    .search-field {
-      input {
-        width: 100%;
-        /* Expand from fixed 320px - overflows on small screens */
-      }
-    }
-
-    .item {
-      margin: 4px 4px 4px 1rem;
-      /* Reduce from 40px left margin on mobile */
+    .search-field {
+      input {
+        width: 100%;
+        /* Expand from fixed 320px - overflows on small screens */
+      }
+    }
+
+    .tree-actions {
+      flex-wrap: wrap;
+    }
+
+    .item {
+      margin: 4px 4px 4px 1rem;
+      /* Reduce from 40px left margin on mobile */
     }
   }
-}
+}

application/frontend/src/pages/Explorer/explorer.tsx

@@ -65,6 +65,32 @@ export const Explorer = () => {
     }
   };
 
+  const collectExpandableIds = (nodes: TreeDocument[] = []): string[] => {
+    const ids: string[] = [];
+
+    const visit = (node: TreeDocument | null) => {
+      if (!node) {
+        return;
+      }
+      const contains = (node.links || []).filter((link) => link.ltype === TYPE_CONTAINS);
+      if (contains.length > 0) {
+        ids.push(node.id);
+        contains.forEach((child) => visit(child.document));
+      }
+    };
+
+    nodes.forEach((node) => visit(node));
+    return ids;
+  };
+
+  const collapseAll = () => {
+    setCollapsedItems(collectExpandableIds(filteredTree || []));
+  };
+
+  const expandAll = () => {
+    setCollapsedItems([]);
+  };
+
   useEffect(() => {
     if (dataTree.length) {
       const treeCopy = structuredClone(dataTree);
@@ -136,11 +162,7 @@ export const Explorer = () => {
         <h1>Open CRE Explorer</h1>
         <p>
           A visual explorer of Open Common Requirement Enumerations (CREs). Originally created by:{' '}
-          <a
-            target="_blank"
-            rel="noopener noreferrer"
-            href="https://zeljkoobrenovic.github.io/opencre-explorer/"
-          >
+          <a target="_blank" rel="noopener noreferrer" href="https://zeljkoobrenovic.github.io/opencre-explorer/">
             Zeljko Obrenovic
           </a>
           .
@@ -151,6 +173,14 @@ export const Explorer = () => {
             <input id="filter" type="text" placeholder="Search Explorer..." onKeyUp={update} />
             <div id="search-summary"></div>
           </div>
+          <div className="tree-actions">
+            <button type="button" onClick={expandAll}>
+              Expand all
+            </button>
+            <button type="button" onClick={collapseAll}>
+              Collapse all
+            </button>
+          </div>
           <div id="graphs-menu">
             <h4 className="menu-title">Explore visually:</h4>
             <ul>

application/frontend/src/pages/GapAnalysis/GapAnalysis.scss

@@ -5,6 +5,23 @@ main#gap-analysis {
   span.name {
     padding: 0 10px;
   }
+
+  .specialized-cheatsheets {
+    margin-top: 1.5rem;
+
+    h2 {
+      margin-bottom: 0.4rem;
+    }
+
+    p {
+      margin-bottom: 0.9rem;
+      color: #4a5568;
+    }
+
+    table {
+      border-top: 3px solid #2b6cb0;
+    }
+  }
 }
 
 @media (min-width: 0px) and (max-width: 500px) {