Review all v2 pipelines

CHANGELOG.rst

@@ -13,10 +13,16 @@ Version v37.0.0
 - We have added new models AdvisoryV2, AdvisoryAlias, AdvisoryReference, AdvisorySeverity, AdvisoryWeakness, PackageV2 and CodeFixV2.
 - We are using ``avid`` as an internal advisory ID for uniquely identifying advisories.
 - We have a new route ``/v2`` which only support package search which has information on packages that are reported to be affected or fixing by advisories.
-- This version introduces ``/api/v2/advisories-packages`` which has information on packages that are reported to be affected or fixing by advisories.
+- This version introduces ``/api/v3/packages`` which has information on packages that are reported to be affected or fixing by advisories.
 - Pipeline Dashboard improvements #1920.
 - Throttle API requests based on user permissions #1909.
 - Add pipeline to compute Advisory ToDos #1764
+- Use related advisory severity to calculate exploitibility, weighted severity and risk scores
+- Migrate all importers to use the new advisory models. All new advisories have a unique AVID and all importers will use this AVID as the unique identifier for advisories instead of CVE ID or other identifiers used by the data sources #1881.
+- Handle advisories with same and related data https://github.com/aboutcode-org/vulnerablecode/issues/2099.
+- Add a pipeline for exporting VulnerableCode data to FederatedCode #2110.
+- Plan storing of exploits and EPSS based advisories #2069.
+
 
 Version v36.1.3
 ---------------------

PIPELINES-AVID.rst

@@ -0,0 +1,74 @@
+.. list-table:: Pipeline AVID Mapping
+   :header-rows: 1
+   :widths: 35 65
+
+   * - pipeline name
+     - AVID
+   * - alpine_linux_importer_v2
+     - {package_name}/{distroversion}/{version}/{vulnerability_id}
+   * - aosp_dataset_fix_commits
+     - CVE ID of the record
+   * - apache_httpd_importer_v2
+     - CVE ID of the record
+   * - apache_kafka_importer_v2
+     - CVE ID of the record
+   * - apache_tomcat_importer_v2
+     - {page_id}/{cve_id}
+   * - archlinux_importer_v2
+     - AVG ID of the record
+   * - curl_importer_v2
+     - CURL-CVE ID of the record
+   * - debian_importer_v2
+     - {package_name}/{debian_record_id}
+   * - elixir_security_importer_v2
+     - {package_name}/{file_id}
+   * - epss_importer_v2
+     - CVE ID of the record
+   * - fireeye_importer_v2
+     - {file_id}
+   * - gentoo_importer_v2
+     - GLSA ID of the record
+   * - github_osv_importer_v2
+     - ID of the OSV record
+   * - gitlab_importer_v2
+     - Identifier of the GitLab community advisory record
+   * - istio_importer_v2
+     - ISTIO-SECURITY-<ID>
+   * - mattermost_importer_v2
+     - MMSA-<ID>
+   * - mozilla_importer_v2
+     - MFSA-<ID>
+   * - nginx_importer_v2
+     - First alias of the record
+   * - nodejs_security_wg
+     - NPM-<ID>
+   * - nvd_importer_v2
+     - CVE ID of the record
+   * - openssl_importer_v2
+     - CVE ID of the record
+   * - oss_fuzz_importer_v2
+     - ID of the OSV record
+   * - postgresql_importer_v2
+     - CVE ID of the record
+   * - project-kb-msr-2019_v2
+     - Vulnerability ID of the record
+   * - project-kb-statements_v2
+     - Vulnerability ID of the record
+   * - pypa_importer_v2
+     - ID of the OSV record
+   * - pysec_importer_v2
+     - ID of the OSV record
+   * - redhat_importer_v2
+     - RHSA ID of the record
+   * - retiredotnet_importer_v2
+     - retiredotnet-{file_id}
+   * - ruby_importer_v2
+     - {file_id}
+   * - suse_importer_v2
+     - CVE ID of the record
+   * - ubuntu_osv_importer_v2
+     - ID of the OSV record
+   * - vulnrichment_importer_v2
+     - CVE ID of the record
+   * - xen_importer_v2
+     - XSA-<ID>

vulnerabilities/pipelines/v2_importers/alpine_linux_importer.py

@@ -7,6 +7,7 @@
 # See https://aboutcode.org for more information about nexB OSS projects.
 #
 
+import json
 import logging
 from pathlib import Path
 from typing import Any
@@ -244,4 +245,5 @@ def load_advisories(
                 references=references,
                 affected_packages=affected_packages,
                 url=url,
+                original_advisory_text=json.dumps(pkg_infos, indent=2, ensure_ascii=False),
             )

vulnerabilities/pipelines/v2_importers/apache_tomcat_importer.py

@@ -111,6 +111,7 @@ def collect_advisories(self) -> Iterable[AdvisoryDataV2]:
                         summary=advisory_list[0].summary,
                         affected_packages=affected_packages,
                         url=page_url,
+                        original_advisory_text=str(content),
                     )
 
             except Exception as e:

vulnerabilities/pipelines/v2_importers/archlinux_importer.py

@@ -14,10 +14,13 @@
 from packageurl import PackageURL
 from univers.version_range import ArchLinuxVersionRange
 
+from vulnerabilities import severity_systems
 from vulnerabilities.importer import AdvisoryDataV2
 from vulnerabilities.importer import AffectedPackageV2
 from vulnerabilities.importer import ReferenceV2
+from vulnerabilities.importer import VulnerabilitySeverity
 from vulnerabilities.pipelines import VulnerableCodeBaseImporterPipelineV2
+from vulnerabilities.severity_systems import SCORING_SYSTEMS
 from vulnerabilities.utils import fetch_response
 
 
@@ -54,6 +57,7 @@ def parse_advisory(self, record) -> AdvisoryDataV2:
         affected_packages = []
         references = []
         avg_name = record.get("name")
+        severity = record.get("severity")
         aliases = record.get("issues", [])
         aliases.extend(record.get("advisories", []))
         summary = record.get("type", "")
@@ -92,13 +96,22 @@ def parse_advisory(self, record) -> AdvisoryDataV2:
                 )
             )
 
+        severities = [
+            VulnerabilitySeverity(
+                system=severity_systems.ARCHLINUX,
+                value=severity,
+                url="https://security.archlinux.org/{avg_name}.json",
+            )
+        ]
+
         return AdvisoryDataV2(
             advisory_id=avg_name,
             aliases=aliases,
             summary=summary,
             references=references,
             affected_packages=affected_packages,
+            severities=severities,
             weaknesses=[],
             url=f"https://security.archlinux.org/{avg_name}.json",
-            original_advisory_text=json.dumps(record),
+            original_advisory_text=json.dumps(record, indent=2, ensure_ascii=False),
         )

vulnerabilities/pipelines/v2_importers/debian_importer.py

@@ -7,6 +7,7 @@
 # See https://aboutcode.org for more information about nexB OSS projects.
 #
 
+import json
 import re
 from typing import Any
 from typing import Iterable
@@ -171,6 +172,7 @@ def parse(self, pkg_name: str, records: Mapping[str, Any]) -> Iterable[AdvisoryD
                 references=references,
                 weaknesses=weaknesses,
                 url=f"https://security-tracker.debian.org/tracker/{record_identifier}",
+                original_advisory_text=json.dumps(record, indent=2, ensure_ascii=False),
             )
 
 

vulnerabilities/pipelines/v2_importers/epss_importer_v2.py

@@ -83,4 +83,5 @@ def collect_advisories(self) -> Iterable[AdvisoryDataV2]:
                 severities=[severity],
                 references=[references],
                 url=self.advisory_url,
+                original_advisory_text=",".join(epss_row),
             )

vulnerabilities/pipelines/v2_importers/istio_importer.py

@@ -42,6 +42,7 @@ class IstioImporterPipeline(VulnerableCodeBaseImporterPipelineV2):
     spdx_license_expression = "Apache-2.0"
     license_url = "https://github.com/istio/istio.io/blob/master/LICENSE"
     repo_url = "git+https://github.com/istio/istio.io"
+    run_once = True
 
     precedence = 200
 

vulnerabilities/pipelines/v2_importers/mattermost_importer.py

@@ -7,6 +7,7 @@
 # See https://aboutcode.org for more information about nexB OSS projects.
 #
 
+import json
 from typing import Iterable
 
 from packageurl import PackageURL
@@ -122,5 +123,7 @@ def collect_advisories(self) -> Iterable[AdvisoryDataV2]:
                 summary=details,
                 references=[reference],
                 affected_packages=affected_packages,
+                severities=severities,
                 url=self.url,
+                original_advisory_text=json.dumps(advisory, indent=2, ensure_ascii=False),
             )

vulnerabilities/pipelines/v2_importers/nginx_importer.py

@@ -62,7 +62,7 @@ def collect_advisories(self):
         vulnerability_list = soup.select("li p")
         for vulnerability_info in vulnerability_list:
             ngnix_advisory = parse_advisory_data_from_paragraph(vulnerability_info)
-            yield to_advisory_data(ngnix_advisory)
+            yield to_advisory_data(ngnix_advisory, vulnerability_info)
 
 
 class NginxAdvisory(NamedTuple):
@@ -79,7 +79,7 @@ def to_dict(self):
         return self._asdict()
 
 
-def to_advisory_data(nginx_adv: NginxAdvisory) -> AdvisoryDataV2:
+def to_advisory_data(nginx_adv: NginxAdvisory, vulnerability_info) -> AdvisoryDataV2:
     """
     Return AdvisoryDataV2 from an NginxAdvisory tuple.
     """
@@ -150,6 +150,7 @@ def to_advisory_data(nginx_adv: NginxAdvisory) -> AdvisoryDataV2:
         references=nginx_adv.references,
         patches=nginx_adv.patches,
         url="https://nginx.org/en/security_advisories.html",
+        original_advisory_text=str(vulnerability_info),
     )
 
 

vulnerabilities/pipelines/v2_importers/project_kb_msr2019_importer.py

@@ -90,6 +90,7 @@ def collect_advisories(self) -> Iterable[AdvisoryDataV2]:
                     patches=patches,
                     references=references,
                     url="https://github.com/SAP/project-kb/blob/main/MSR2019/dataset/vulas_db_msr2019_release.csv",
+                    original_advisory_text=",".join(row),
                 )
 
     def clean_downloads(self):

vulnerabilities/pipelines/v2_importers/project_kb_statements_importer.py

@@ -168,6 +168,7 @@ def collect_advisories(self) -> Iterable[AdvisoryDataV2]:
                 references=references,
                 patches=patches,
                 url=advisory_url,
+                original_advisory_text=saneyaml.dump(yaml_data, indent=2),
             )
 
     def clean_downloads(self):

vulnerabilities/pipelines/v2_importers/retiredotnet_importer.py

@@ -118,6 +118,7 @@ def collect_advisories(self):
                     affected_packages=affected_packages,
                     references=vuln_reference,
                     url=advisory_url,
+                    original_advisory_text=json.dumps(json_doc, indent=2, ensure_ascii=False),
                 )
 
     @staticmethod

vulnerabilities/pipelines/v2_importers/ruby_importer.py

@@ -7,6 +7,7 @@
 # See https://aboutcode.org for more information about nexB OSS projects.
 #
 
+import json
 import logging
 from pathlib import Path
 from typing import Iterable
@@ -129,6 +130,7 @@ def parse_ruby_advisory(advisory_id, record, schema_type, advisory_url):
             severities=get_severities(record),
             date_published=get_publish_time(record),
             url=advisory_url,
+            original_advisory_text=json.dumps(record, indent=2, ensure_ascii=False),
         )
 
     elif schema_type == "rubies":
@@ -147,6 +149,7 @@ def parse_ruby_advisory(advisory_id, record, schema_type, advisory_url):
             references=get_references(record),
             date_published=get_publish_time(record),
             url=advisory_url,
+            original_advisory_text=json.dumps(record, indent=2, ensure_ascii=False),
         )
 
 

vulnerabilities/pipelines/v2_importers/suse_score_importer.py

@@ -7,6 +7,7 @@
 # See https://aboutcode.org for more information about nexB OSS projects.
 #
 
+import json
 from typing import Iterable
 
 from vulnerabilities import severity_systems
@@ -67,4 +68,7 @@ def collect_advisories(self) -> Iterable[AdvisoryDataV2]:
                 severities=severities,
                 references=[],
                 url=self.url,
+                original_advisory_text=json.dumps(
+                    self.score_data[cve_id], indent=2, ensure_ascii=False
+                ),
             )

vulnerabilities/severity_systems.py

@@ -169,6 +169,14 @@ def get(self, scoring_elements: str) -> dict:
     "Low",
 ]
 
+ARCHLINUX.choices = [
+    "Critical",
+    "High",
+    "Medium",
+    "Low",
+    "Very Low",
+]
+
 # This is essentially identical to apache_http except for the addition of the "High" score,
 # which seems to be used interchangeably for "Important".
 APACHE_TOMCAT = ScoringSystem(

vulnerabilities/templates/index.html

@@ -33,6 +33,9 @@
                 </a>
             </p>
         </div>
+        <div class="notification is-info is-light has-text-centered">
+            ATTENTION: We will be deprecating V1 and V2 API by 30th June 2026. V3 endpoint is live now. Please migrate to V3 API before the deprecation date. For more details, please refer to this <a href="https://aboutcode-org.github.io/www.aboutcode.org/blog/vcio-migration-announcement-aboutcode" target="_blank" class="has-text-link is-underlined">blog</a>.
+        </div>
     </div>
 </section>
 {% endblock %}

vulnerabilities/tests/test_data/archlinux/archlinux_advisoryv2-expected.json

@@ -29,7 +29,13 @@
       }
     ],
     "patches": [],
-    "severities": [],
+    "severities": [
+      {
+        "system": "generic_textual",
+        "value": "Unknown",
+        "scoring_elements": ""
+      }
+    ],
     "date_published": null,
     "weaknesses": [],
     "url": "https://security.archlinux.org/AVG-2781.json"
@@ -66,7 +72,13 @@
       }
     ],
     "patches": [],
-    "severities": [],
+    "severities": [
+      {
+        "system": "generic_textual",
+        "value": "Unknown",
+        "scoring_elements": ""
+      }
+    ],
     "date_published": null,
     "weaknesses": [],
     "url": "https://security.archlinux.org/AVG-2780.json"
@@ -107,7 +119,13 @@
       }
     ],
     "patches": [],
-    "severities": [],
+    "severities": [
+      {
+        "system": "generic_textual",
+        "value": "Low",
+        "scoring_elements": ""
+      }
+    ],
     "date_published": null,
     "weaknesses": [],
     "url": "https://security.archlinux.org/AVG-4.json"