Store advisories in security_advisories cluster

Signed-off-by: Keshav Priyadarshi <git@keshav.space>

Author: keshav-space

vulnerabilities/pipelines/exporters/federate_vulnerabilities.py

@@ -36,8 +36,8 @@ def steps(cls):
             cls.check_federatedcode_eligibility,
             cls.create_federatedcode_working_dir,
             cls.fetch_federation_config,
-            cls.clone_vulnerabilities_repo,
-            cls.publish_package_vulnerabilities,
+            cls.clone_federation_repository,
+            cls.publish_package_related_advisories,
             cls.publish_advisories,
             cls.delete_working_dir,
         )
@@ -56,54 +56,44 @@ def fetch_federation_config(self):
             name="aboutcode-data",
             remote_root_url="https://github.com/aboutcode-data",
         )
-        self.data_cluster = data_federation.get_cluster("purls")
+        self.data_cluster = data_federation.get_cluster("security_advisories")
 
-    def clone_vulnerabilities_repo(self):
+    def clone_federation_repository(self):
         self.repo = federatedcode.clone_repository(
             repo_url=settings.FEDERATEDCODE_VULNERABILITIES_REPO,
-            clone_path=self.working_path / "vulnerabilities-data",
+            clone_path=self.working_path / "advisories-data",
             logger=self.log,
         )
 
-    def publish_package_vulnerabilities(self):
-        """Publish package vulnerabilities to FederatedCode"""
+    def publish_package_related_advisories(self):
+        """Publish package advisories relations to FederatedCode"""
         repo_path = Path(self.repo.working_dir)
         commit_count = 1
         batch_size = 2000
-        chunk_size = 1000
+        chunk_size = 500
         files_to_commit = set()
 
         distinct_packages_count = (
-            PackageV2.objects.values("type", "namespace", "name")
-            .distinct("type", "namespace", "name")
+            PackageV2.objects.values("type", "namespace", "name", "version")
+            .distinct("type", "namespace", "name", "version")
             .count()
         )
         package_qs = package_prefetched_qs()
         grouped_packages = itertools.groupby(
             package_qs.iterator(chunk_size=chunk_size),
-            key=attrgetter("type", "namespace", "name"),
+            key=attrgetter("type", "namespace", "name", "version"),
         )
 
-        self.log(f"Exporting vulnerabilities for {distinct_packages_count} packages.")
+        self.log(f"Exporting advisory relation for {distinct_packages_count} packages.")
         progress = LoopProgress(
             total_iterations=distinct_packages_count,
             progress_step=5,
             logger=self.log,
         )
         for _, packages in progress.iter(grouped_packages):
-            package_urls, package_vulnerabilities = get_package_vulnerabilities(packages)
-            purl = package_urls[0]
-            package_repo, datafile_path = self.data_cluster.get_datafile_repo_and_path(purl=purl)
-            package_vulnerability_path = datafile_path.replace("/purls.yml", "/vulnerabilities.yml")
-            package_vulnerability_path = f"packages/{package_repo}/{package_vulnerability_path}"
-            package_path = f"packages/{package_repo}/{datafile_path}"
-
-            write_file(
-                repo_path=repo_path,
-                file_path=package_path,
-                data=package_urls,
-            )
-            files_to_commit.add(package_path)
+            purl, package_vulnerabilities = get_package_related_advisory(packages)
+            package_repo, datafile_path = self.data_cluster.get_datafile_repo_and_path(purl)
+            package_vulnerability_path = f"packages/{package_repo}/{datafile_path}"
 
             write_file(
                 repo_path=repo_path,
@@ -114,7 +104,7 @@ def publish_package_vulnerabilities(self):
 
             if len(files_to_commit) > batch_size:
                 if federatedcode.commit_and_push_changes(
-                    commit_message=self.commit_message("package vulnerabilities", commit_count),
+                    commit_message=self.commit_message("package advisory relations", commit_count),
                     repo=self.repo,
                     files_to_commit=files_to_commit,
                     logger=self.log,
@@ -125,7 +115,7 @@ def publish_package_vulnerabilities(self):
         if files_to_commit:
             federatedcode.commit_and_push_changes(
                 commit_message=self.commit_message(
-                    "package vulnerabilities",
+                    "package advisory relations",
                     commit_count,
                     commit_count,
                 ),
@@ -134,7 +124,7 @@ def publish_package_vulnerabilities(self):
                 logger=self.log,
             )
 
-        self.log(f"Federated {distinct_packages_count} package vulnerabilities.")
+        self.log(f"Federated {distinct_packages_count} package advisories.")
 
     def publish_advisories(self):
         """Publish advisory to FederatedCode"""
@@ -146,15 +136,15 @@ def publish_advisories(self):
         advisory_qs = advisory_prefetched_qs()
         advisory_count = advisory_qs.count()
 
-        self.log(f"Exporting vulnerabilities for {advisory_count} advisory.")
+        self.log(f"Exporting {advisory_count} advisory.")
         progress = LoopProgress(
             total_iterations=advisory_count,
             progress_step=5,
             logger=self.log,
         )
         for advisory in progress.iter(advisory_qs.iterator(chunk_size=chunk_size)):
             advisory_data = serialize_advisory(advisory)
-            adv_file = f"vulnerabilities/{advisory.avid}.yml"
+            adv_file = f"advisories/{advisory.avid}.yml"
             write_file(
                 repo_path=repo_path,
                 file_path=adv_file,
@@ -184,7 +174,7 @@ def publish_advisories(self):
                 logger=self.log,
             )
 
-        self.log(f"Successfully federated {advisory_count} vulnerabilities.")
+        self.log(f"Successfully federated {advisory_count} advisories.")
 
     def delete_working_dir(self):
         """Remove temporary working dir."""
@@ -200,7 +190,7 @@ def commit_message(
         commit_count,
         total_commit_count="many",
     ):
-        """Commit message for pushing Package vulnerability."""
+        """Commit message for pushing package vulnerability."""
         return federatedcode.commit_message(
             item_type=item_type,
             commit_count=commit_count,
@@ -211,30 +201,48 @@ def commit_message(
 def package_prefetched_qs():
     return (
         PackageV2.objects.order_by("type", "namespace", "name", "version")
-        .only("id", "package_url", "type", "namespace", "name", "version")
+        .only("package_url", "type", "namespace", "name", "version")
         .prefetch_related(
             Prefetch(
                 "affected_in_impacts",
-                queryset=ImpactedPackage.objects.only("id", "advisory_id").prefetch_related(
+                queryset=ImpactedPackage.objects.only("advisory_id").prefetch_related(
                     Prefetch(
                         "advisory",
-                        queryset=AdvisoryV2.objects.only("id", "avid"),
+                        queryset=AdvisoryV2.objects.only("avid"),
                     )
                 ),
             ),
             Prefetch(
                 "fixed_in_impacts",
-                queryset=ImpactedPackage.objects.only("id", "advisory_id").prefetch_related(
+                queryset=ImpactedPackage.objects.only("advisory_id").prefetch_related(
                     Prefetch(
                         "advisory",
-                        queryset=AdvisoryV2.objects.only("id", "avid"),
+                        queryset=AdvisoryV2.objects.only("avid"),
                     )
                 ),
             ),
         )
     )
 
 
+def get_package_related_advisory(packages):
+    package_vulnerabilities = []
+    for package in packages:
+        affected_by_vulnerabilities = [
+            impact.advisory.avid for impact in package.affected_in_impacts.all()
+        ]
+        fixing_vulnerabilities = [impact.advisory.avid for impact in package.fixed_in_impacts.all()]
+
+        package_vulnerability = {
+            "purl": package.package_url,
+            "affected_by_advisories": sorted(affected_by_vulnerabilities),
+            "fixing_advisories": sorted(fixing_vulnerabilities),
+        }
+        package_vulnerabilities.append(package_vulnerability)
+
+    return package.package_url, package_vulnerabilities
+
+
 def advisory_prefetched_qs():
     return AdvisoryV2.objects.prefetch_related(
         "impacted_packages",
@@ -245,29 +253,6 @@ def advisory_prefetched_qs():
     )
 
 
-def get_package_vulnerabilities(packages):
-    """Return list of PURLs and serialized package vulnerability"""
-    package_urls = []
-    package_vulnerabilities = []
-    for package in packages:
-        package_urls.append(package.package_url)
-        package_vulnerabilities.append(serialize_package_vulnerability(package))
-    return package_urls, package_vulnerabilities
-
-
-def serialize_package_vulnerability(package):
-    affected_by_vulnerabilities = [
-        impact.advisory.avid for impact in package.affected_in_impacts.all()
-    ]
-    fixing_vulnerabilities = [impact.advisory.avid for impact in package.fixed_in_impacts.all()]
-
-    return {
-        "purl": package.package_url,
-        "affected_by_vulnerabilities": affected_by_vulnerabilities,
-        "fixing_vulnerabilities": fixing_vulnerabilities,
-    }
-
-
 def serialize_severity(sev):
     return {
         "score": sev.value,
@@ -288,7 +273,7 @@ def serialize_references(reference):
 
 def serialize_advisory(advisory):
     """Return a plain data mapping serialized from advisory object."""
-    aliases = [a.alias for a in advisory.aliases.all()]
+    aliases = sorted([a.alias for a in advisory.aliases.all()])
     severities = [serialize_severity(sev) for sev in advisory.severities.all()]
     weaknesses = [wkns.cwe for wkns in advisory.weaknesses.all()]
     references = [serialize_references(ref) for ref in advisory.references.all()]

vulnerabilities/tests/pipelines/exporters/test_federate_vulnerabilities.py

@@ -37,7 +37,7 @@ class TestFederatePackageVulnerabilities(TestCase):
     def setUp(self):
         self.logger = TestLogger()
 
-        advisory = AdvisoryDataV2(
+        advisory1 = AdvisoryDataV2(
             summary="Test advisory",
             aliases=["CVE-2025-0001"],
             references=[],
@@ -51,53 +51,65 @@ def setUp(self):
                     introduced_by_commit_patches=[],
                     fixed_by_commit_patches=[],
                 ),
+            ],
+            patches=[],
+            advisory_id="ADV-001",
+            date_published=datetime.now() - timedelta(days=10),
+            url="https://example.com/advisory/1",
+        )
+        advisory2 = AdvisoryDataV2(
+            summary="Test advisory2",
+            aliases=["CVE-2025-0002"],
+            references=[],
+            severities=[],
+            weaknesses=[],
+            affected_packages=[
                 AffectedPackageV2(
                     package=PackageURL.from_string("pkg:npm/foobar"),
-                    affected_version_range=VersionRange.from_string("vers:npm/<=3.2.3"),
-                    fixed_version_range=VersionRange.from_string("vers:npm/3.2.4"),
+                    affected_version_range=VersionRange.from_string("vers:npm/>=1.2.4"),
+                    fixed_version_range=VersionRange.from_string("vers:npm/2.0.0"),
                     introduced_by_commit_patches=[],
                     fixed_by_commit_patches=[],
                 ),
             ],
             patches=[],
-            advisory_id="ADV-123",
+            advisory_id="ADV-002",
             date_published=datetime.now() - timedelta(days=10),
-            url="https://example.com/advisory/1",
+            url="https://example.com/advisory/2",
         )
         insert_advisory_v2(
-            advisory=advisory,
+            advisory=advisory1,
+            pipeline_id="test_pipeline_v2",
+        )
+        insert_advisory_v2(
+            advisory=advisory2,
             pipeline_id="test_pipeline_v2",
         )
 
     @patch(
-        "vulnerabilities.pipelines.exporters.federate_vulnerabilities.FederatePackageVulnerabilities.clone_vulnerabilities_repo"
+        "vulnerabilities.pipelines.exporters.federate_vulnerabilities.FederatePackageVulnerabilities.clone_federation_repository"
     )
     @patch("vulnerabilities.pipes.federatedcode.commit_and_push_changes")
     @patch("vulnerabilities.pipes.federatedcode.check_federatedcode_configured_and_available")
     def test_vulnerabilities_federation_v2(self, mock_check_fed, mock_commit, mock_clone):
         mock_check_fed.return_value = None
         mock_commit.return_value = None
-        mock_clone.__name__ = "clone_vulnerabilities_repo"
+        mock_clone.__name__ = "clone_federation_repository"
 
         working_dir = Path(tempfile.mkdtemp())
-        print(working_dir)
-
         pipeline = FederatePackageVulnerabilities()
         pipeline.repo = Repo.init(working_dir)
         pipeline.log = self.logger.write
         pipeline.execute()
-        print(self.logger.getvalue())
 
-        result_purl_yml = next(working_dir.rglob("purls.yml"))
-        result_vulnerabilities_yml = next(working_dir.rglob("vulnerabilities.yml"))
-        result_advisory_yml = next(working_dir.rglob("ADV-123.yml"))
+        result_advisories_yml = next(working_dir.rglob("1.2.4/advisories.yml"))
+        result_advisory1_yml = next(working_dir.rglob("ADV-001.yml"))
+        result_advisory2_yml = next(working_dir.rglob("ADV-002.yml"))
 
-        expected_purl_yml = TEST_DATA / "purls-expected.yml"
-        expected_vulnerabilities_yml = TEST_DATA / "vulnerabilities-expected.yml"
-        expected_advisory_yml = TEST_DATA / "ADV-123-expected.yml"
+        expected_advisories_yml = TEST_DATA / "1.2.4" / "advisories-expected.yml"
+        expected_advisory1_yml = TEST_DATA / "ADV-001-expected.yml"
+        expected_advisory2_yml = TEST_DATA / "ADV-002-expected.yml"
 
-        util_tests.check_results_and_expected_files(result_purl_yml, expected_purl_yml)
-        util_tests.check_results_and_expected_files(
-            result_vulnerabilities_yml, expected_vulnerabilities_yml
-        )
-        util_tests.check_results_and_expected_files(result_advisory_yml, expected_advisory_yml)
+        util_tests.check_results_and_expected_files(result_advisories_yml, expected_advisories_yml)
+        util_tests.check_results_and_expected_files(result_advisory1_yml, expected_advisory1_yml)
+        util_tests.check_results_and_expected_files(result_advisory2_yml, expected_advisory2_yml)

vulnerabilities/tests/test_data/exporters/federate_vulnerabilities/1.2.4/advisories-expected.yml

@@ -0,0 +1,5 @@
+- purl: pkg:npm/foobar@1.2.4
+  affected_by_advisories:
+    - test_pipeline_v2/ADV-002
+  fixing_advisories:
+    - test_pipeline_v2/ADV-001

…ate_vulnerabilities/ADV-123-expected.yml …ate_vulnerabilities/ADV-001-expected.ymlvulnerabilities/tests/test_data/exporters/federate_vulnerabilities/ADV-123-expected.yml renamed to vulnerabilities/tests/test_data/exporters/federate_vulnerabilities/ADV-001-expected.yml vulnerabilities/tests/test_data/exporters/federate_vulnerabilities/ADV-123-expected.yml renamed to vulnerabilities/tests/test_data/exporters/federate_vulnerabilities/ADV-001-expected.yml

@@ -1,5 +1,5 @@
-advisory_id: ADV-123
-datasource_id: test_pipeline_v2/ADV-123
+advisory_id: ADV-001
+datasource_id: test_pipeline_v2/ADV-001
 datasource_url: https://example.com/advisory/1
 aliases:
   - CVE-2025-0001
@@ -8,9 +8,6 @@ impacted_packages:
   - purl: pkg:npm/foobar
     affected_versions: vers:npm/<=1.2.3
     fixed_versions: vers:npm/1.2.4
-  - purl: pkg:npm/foobar
-    affected_versions: vers:npm/<=3.2.3
-    fixed_versions: vers:npm/3.2.4
 severities: []
 weaknesses: []
 references: []

vulnerabilities/tests/test_data/exporters/federate_vulnerabilities/ADV-002-expected.yml

@@ -0,0 +1,13 @@
+advisory_id: ADV-002
+datasource_id: test_pipeline_v2/ADV-002
+datasource_url: https://example.com/advisory/2
+aliases:
+  - CVE-2025-0002
+summary: Test advisory2
+impacted_packages:
+  - purl: pkg:npm/foobar
+    affected_versions: vers:npm/>=1.2.4
+    fixed_versions: vers:npm/2.0.0
+severities: []
+weaknesses: []
+references: []