fix(export): exclude raw_html from JSON exports when include_raw_html is false

src/ciberwebscan/export/base.py

@@ -165,7 +165,10 @@ def _serialize_item(self, item: Any) -> dict[str, Any]:
         Serialize an item to a dictionary.
 
         Handles Pydantic models, dataclasses, and plain dicts.
-        Strips ``raw_html`` when *include_raw* is False.
+        When *include_raw* is False, strips ``raw_html`` from the top level
+        (e.g. a serialized ``ScrapeResult``) and from a nested ``scrape``
+        mapping (e.g. ``AnalysisReport``, where the field lives at
+        ``data["scrape"]["raw_html"]``). The source object is never mutated.
         """
         if hasattr(item, "model_dump"):
             # Pydantic v2
@@ -187,6 +190,13 @@ def _serialize_item(self, item: Any) -> dict[str, Any]:
 
         if not self.include_raw:
             data.pop("raw_html", None)
+            scrape = data.get("scrape")
+            # Copy before stripping: plain-dict input is only shallow-copied
+            # above, so popping in place would mutate the caller's mapping.
+            if isinstance(scrape, dict) and "raw_html" in scrape:
+                scrape = dict(scrape)
+                scrape.pop("raw_html", None)
+                data["scrape"] = scrape
 
         return data
 

src/ciberwebscan/export/json.py

@@ -276,17 +276,15 @@ def export_report(self, report: AnalysisReport) -> None:
             )
 
         try:
-            # Serialize the report
-            data = self._serialize_item(report)
-
-            # Remove raw data if not requested
-            if not self.include_raw and "scrape" in data and data["scrape"]:
-                data["scrape"].pop("raw_html", None)
-
-            # Calculate summary if not done
+            # Calculate summary before serializing so the single
+            # serialization below already contains the updated counters.
             if hasattr(report, "calculate_summary"):
                 report.calculate_summary()
-                data = self._serialize_item(report)
+
+            # Serialize once; _serialize_item strips raw_html when
+            # include_raw is False (no rebuild afterwards, or the
+            # sanitization would be lost).
+            data = self._serialize_item(report)
 
             # Write to stream or file
             if self._stream:

tests/unit/export/test_base.py

@@ -191,6 +191,115 @@ def test_serialize_unsupported_type_raises(self):
         with pytest.raises(ExportValidationError, match="Cannot serialize"):
             exporter._serialize_item("plain string")
 
+    def test_serialize_scrape_result_strips_top_level_raw_html(self):
+        """_serialize_item strips raw_html from a direct ScrapeResult."""
+        from ciberwebscan.export.models import ScrapeResult
+
+        exporter = ConcreteExporter(stream=StringIO(), include_raw=False)
+        scrape = ScrapeResult(
+            url="https://fake.test",
+            status_code=200,
+            raw_html="<html><body>FAKE_MARKER_HTML</body></html>",
+        )
+
+        result = exporter._serialize_item(scrape)
+
+        assert "raw_html" not in result
+        assert result["url"] == "https://fake.test"
+
+    def test_serialize_analysis_report_strips_nested_raw_html(self):
+        """_serialize_item strips scrape.raw_html from an AnalysisReport."""
+        from ciberwebscan.export.models import (
+            AnalysisReport,
+            ExportMeta,
+            ScrapeResult,
+        )
+
+        exporter = ConcreteExporter(stream=StringIO(), include_raw=False)
+        report = AnalysisReport(
+            meta=ExportMeta(target_url="https://fake.test"),
+            scrape=ScrapeResult(
+                url="https://fake.test",
+                status_code=200,
+                raw_html="<html><body>FAKE_MARKER_HTML</body></html>",
+            ),
+        )
+
+        result = exporter._serialize_item(report)
+
+        assert "raw_html" not in result["scrape"]
+        assert result["scrape"]["url"] == "https://fake.test"
+
+    def test_serialize_analysis_report_strips_empty_raw_html(self):
+        """Empty-string raw_html is also stripped when include_raw=False."""
+        from ciberwebscan.export.models import (
+            AnalysisReport,
+            ExportMeta,
+            ScrapeResult,
+        )
+
+        exporter = ConcreteExporter(stream=StringIO(), include_raw=False)
+        report = AnalysisReport(
+            meta=ExportMeta(target_url="https://fake.test"),
+            scrape=ScrapeResult(url="https://fake.test", status_code=200, raw_html=""),
+        )
+
+        result = exporter._serialize_item(report)
+
+        assert "raw_html" not in result["scrape"]
+
+    def test_serialize_keeps_raw_html_when_include_raw(self):
+        """include_raw=True preserves raw_html at both levels."""
+        from ciberwebscan.export.models import (
+            AnalysisReport,
+            ExportMeta,
+            ScrapeResult,
+        )
+
+        marker = "<html><body>FAKE_MARKER_HTML</body></html>"
+        exporter = ConcreteExporter(stream=StringIO(), include_raw=True)
+        report = AnalysisReport(
+            meta=ExportMeta(target_url="https://fake.test"),
+            scrape=ScrapeResult(
+                url="https://fake.test", status_code=200, raw_html=marker
+            ),
+        )
+
+        result = exporter._serialize_item(report)
+
+        assert result["scrape"]["raw_html"] == marker
+
+    def test_serialize_handles_missing_and_unexpected_scrape(self):
+        """Missing, null or non-mapping scrape must not raise."""
+        exporter = ConcreteExporter(stream=StringIO(), include_raw=False)
+
+        # scrape absent entirely
+        result = exporter._serialize_item({"meta": {"target_url": "x"}})
+        assert "scrape" not in result
+
+        # scrape null
+        result = exporter._serialize_item({"scrape": None})
+        assert result["scrape"] is None
+
+        # scrape with unexpected (non-mapping) structure
+        result = exporter._serialize_item({"scrape": "not-a-mapping"})
+        assert result["scrape"] == "not-a-mapping"
+
+    def test_serialize_does_not_mutate_plain_dict_input(self):
+        """Stripping nested raw_html must not alter the caller's dict."""
+        exporter = ConcreteExporter(stream=StringIO(), include_raw=False)
+        caller_dict = {
+            "scrape": {
+                "url": "https://fake.test",
+                "raw_html": "<html><body>FAKE_MARKER_HTML</body></html>",
+            }
+        }
+
+        result = exporter._serialize_item(caller_dict)
+
+        assert "raw_html" not in result["scrape"]
+        assert "raw_html" in caller_dict["scrape"]
+
     def test_format_datetime(self):
         """Test datetime formatting."""
         exporter = ConcreteExporter(stream=StringIO())

tests/unit/export/test_json.py

@@ -21,13 +21,23 @@
     dumps,
     export_to_json,
 )
+from ciberwebscan.export.models import (
+    AnalysisReport,
+    ExportMeta,
+    ScrapeResult,
+    Severity,
+    SSLFinding,
+    SSLResult,
+)
 
 pytestmark = pytest.mark.unit
 
 # =============================================================================
 # Test Fixtures
 # =============================================================================
 
+FAKE_MARKER_HTML = "<html><body>FAKE_MARKER_HTML</body></html>"
+
 
 class Color(Enum):
     RED = "red"
@@ -92,6 +102,39 @@ def mock_report():
     return report
 
 
+@pytest.fixture
+def report_with_raw_html() -> AnalysisReport:
+    """Real AnalysisReport with non-null scrape.raw_html."""
+    return AnalysisReport(
+        meta=ExportMeta(target_url="https://fake.test"),
+        scrape=ScrapeResult(
+            url="https://fake.test",
+            status_code=200,
+            content_type="text/html",
+            title="Fake",
+            raw_html=FAKE_MARKER_HTML,
+        ),
+    )
+
+
+@pytest.fixture
+def report_with_critical_finding() -> AnalysisReport:
+    """Real AnalysisReport with one critical SSL finding (summary unchecked)."""
+    return AnalysisReport(
+        meta=ExportMeta(target_url="https://fake.test"),
+        ssl=SSLResult(
+            is_https=True,
+            findings=[
+                SSLFinding(
+                    title="Weak protocol",
+                    description="TLS 1.0 enabled",
+                    severity=Severity.CRITICAL,
+                )
+            ],
+        ),
+    )
+
+
 # =============================================================================
 # JSON Serializer Tests
 # =============================================================================
@@ -376,6 +419,117 @@ def test_export_report_no_output_raises(self, mock_report):
             exporter.export_report(mock_report)
 
 
+class TestJSONExporterRawHtmlExclusion:
+    """Regression tests: raw_html must honour include_raw in export_report."""
+
+    def test_export_report_excludes_raw_html_by_default(
+        self, report_with_raw_html: AnalysisReport
+    ):
+        """include_raw=False must strip scrape.raw_html from the final JSON."""
+        stream = StringIO()
+
+        exporter = JSONExporter(stream=stream)
+        assert exporter.include_raw is False
+        exporter.export_report(report_with_raw_html)
+
+        raw_output = stream.getvalue()
+        content = json.loads(raw_output)
+        assert "raw_html" not in content["scrape"]
+        assert "FAKE_MARKER_HTML" not in raw_output
+
+    def test_export_report_includes_raw_html_when_enabled(
+        self, report_with_raw_html: AnalysisReport
+    ):
+        """include_raw=True must preserve the HTML exactly."""
+        stream = StringIO()
+
+        exporter = JSONExporter(stream=stream, include_raw=True)
+        exporter.export_report(report_with_raw_html)
+
+        content = json.loads(stream.getvalue())
+        assert content["scrape"]["raw_html"] == FAKE_MARKER_HTML
+
+    def test_export_report_calculates_summary(
+        self, report_with_critical_finding: AnalysisReport
+    ):
+        """Summary counters and risk_score must be updated in the output."""
+        stream = StringIO()
+
+        exporter = JSONExporter(stream=stream)
+        exporter.export_report(report_with_critical_finding)
+
+        content = json.loads(stream.getvalue())
+        assert content["critical_findings"] == 1
+        assert content["risk_score"] == 40
+        # Fields the summary did not touch stay at their defaults
+        assert content["high_findings"] == 0
+        assert content["info_findings"] == 0
+
+    def test_export_report_preserves_source_object(
+        self, report_with_raw_html: AnalysisReport
+    ):
+        """Sanitizing the output must not mutate the source report."""
+        stream = StringIO()
+
+        exporter = JSONExporter(stream=stream)
+        exporter.export_report(report_with_raw_html)
+
+        assert report_with_raw_html.scrape is not None
+        assert report_with_raw_html.scrape.raw_html == FAKE_MARKER_HTML
+
+    def test_export_report_null_raw_html_stays_omitted(self):
+        """raw_html=None keeps being omitted via exclude_none (unchanged)."""
+        report = AnalysisReport(
+            meta=ExportMeta(target_url="https://fake.test"),
+            scrape=ScrapeResult(
+                url="https://fake.test", status_code=200, raw_html=None
+            ),
+            fingerprint=None,
+        )
+        stream = StringIO()
+
+        exporter = JSONExporter(stream=stream)
+        exporter.export_report(report)
+
+        content = json.loads(stream.getvalue())
+        assert "raw_html" not in content["scrape"]
+        # exclude_none behaviour unchanged: null sections stay omitted
+        assert "fingerprint" not in content
+
+    def test_export_report_null_sections_present_when_include_raw(self):
+        """include_raw=True keeps exclude_none disabled (unchanged)."""
+        report = AnalysisReport(
+            meta=ExportMeta(target_url="https://fake.test"),
+            scrape=ScrapeResult(
+                url="https://fake.test", status_code=200, raw_html=None
+            ),
+            fingerprint=None,
+        )
+        stream = StringIO()
+
+        exporter = JSONExporter(stream=stream, include_raw=True)
+        exporter.export_report(report)
+
+        content = json.loads(stream.getvalue())
+        assert "raw_html" in content["scrape"]
+        assert content["scrape"]["raw_html"] is None
+        assert "fingerprint" in content
+
+    def test_export_report_empty_string_raw_html_excluded(self):
+        """raw_html="" is not None, so only the explicit strip can remove it."""
+        report = AnalysisReport(
+            meta=ExportMeta(target_url="https://fake.test"),
+            scrape=ScrapeResult(url="https://fake.test", status_code=200, raw_html=""),
+        )
+        stream = StringIO()
+
+        exporter = JSONExporter(stream=stream)
+        exporter.export_report(report)
+
+        content = json.loads(stream.getvalue())
+        assert "raw_html" not in content["scrape"]
+
+
 class TestExportToJson:
     """Tests for export_to_json convenience function."""