Skip to content

Commit 8c369e8

Browse files
committed
Report full-scan findings as repository findings, not new ones
A run with no baseline creates a full scan and suppresses blocking, because there is nothing to compare against and so nothing can be attributed to the change. When an alert-bearing output format is enabled the scan still carries every finding in the repository, and the console summary labeled those `NEW` and their link `Diff Url`. Both are wrong for a full scan, and the first contradicts the exit code: the summary reported blocking issues while the run exited 0, which reads as gating that silently failed rather than gating that correctly did not apply. Label the counts and the link by what the run produced, and say why the counts do not gate. The alert list itself is left alone, since SARIF and JSON output read it and renaming their fields would break consumers.
1 parent 21d6912 commit 8c369e8

2 files changed

Lines changed: 92 additions & 7 deletions

File tree

‎socketsecurity/output.py‎

Lines changed: 18 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -229,10 +229,8 @@ def save_sbom_file(self, diff_report: Diff, sbom_file_name: Optional[str] = None
229229

230230
def build_summary_text(self, diff_report: Diff) -> str:
231231
"""Render the console summary text for stdout and file output."""
232-
if (
233-
getattr(diff_report, "is_full_scan", False)
234-
and not getattr(diff_report, "alerts_fetched", False)
235-
):
232+
is_full_scan = getattr(diff_report, "is_full_scan", False)
233+
if is_full_scan and not getattr(diff_report, "alerts_fetched", False):
236234
lines = ["Full scan completed. Findings were not fetched for console output."]
237235
report_link = getattr(diff_report, "report_url", "") or getattr(
238236
diff_report, "diff_url", ""
@@ -264,20 +262,33 @@ def build_summary_text(self, diff_report: Diff) -> str:
264262
selected_diff = clone_diff_with_selected_alerts(diff_report, selected_alerts)
265263
console_security_comment = Messages.create_console_security_alert_table(selected_diff)
266264

265+
# A full scan has no baseline, so everything it carries is a finding in the
266+
# repository rather than one the change introduced. Calling those NEW would
267+
# also contradict the exit code, which stays 0 because no finding can be
268+
# attributed to the change.
269+
blocking_label = "Blocking issues" if is_full_scan else "NEW blocking issues"
270+
warning_label = "Warning issues" if is_full_scan else "NEW warning issues"
271+
link_label = "Report Url" if is_full_scan else "Diff Url"
272+
267273
lines = ["Security issues detected by Socket Security:"]
268274
if new_blocking > 0:
269-
lines.append(f" - NEW blocking issues: {new_blocking}")
275+
lines.append(f" - {blocking_label}: {new_blocking}")
270276
if new_warning > 0:
271-
lines.append(f" - NEW warning issues: {new_warning}")
277+
lines.append(f" - {warning_label}: {new_warning}")
272278
if unchanged_blocking > 0:
273279
lines.append(
274280
f" - EXISTING blocking issues: {unchanged_blocking} (causing failure due to --strict-blocking)"
275281
)
276282
if unchanged_warning > 0:
277283
lines.append(f" - EXISTING warning issues: {unchanged_warning}")
284+
if is_full_scan:
285+
lines.append(
286+
" Reported against the whole repository, with no baseline to compare "
287+
"against, so these do not affect the exit code."
288+
)
278289

279290
report_link = getattr(diff_report, "report_url", "") or getattr(diff_report, "diff_url", "")
280-
lines.append(f"Diff Url: {report_link}")
291+
lines.append(f"{link_label}: {report_link}")
281292
lines.append("")
282293
lines.append(str(console_security_comment))
283294
return "\n".join(lines)

‎tests/unit/test_summary_text.py‎

Lines changed: 74 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,74 @@
1+
"""How the console summary distinguishes a comparison from a full scan.
2+
3+
A full scan carries every finding in the repository, not the ones a change
4+
introduced, and blocking is suppressed for exactly that reason. The summary has
5+
to say so, or the reported counts read as gating failures that returned 0.
6+
"""
7+
from unittest.mock import MagicMock
8+
9+
from socketsecurity.config import CliConfig
10+
from socketsecurity.core.classes import Diff, Issue
11+
from socketsecurity.output import OutputHandler
12+
13+
14+
def _issue(name: str, error: bool = False, warn: bool = False) -> Issue:
15+
return Issue(
16+
pkg_name=name,
17+
pkg_version="1.0.0",
18+
severity="high",
19+
title=f"Vuln in {name}",
20+
description="test",
21+
type="vulnerability",
22+
manifests="pom.xml",
23+
pkg_type="maven",
24+
key=f"key-{name}",
25+
purl=f"pkg:maven/{name}@1.0.0",
26+
url=f"https://socket.dev/maven/package/{name}/overview/1.0.0",
27+
error=error,
28+
warn=warn,
29+
)
30+
31+
32+
def _handler() -> OutputHandler:
33+
return OutputHandler(CliConfig.from_args(["--api-token", "test"]), MagicMock())
34+
35+
36+
def _diff(is_full_scan: bool) -> Diff:
37+
diff = Diff()
38+
diff.new_alerts = [_issue("alpha", error=True), _issue("beta", warn=True)]
39+
diff.id = "scan-id"
40+
diff.report_url = "https://socket.dev/dashboard/org/test/sbom/scan-id"
41+
diff.diff_url = diff.report_url
42+
diff.is_full_scan = is_full_scan
43+
diff.alerts_fetched = is_full_scan
44+
return diff
45+
46+
47+
def test_comparison_reports_findings_as_new():
48+
summary = _handler().build_summary_text(_diff(is_full_scan=False))
49+
50+
assert "NEW blocking issues: 1" in summary
51+
assert "NEW warning issues: 1" in summary
52+
assert "Diff Url:" in summary
53+
54+
55+
def test_full_scan_does_not_report_findings_as_new():
56+
summary = _handler().build_summary_text(_diff(is_full_scan=True))
57+
58+
assert "NEW" not in summary
59+
assert "Blocking issues: 1" in summary
60+
assert "Warning issues: 1" in summary
61+
62+
63+
def test_full_scan_labels_its_link_as_a_report():
64+
summary = _handler().build_summary_text(_diff(is_full_scan=True))
65+
66+
assert "Report Url:" in summary
67+
assert "Diff Url:" not in summary
68+
69+
70+
def test_full_scan_explains_why_the_counts_do_not_gate():
71+
summary = _handler().build_summary_text(_diff(is_full_scan=True))
72+
73+
assert "no baseline to compare against" in summary
74+
assert "do not affect the exit code" in summary

0 commit comments

Comments
 (0)