diff --git a/README.md b/README.md index 2d4d64c1..b5ee4e0b 100644 --- a/README.md +++ b/README.md @@ -22,7 +22,7 @@ SkillSpector is part of the [NVIDIA Verified Skills pipeline](https://docs.nvidi ## Features - **Multi-format input**: Scan Git repos, URLs, zip files, directories, or single files -- **68 vulnerability patterns** across 17 categories: prompt injection, data exfiltration, privilege escalation, supply chain, excessive agency, output handling, system prompt leakage, memory poisoning, tool misuse, rogue agent, anti-refusal, trigger abuse, dangerous code (AST), taint tracking, YARA signatures, MCP least privilege, and MCP tool poisoning +- **69 vulnerability patterns** across 17 categories: prompt injection, data exfiltration, privilege escalation, supply chain, excessive agency, output handling, system prompt leakage, memory poisoning, tool misuse, rogue agent, anti-refusal, trigger abuse, dangerous code (AST), taint tracking, YARA signatures, MCP least privilege, and MCP tool poisoning - **Two-stage analysis**: Fast static analysis + optional LLM semantic evaluation - **Live vulnerability lookups**: SC4 queries [OSV.dev](https://osv.dev) for real-time CVE data with automatic offline fallback - **Multiple output formats**: Terminal, JSON, Markdown, and SARIF reports @@ -352,7 +352,7 @@ claude mcp add skillspector -- skillspector mcp ## Vulnerability Patterns -SkillSpector detects **68 vulnerability patterns** across 17 categories: +SkillSpector detects **69 vulnerability patterns** across 17 categories: ### Prompt Injection (5 patterns) @@ -389,7 +389,7 @@ SkillSpector detects **68 vulnerability patterns** across 17 categories: | PE2 | Sudo/Root Execution | MEDIUM | Invoking elevated system privileges | | PE3 | Credential Access | HIGH | Reading SSH keys, tokens, passwords | -### Supply Chain (6 patterns) +### Supply Chain (7+ patterns) | ID | Pattern | Severity | Description | |----|---------|----------|-------------| @@ -399,6 +399,7 @@ SkillSpector detects **68 vulnerability patterns** across 17 categories: | SC4 | Known Vulnerable Dependencies | HIGH | Dependencies with known CVEs (live OSV.dev lookup) | | SC5 | Abandoned Dependencies | MEDIUM | Unmaintained packages without security updates | | SC6 | Typosquatting | HIGH | Package names similar to popular packages | +| SC8 | Shipped Python Bytecode | HIGH | `__pycache__` / `.pyc` present (discovery skips; malicious bytecode bypass) | ### Excessive Agency (4 patterns) diff --git a/src/skillspector/nodes/analyzers/pattern_defaults.py b/src/skillspector/nodes/analyzers/pattern_defaults.py index bb0a7f2b..ebafcd01 100644 --- a/src/skillspector/nodes/analyzers/pattern_defaults.py +++ b/src/skillspector/nodes/analyzers/pattern_defaults.py @@ -91,6 +91,7 @@ class PatternCategory(StrEnum): "SC5": "Dependency appears abandoned or unmaintained. Abandoned packages no longer receive security patches, leaving known and future vulnerabilities unaddressed.", "SC6": "Package name closely resembles a popular package, suggesting possible typosquatting. Attackers publish malicious packages with similar names to trick developers into installing them.", "SC7": "Code pulls a container image with signature or registry verification disabled (--disable-content-trust, DOCKER_CONTENT_TRUST=0, --insecure-registry). This accepts tampered or unverified images and is a container supply-chain risk.", + "SC8": "Skill ships Python bytecode (__pycache__/ or .pyc/.pyo). Discovery skips these paths, so malicious bytecode can score SAFE while decoy sources look clean.", # Trigger Abuse "TR1": "Skill uses overly broad trigger patterns that match common words or phrases, causing it to activate in unintended contexts and potentially shadow other skills.", "TR2": "Skill trigger shadows a common built-in command or another skill's trigger, potentially intercepting requests meant for trusted functionality.", @@ -181,6 +182,7 @@ class PatternCategory(StrEnum): "SC5": PatternCategory.SUPPLY_CHAIN.value, "SC6": PatternCategory.SUPPLY_CHAIN.value, "SC7": PatternCategory.SUPPLY_CHAIN.value, + "SC8": PatternCategory.SUPPLY_CHAIN.value, "TR1": PatternCategory.TRIGGER_ABUSE.value, "TR2": PatternCategory.TRIGGER_ABUSE.value, "TR3": PatternCategory.TRIGGER_ABUSE.value, @@ -259,6 +261,7 @@ class PatternCategory(StrEnum): "SC5": "Abandoned Dependency", "SC6": "Typosquatting Dependency", "SC7": "Untrusted Container Image", + "SC8": "Shipped Python Bytecode", "TR1": "Overly Broad Trigger", "TR2": "Shadow Command Trigger", "TR3": "Keyword Baiting Trigger", @@ -344,6 +347,7 @@ class PatternCategory(StrEnum): "SC5": "Replace the abandoned dependency with an actively maintained alternative. Check the package's repository for last commit date and open issues.", "SC6": "Verify the package name is correct and not a typosquatting variant. Compare against the official package name on PyPI or npm.", "SC7": "Keep image signature verification (Docker Content Trust / cosign) and registry TLS enabled. Pull only signed images from trusted registries; never disable content-trust or use insecure registries in skill code.", + "SC8": "Do not ship __pycache__/ or .pyc/.pyo in skills. Delete bytecode before packaging; if presence is intentional for a lab fixture, quarantine it outside the skill install path.", # Trigger Abuse "TR1": "Use specific, narrow trigger patterns that match only the skill's intended use case. Avoid single-word or common-phrase triggers.", "TR2": "Choose triggers that do not conflict with built-in commands or other skills. Prefix with a unique namespace if necessary.", diff --git a/src/skillspector/nodes/analyzers/static_patterns_supply_chain.py b/src/skillspector/nodes/analyzers/static_patterns_supply_chain.py index f781d7e5..63a6b55e 100644 --- a/src/skillspector/nodes/analyzers/static_patterns_supply_chain.py +++ b/src/skillspector/nodes/analyzers/static_patterns_supply_chain.py @@ -13,13 +13,14 @@ # See the License for the specific language governing permissions and # limitations under the License. -"""Static patterns: supply chain (SC1–SC7) and trigger analysis (TR1–TR3). +"""Static patterns: supply chain (SC1–SC8) and trigger analysis (TR1–TR3). SC1–SC3: regex-based pattern matching (original implementation). SC4: Known vulnerable dependencies — live OSV.dev lookup with static fallback. SC5: Abandoned dependencies — flags known-abandoned or archived packages. SC6: Typosquatting — flags package names similar to popular packages. SC7: Untrusted container image — flags image signature / registry-verification bypass. +SC8: Shipped Python bytecode — flags __pycache__/ and *.pyc/*.pyo that discovery skips. TR1–TR3: Trigger analysis — flags overly broad, shadowing, or baiting triggers. Node and analyze() in one module. @@ -27,9 +28,11 @@ from __future__ import annotations +import os import re import sys import tomllib +from pathlib import Path from urllib.parse import urlparse from packaging.requirements import InvalidRequirement, Requirement @@ -1185,13 +1188,85 @@ def _analyze_triggers(manifest: dict[str, object], skill_path: str) -> list[Find return findings +# --------------------------------------------------------------------------- +# SC8: Shipped Python bytecode (closes silent __pycache__ / .pyc skip) +# --------------------------------------------------------------------------- + +# Still skip heavy/vendor trees for SC8, but *do* descend into __pycache__. +_SC8_SKIP_DIRS = frozenset({".git", "node_modules", ".venv", "venv", ".tox", ".pytest_cache"}) +_SC8_BYTECODE_SUFFIXES = (".pyc", ".pyo") + + +def _analyze_shipped_bytecode(skill_path: str) -> list[Finding]: + """Emit SC8 when a skill ships __pycache__ dirs or .pyc/.pyo files. + + ``build_context`` excludes ``__pycache__`` from inventory and + ``static_runner`` treats ``.pyc`` as binary, so malicious bytecode can + otherwise score SAFE. Presence alone is a HIGH supply-chain signal; + full disassembly can come later. + """ + findings: list[Finding] = [] + if not skill_path or not isinstance(skill_path, str): + return findings + root = Path(skill_path) + if not root.is_dir(): + return findings + + for dirpath, dirnames, filenames in os.walk(root): + dirnames[:] = sorted(name for name in dirnames if name not in _SC8_SKIP_DIRS) + rel_dir = Path(dirpath).relative_to(root).as_posix() + if rel_dir == ".": + rel_dir = "" + + for dirname in list(dirnames): + if dirname != "__pycache__": + continue + rel = f"{rel_dir}/{dirname}/" if rel_dir else f"{dirname}/" + af = AnalyzerFinding( + rule_id="SC8", + message="Skill ships a __pycache__ directory that normal discovery skips", + severity=Severity.HIGH, + location=Location(file=rel, start_line=1), + confidence=0.95, + tags=[PatternCategory.SUPPLY_CHAIN.value], + matched_text=rel, + context=( + "Python may load .pyc from this directory even when decoy " + ".py sources look clean (PEP 552 UNCHECKED_HASH)." + ), + ) + findings.append(analyzer_finding_to_finding(af)) + + for filename in sorted(filenames): + lower = filename.lower() + if not lower.endswith(_SC8_BYTECODE_SUFFIXES): + continue + rel = f"{rel_dir}/{filename}" if rel_dir else filename + af = AnalyzerFinding( + rule_id="SC8", + message="Skill ships Python bytecode (.pyc/.pyo) that normal analysis skips", + severity=Severity.HIGH, + location=Location(file=rel, start_line=1), + confidence=0.95, + tags=[PatternCategory.SUPPLY_CHAIN.value], + matched_text=filename, + context=( + "Bytecode is excluded from content analysis; a malicious " + ".pyc can execute while source decoys remain clean." + ), + ) + findings.append(analyzer_finding_to_finding(af)) + + return findings + + # --------------------------------------------------------------------------- # Graph node # --------------------------------------------------------------------------- def node(state: SkillspectorState) -> AnalyzerNodeResponse: - """Run supply_chain patterns (SC1–SC6) and trigger analysis (TR1–TR3).""" + """Run supply_chain patterns (SC1–SC8) and trigger analysis (TR1–TR3).""" # SC1–SC3 via static_runner response = static_runner.run_static_patterns_with_ledger(state, [sys.modules[__name__]]) findings = response["findings"] @@ -1201,7 +1276,7 @@ def record_extra_findings( extra_findings: list[Finding], fallback_analyzer_id: str, ) -> None: - """Attach dependency/manifest findings to the matching completed work item.""" + """Attach supplemental findings to the matching completed work item.""" if not extra_findings: return finding_ids = [finding.finding_id for finding in extra_findings] @@ -1263,6 +1338,22 @@ def record_extra_findings( f"{ANALYZER_ID}_triggers", ) + # SC8: shipped bytecode / __pycache__ (discovery otherwise skips these) + skill_path = state.get("skill_path") or "" + if isinstance(skill_path, str) and skill_path.strip(): + bytecode_findings = _analyze_shipped_bytecode(skill_path) + findings.extend(bytecode_findings) + for finding_path in sorted({finding.file.rstrip("/") for finding in bytecode_findings}): + record_extra_findings( + finding_path, + [ + finding + for finding in bytecode_findings + if finding.file.rstrip("/") == finding_path + ], + f"{ANALYZER_ID}_bytecode", + ) + logger.info("%s: %d findings", ANALYZER_ID, len(findings)) response["analyzer_status_events"] = [ analyzer_status_for_events(ANALYZER_ID, response["inspection_ledger"]) diff --git a/src/skillspector/nodes/report.py b/src/skillspector/nodes/report.py index fb7dc552..68c0760a 100644 --- a/src/skillspector/nodes/report.py +++ b/src/skillspector/nodes/report.py @@ -156,6 +156,11 @@ def _severity_to_sarif_level(severity: str) -> Literal["error", "warning", "note _MAX_OCCURRENCES_PER_RULE = 3 _DIMINISHING_WEIGHTS = (1.0, 0.5, 0.25) +# Some findings describe artifacts whose unanalyzed contents can execute. Their +# presence must block installation even when ordinary confidence-weighted, +# per-rule scoring would otherwise keep the aggregate below the CLI threshold. +_RISK_SCORE_FLOORS_BY_RULE_ID = {"SC8": 51} + def _compute_risk_score( findings: list[Finding], @@ -220,7 +225,15 @@ def _compute_risk_score( score += contribution - final_score = min(100, max(0, int(score))) + score_floor = max( + ( + _RISK_SCORE_FLOORS_BY_RULE_ID.get(f.rule_id, 0) + for f in sorted_findings + if max(0.0, min(1.0, f.confidence)) > 0.0 + ), + default=0, + ) + final_score = min(100, max(score_floor, int(score))) severity_band = "LOW" for threshold, band in _RISK_SEVERITY_BANDS: diff --git a/tests/nodes/analyzers/test_sc8_shipped_bytecode.py b/tests/nodes/analyzers/test_sc8_shipped_bytecode.py new file mode 100644 index 00000000..aa45f2db --- /dev/null +++ b/tests/nodes/analyzers/test_sc8_shipped_bytecode.py @@ -0,0 +1,66 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import json +from pathlib import Path + +from typer.testing import CliRunner + +from skillspector.cli import app +from skillspector.nodes.analyzers import static_patterns_supply_chain as supply_chain + + +def test_sc8_flags_pycache_and_pyc(tmp_path: Path) -> None: + cache = tmp_path / "scripts" / "__pycache__" + cache.mkdir(parents=True) + (cache / "evil.cpython-312.pyc").write_bytes(b"\x00") + (tmp_path / "orphan.pyc").write_bytes(b"\x00") + (tmp_path / "clean.py").write_text("print('ok')\n", encoding="utf-8") + + findings = supply_chain._analyze_shipped_bytecode(str(tmp_path)) + rule_ids = {f.rule_id for f in findings} + assert rule_ids == {"SC8"} + paths = {f.file for f in findings} + assert "scripts/__pycache__/" in paths + assert "scripts/__pycache__/evil.cpython-312.pyc" in paths + assert "orphan.pyc" in paths + assert all(f.severity == "HIGH" for f in findings) + + +def test_sc8_clean_tree_has_no_findings(tmp_path: Path) -> None: + (tmp_path / "SKILL.md").write_text("# demo\n", encoding="utf-8") + (tmp_path / "main.py").write_text("x = 1\n", encoding="utf-8") + assert supply_chain._analyze_shipped_bytecode(str(tmp_path)) == [] + + +def test_sc8_single_pyc_blocks_install_and_cli_exit(tmp_path: Path) -> None: + (tmp_path / "SKILL.md").write_text( + "---\nname: shipped-bytecode\n---\n# Shipped bytecode\n", encoding="utf-8" + ) + (tmp_path / "payload.pyc").write_bytes(b"\x00") + + result = CliRunner().invoke( + app, + ["scan", str(tmp_path), "--format", "json", "--no-llm"], + ) + + assert result.exit_code == 1, result.output + report = json.loads(result.output) + assert report["risk_assessment"] == { + "score": 51, + "severity": "HIGH", + "recommendation": "DO_NOT_INSTALL", + } + assert any(issue["id"] == "SC8" for issue in report["issues"]) diff --git a/tests/nodes/test_report.py b/tests/nodes/test_report.py index 74dff645..2f40e7d7 100644 --- a/tests/nodes/test_report.py +++ b/tests/nodes/test_report.py @@ -84,6 +84,13 @@ def test_single_finding_partial_confidence_scales_score(self) -> None: score, _, _ = _compute_risk_score(findings, False) assert score == 12 # 25 * 1.0 * 0.5 = 12.5 -> int(12.5) = 12 + def test_shipped_bytecode_enforces_blocking_risk_floor(self) -> None: + findings = [_finding("SC8", "HIGH", confidence=0.95, file="payload.pyc")] + score, band, recommendation = _compute_risk_score(findings, False) + assert score == 51 + assert band == "HIGH" + assert recommendation == "DO_NOT_INSTALL" + def test_unknown_severity_defaults_to_low_points(self) -> None: f = _finding("R1", "LOW") f.severity = ""