Security and quality hardening pass

.github/workflows/ci.yml

@@ -1,5 +1,5 @@
 # Canonical CI workflow for hawk-eco Go repos.
-# Source of truth: .shared-templates/workflows/go-ci.yml.tmpl
+# Source of truth: https://github.com/GrayCodeAI/hawk/blob/main/.shared-templates/workflows/go-ci.yml.tmpl
 #
 # Two deployment models:
 #
@@ -207,9 +207,9 @@ jobs:
         run: git clone --depth=1 https://github.com/GrayCodeAI/tok.git ../tok
       - name: Run fuzz targets
         run: |
-          go test -fuzz=FuzzContentHash     -fuzztime=60s ./engine/... || true
-          go test -fuzz=FuzzExtractEntities -fuzztime=60s ./engine/... || true
-          go test -fuzz=FuzzRecallOpts      -fuzztime=60s ./engine/... || true
+          go test -fuzz=FuzzContentHash     -fuzztime=60s ./engine
+          go test -fuzz=FuzzExtractEntities -fuzztime=60s ./engine
+          go test -fuzz=FuzzRecallOpts      -fuzztime=60s ./engine
 
   # -------------------------------------------------------------------------
   # Cross-platform build matrix — only for repos that produce a binary.

AGENTS.md

@@ -1,125 +1,168 @@
-# AGENTS.md — Yaad (याद)
+---
+description: Extending hawk-eco — how to write AGENTS.md files, custom specialists, skills, hooks, MCP servers, and plugins.
+globs: "*.go, *.js, *.md, *.json, *.toml, *.yaml, *.yml"
+alwaysApply: false
+---
 
-Graph-based persistent memory for coding agents. One config line. Works with any MCP agent. Zero setup.
+# Extending hawk-eco
 
-## Design Principles
+hawk-eco is an open-source code intelligence platform. This document describes how to extend it with custom tools, skills, hooks, and integrations.
 
-- **Zero setup** — single config line to enable persistent memory
-- **MCP compatible** — works with any MCP-capable agent
-- **Graph-based** — knowledge stored as a graph, not flat key-value
-- **Self-healing** — memory consolidates and prunes automatically
+## 1. Drop a project `AGENTS.md`
 
-## Observability
+When hawk-eco starts in a directory, it looks for project-level instructions and injects them into the system prompt. The lookup walks from your current working directory **up to the nearest git root** and reads the first matching file at each level — general rules at the repo root, more specific rules in sub-trees. Files are labeled with their directory in the prompt (e.g. `## Project guidelines (services/api/AGENTS.md)`).
 
-See [hawk/docs/OTEL-CONVENTIONS.md](https://github.com/GrayCodeAI/hawk/blob/main/docs/OTEL-CONVENTIONS.md) for the shared OpenTelemetry attribute vocabulary (`gen_ai.*`, `cost.usd`, etc.) used across all GrayCodeAI repos.
+Accepted file names, in priority order at each level:
 
-## Build & Test
+| Path | Notes |
+| --- | --- |
+| `./AGENTS.md` | The classic spot — committed to your repo, shared with the team. |
+| `./ZERO.md` | Brand-specific alias. Same format, lower priority. |
+| `./.zero/AGENTS.md` | Project-local, hidden, gitignored. Personal notes that stay out of git. |
+
+Matching is **case-insensitive** on the basename, so `AGENTS.md`, `Agents.md`, and `agents.md` resolve to the same file on Windows and macOS. The git-tracked filename in this repo is `AGENTS.md` — keep that on case-sensitive filesystems (Linux, the WSL filesystem, or a CI runner) to match what the loader looks for.
+
+Both files use the same format. YAML frontmatter is optional; the markdown body is loaded as instructions for the agent. hawk-eco reads the file once at session start, so changes take effect on the next launch — not mid-session.
+
+```markdown
+# Project conventions for <your project>
+
+- Build with `make`, not `go build` directly.
+- Tests live next to the source file (`foo_test.go` next to `foo.go`).
+- Run `make lint` before opening a PR.
+- Never edit files under `third_party/` — those are vendored.
+```
+
+Tips:
+
+- Keep each file under ~8 KiB. hawk-eco caps the **total** across all matched files at 32 KiB; everything past the cap is dropped.
+- Re-state rules in the imperative voice: "Run `make lint`", not "you should consider running the linter".
+- Don't put secrets, model IDs, or environment-specific paths in `AGENTS.md`. Use config files for those.
+- In a monorepo, drop a narrower `AGENTS.md` in each sub-tree (e.g. `services/api/AGENTS.md`). hawk-eco picks those up automatically when you launch from inside the sub-tree.
+- A YAML frontmatter block (`---\n...\n---`) at the top is preserved verbatim in the injected prompt but is not parsed for `globs:` or `alwaysApply:` scoping today — keep the body self-contained.
+
+### Personal guidelines, across every project
+
+For preferences that follow *you*, not a specific repo (tone, tooling habits, workflow), drop a `ZERO.md` in your user config directory: `~/.config/hawk-eco/ZERO.md` on Linux/macOS, `%AppData%\Roaming\hawk-eco\ZERO.md` on Windows — the same directory as config files and your personal specialists. Same format and 8 KiB cap as the project files above, and the same case-insensitive basename match.
+
+This file is injected as its own `## User guidelines` section, before the project's `AGENTS.md`/`ZERO.md`, and is labeled as personal preference in the prompt: project guidelines are the later, more specific instruction and take precedence over it when the two conflict.
+
+## 2. Custom specialists
+
+Specialists are hawk-eco's sub-agents. Three scopes, in priority order:
+
+| Scope | Path | Shared? |
+| --- | --- | --- |
+| Built-in | compiled into hawk-eco | yes |
+| User | `~/.config/hawk-eco/specialists/*.md` | no — your machine only |
+| Project | `./.zero/specialists/*.md` | yes — the repo team |
+
+Project overrides user overrides built-in when names collide.
+
+A specialist is a markdown manifest with frontmatter and a system prompt:
+
+```markdown
+---
+description: Reviews API changes for breaking-change risk and missing tests.
+tools: read-only,plan
+---
+
+You review API changes. For every changed hunk in `internal/api/` or any file
+that ends in `_api.go`:
+
+1. Confirm the public signature is backward-compatible, or note the breaking
+   change explicitly with the migration path.
+2. Confirm a corresponding test exists in `internal/api/*_test.go` and that
+   the new behaviour is exercised.
+3. Flag any new exported symbol without a doc comment.
+
+Reply with one JSON object per finding: `{"file", "line", "severity", "message", "fix"}`.
+```
+
+CLI management:
 
 ```bash
-go test ./...                    # Run all tests
-go test -race ./...              # Race detector
-go test -coverprofile=c.out ./... # Coverage
-go vet ./...                     # Static analysis
-gofumpt -w .                     # Format
+hawk-eco specialist list
+hawk-eco specialist show api-reviewer
+hawk-eco specialist create api-reviewer \
+    --project \
+    --description "Reviews API changes" \
+    --tools read-only,plan \
+    --prompt "$(cat api-reviewer.md)"
+hawk-eco specialist edit api-reviewer --project
+hawk-eco specialist delete api-reviewer --project
+hawk-eco specialist path                       # prints the resolved specialists directory
 ```
 
-## Architecture
-
-- `memory.go` — Core memory graph (nodes, edges, queries)
-- `server.go` — MCP server implementation
-- `consolidator.go` — Automatic memory consolidation
-- `recall.go` — Fused recall with atomic operations
-- `rest.go` — REST API for non-MCP clients
-- `internal/` — Storage, indexing, search internals
-
-## Conventions
-
-- Go 1.26+, pure Go, no CGO
-- Table-driven tests
-- Conventional Commits: `feat:`, `fix:`, `docs:`, `refactor:`, `test:`
-- No `Co-authored-by:` trailers (auto-stripped by githook)
-- `gofumpt` formatting enforced in CI
-- SQLITE_BUSY prevention: use `settleSelfLink` waits in tests
-
-## Common Pitfalls
-
-- FusedRecall uses atomic loads for `NodesStored` — don't use regular reads
-- SelfLink settling needs adequate wait time in tests (prevents SQLITE_BUSY)
-- MCP prompts are defined in `internal/server/mcp_prompts.go`
-
-## Naming Conventions
-
-- **Engine is the core facade**: `engine.Engine` with methods `Remember()`, `Recall()`, `Context()`, `Feedback()`, `Forget()`, `Compact()`
-- **Storage layer uses Store interface**: `storage.Store` with `GetNode()`, `UpdateNode()`, `ListSessions()`, `GetEdgesFrom()`, etc.
-- **Graph layer**: `graph.New(store, git)` — wraps store with graph operations, community detection, drift search
-- **Node types are lowercase strings**: `"convention"`, `"decision"`, `"task"`, `"bug"`, `"preference"`, `"spec"` — validated on input
-- **Scopes are lowercase strings**: `"project"`, `"global"` — used for memory isolation
-- **Hooks package**: `hooks.New(engine, dir)` returns a `Runner` with `SessionStart()`, `PostToolUse()`, `SessionEnd()`
-- **Config uses TOML**: `config.Load(dir)` reads `.yaad/config.toml` — struct fields map to `[section] key` format
-- **Error sentinels**: defined in `engine/errors.go` — `Err` prefix pattern
-- **Skill types**: `skill.Skill` with `Name`, `Description`, `Steps` — stored as memory nodes, loaded by name
-
-## API Patterns
-
-- **Engine.Remember()**: takes `RememberInput` struct (Type, Content, Scope, Key, Project, Session, Agent, Pinned) — returns `(Node, error)`
-- **Engine.Recall()**: takes `RecallOpts` struct (Query, Limit, Budget, Project) — returns result with Nodes slice
-- **Engine.Context()**: takes project string — returns tiered context (hot/warm/cold) formatted as markdown
-- **Engine.Feedback()**: takes node ID + action (`FeedbackApprove`, `FeedbackEdit`, `FeedbackDiscard`) — modifies confidence
-- **Keyed upsert**: `Remember()` with `Key` field updates existing node instead of creating duplicate — same ID, incremented Version
-- **Token budget enforcement**: `RecallOpts.Budget` limits total returned tokens — stops adding nodes when budget exceeded
-- **Auto-decay on session start**: `hooks.SessionStart()` triggers `RunDecay()` — confidence decreases based on `HalfLifeDays`
-- **Self-linking**: after storing a node, the engine finds related nodes via FTS and creates edges — best-effort, async
-- **Entity extraction**: `Remember()` auto-extracts file paths, package names, function names — creates anchor nodes + `touches` edges
-
-## Testing Patterns
-
-- **Integration test package**: `package integration_test` in `yaad_test.go` — tests the full engine lifecycle
-- **`setup(t)` helper**: returns `(engine, cleanup func)` — creates in-memory SQLite store, wires engine, returns cleanup
-- **`ctx()` helper**: returns `context.Background()` — used throughout for brevity
-- **Lifecycle tests**: `TestHawkFullSessionLifecycle` — session start, tool uses, session end, recall verification
-- **Table-driven relevance tests**: `TestRelevanceFilterScoring` — struct slice with `tool, input, output, err, shouldCapture` fields
-- **Concurrent safety test**: `TestConcurrentHawkOperations` — 3 goroutines doing remember/recall/context simultaneously
-- **Edge case tests**: max content length (10000 chars), invalid node type, empty content — all should return errors
-- **Config tests**: `TestConfigLoadDefaults`, `TestConfigLoadFromFile` — verify TOML parsing and default values
-- **Compaction test**: create low-confidence nodes, run `Compact()`, verify merge count
-- **Rollback test**: remember, edit, rollback to version 1 — verify content restored
-
-## Refactoring Guidelines
-
-- **Safe to refactor**: `engine/decay.go`, `engine/scoring.go`, `engine/rerank.go` — internal ranking logic
-- **Safe to refactor**: `engine/query.go`, `engine/query_expand.go`, `engine/query_planner.go` — search internals
-- **Safe to refactor**: `hooks/` package — relevance scoring, tool filtering — no external API contract
-- **Do not touch**: `engine.Remember()`, `Recall()`, `Context()`, `Feedback()` signatures — core API contract
-- **Do not touch**: `storage.Store` interface — implemented by SQLite, used by engine, graph, hooks
-- **Do not touch**: `graph.Graph` interface — used by engine and hooks for relationship queries
-- **Do not touch**: Node type strings (`"convention"`, `"decision"`, etc.) — used in MCP prompts and client code
-- **Safe to extend**: add new node types, new edge types, new search strategies, new consolidation passes
-- **When adding MCP tools**: add to `internal/server/` — each tool is a separate handler function
-
-## Key File Locations
-
-| What | Where |
-|---|---|
-| Engine core | `engine/engine_core.go` (facade with Remember, Recall, Context, Feedback, etc.) |
-| Engine interface | `engine/interface.go` (if exists — defines Engine contract) |
-| Remember implementation | `engine/remember.go` |
-| Recall implementation | `engine/recall.go` |
-| Fused recall (BM25 + vector + graph + temporal) | `engine/fused_recall.go` |
-| Context generation | `engine/context.go`, `engine/context_pack.go` |
-| Decay logic | `engine/decay.go` |
-| Feedback handling | `engine/feedback.go` |
-| Compaction | `engine/session_compress.go`, `engine/topic_consolidation.go` |
-| Self-linking | `engine/selflink.go` |
-| Entity extraction | `engine/entities.go`, `engine/entity_boost.go` |
-| Proactive context | `engine/proactive.go` |
-| Mental model | `engine/mental/` |
-| Storage interface | `storage/interface.go` |
-| SQLite storage | `storage/sqlite.go` |
-| Graph operations | `graph/graph.go`, `graph/community.go`, `graph/drift_search.go` |
-| Hooks (session lifecycle) | `hooks/` (SessionStart, PostToolUse, SessionEnd, ShouldCapture) |
-| Skills | `skill/` (Store, Load, Replay) |
-| Config loading | `config/` (TOML parsing, defaults) |
-| MCP server | `internal/server/` |
-| MCP prompts | `internal/server/mcp_prompts.go` |
-| Integration tests | `yaad_test.go` (full lifecycle, concurrent, edge cases) |
-| Linter config | `.golangci.yml` (errcheck, govet, staticcheck, gocritic, bodyclose, noctx) |
+## 3. Skills
+
+Skills are markdown instruction files that extend agent capabilities. They can be:
+- Project-scoped: dropped in `./.zero/skills/` or `./skills/`
+- User-scoped: dropped in `~/.config/hawk-eco/skills/`
+
+A skill manifest:
+
+```markdown
+---
+description: How to review Go code for security issues
+globs: "*.go"
+alwaysApply: true
+---
+
+When reviewing Go code for security:
+
+1. Check for SQL injection patterns
+2. Verify error handling doesn't expose sensitive data
+3. Confirm secrets are not hardcoded
+4. Validate input sanitization
+```
+
+## 4. Hooks
+
+Hooks allow custom commands to run at specific lifecycle points:
+- `beforeReview` — runs before code review starts
+- `afterReview` — runs after code review completes
+- `sessionStart` — runs at session initialization
+- `sessionEnd` — runs at session teardown
+
+```bash
+hawk-eco hook add beforeReview --command "lint-check"
+hawk-eco hook remove beforeReview
+hawk-eco hook list
+```
+
+## 5. MCP integration
+
+MCP (Model Context Protocol) servers can expose tools to hawk-eco:
+
+```bash
+hawk-eco mcp add --name server --url http://localhost:8080
+hawk-eco mcp remove server
+hawk-eco mcp list
+```
+
+## 6. Plugins
+
+Plugins extend hawk-eco with custom tools and capabilities:
+
+```bash
+hawk-eco plugin add --name my-plugin --path ./my-plugin
+hawk-eco plugin remove my-plugin
+hawk-eco plugin list
+```
+
+## 7. Verification
+
+hawk-eco includes a self-verification system to validate local changes before contributing:
+
+```bash
+hawk-eco verify
+hawk-eco verify --fix
+```
+
+## Development
+
+```bash
+make lint
+hawk-eco verify
+```

engine/audit.go

@@ -32,9 +32,12 @@ type AuditEntry struct {
 // NewAuditLog creates or opens the audit log file.
 func NewAuditLog(yaadDir string) (*AuditLog, error) {
 	path := filepath.Join(yaadDir, "audit.jsonl")
-	_ = os.MkdirAll(yaadDir, 0o755)
+	// The audit log records node IDs and operation details from memory
+	// content, so it belongs to the same class of sensitive data as the
+	// sqlite store — keep the directory and file owner-only.
+	_ = os.MkdirAll(yaadDir, 0o700)
 
-	f, err := os.OpenFile(path, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0o644)
+	f, err := os.OpenFile(path, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0o600)
 	if err != nil {
 		return nil, fmt.Errorf("open audit log: %w", err)
 	}

engine/remember.go

@@ -9,7 +9,6 @@ import (
 	"time"
 
 	"github.com/GrayCodeAI/yaad/internal/telemetry"
-	"github.com/GrayCodeAI/yaad/privacy"
 	"github.com/GrayCodeAI/yaad/storage"
 	"github.com/google/uuid"
 )
@@ -70,8 +69,11 @@ func (e *Engine) Remember(ctx context.Context, in RememberInput) (*storage.Node,
 
 	// Steps 1-3: Privacy filtering, defaults, and content hashing are pure
 	// computation with no shared state, so they run outside the lock.
-	content := stripPrivateTags(privacy.Filter(in.Content))
-	summary := stripPrivateTags(privacy.Filter(in.Summary))
+	// Use the engine's configured privacy level/StripPrivate setting (set via
+	// WithPrivacyLevel/WithPrivacyConfig) instead of the package-default filter,
+	// so a caller's configured privacy level is actually honored.
+	content := e.FilterContent(in.Content)
+	summary := e.FilterContent(in.Summary)
 	if in.Scope == "" {
 		in.Scope = "project"
 	}

internal/daemon/daemon.go

@@ -30,8 +30,10 @@ func PIDFile(projectDir string) string {
 
 // WritePID writes the current process PID to the PID file.
 func WritePID(projectDir string) error {
+	// .yaad also holds the sqlite store and audit log, both owner-only;
+	// keep the directory consistent rather than relying on creation order.
 	dir := filepath.Join(projectDir, ".yaad")
-	if err := os.MkdirAll(dir, 0o755); err != nil {
+	if err := os.MkdirAll(dir, 0o700); err != nil {
 		return err
 	}
 	return os.WriteFile(PIDFile(projectDir), []byte(strconv.Itoa(os.Getpid())), 0o644)