From d5a5a6b9fb8a77d9f24a605c6d84b8c847e96fb9 Mon Sep 17 00:00:00 2001 From: Lakshman Patel Date: Sun, 5 Jul 2026 20:34:48 +0530 Subject: [PATCH 1/4] chore: commit wip changes on feature/wip-20260705 --- .github/workflows/ci.yml | 2 +- lefthook.yml | 5 ++++- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 0c40487..6a09a20 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -1,5 +1,5 @@ # Canonical CI workflow for hawk-eco Go repos. -# Source of truth: .shared-templates/workflows/go-ci.yml.tmpl +# Source of truth: https://github.com/GrayCodeAI/hawk/blob/main/.shared-templates/workflows/go-ci.yml.tmpl # # Two deployment models: # diff --git a/lefthook.yml b/lefthook.yml index 7d5bdaf..edab577 100644 --- a/lefthook.yml +++ b/lefthook.yml @@ -1,5 +1,5 @@ # Canonical lefthook config for hawk-eco Go repos. -# Source of truth: .shared-templates/lefthook.yml.tmpl +# Source of truth: https://github.com/GrayCodeAI/hawk/blob/main/.shared-templates/lefthook.yml.tmpl # # Install lefthook: # brew install lefthook (macOS) @@ -75,6 +75,9 @@ pre-commit: pre-push: commands: + boundaries: + run: bash ./scripts/check-ecosystem-boundaries.sh + test: run: go test ./... -count=1 -timeout=60s From 65ba17615889fe53ee49dae5e1daa507f2a78f09 Mon Sep 17 00:00:00 2001 From: Lakshman Patel Date: Tue, 7 Jul 2026 14:47:18 +0530 Subject: [PATCH 2/4] refactor: consolidate version into root version.go with //go:embed VERSION - Remove internal/version/ package (embedded stale co-located copy) - Add root version.go with //go:embed VERSION as single source of truth - Update rest handler to import root yaad package instead of internal/version --- internal/server/rest_handlers.go | 6 +-- internal/version/VERSION | 1 - internal/version/version.go | 52 -------------------- internal/version/version_test.go | 84 -------------------------------- version.go | 32 ++++++++++++ 5 files changed, 35 insertions(+), 140 deletions(-) delete mode 100644 internal/version/VERSION delete mode 100644 internal/version/version.go delete mode 100644 internal/version/version_test.go create mode 100644 version.go diff --git a/internal/server/rest_handlers.go b/internal/server/rest_handlers.go index a4dd4e8..511fb96 100644 --- a/internal/server/rest_handlers.go +++ b/internal/server/rest_handlers.go @@ -16,7 +16,7 @@ import ( gitwatch "github.com/GrayCodeAI/yaad/git" "github.com/GrayCodeAI/yaad/graph" "github.com/GrayCodeAI/yaad/internal/telemetry" - "github.com/GrayCodeAI/yaad/internal/version" + yaadversion "github.com/GrayCodeAI/yaad" "github.com/GrayCodeAI/yaad/storage" "github.com/google/uuid" ) @@ -271,11 +271,11 @@ func (s *RESTServer) handleHealth(w http.ResponseWriter, r *http.Request) { httpJSON(w, map[string]string{"status": "error", "error": err.Error()}, 503) return } - httpJSON(w, map[string]string{"status": "ok", "version": version.String()}, 200) + httpJSON(w, map[string]string{"status": "ok", "version": yaadversion.String()}, 200) } func (s *RESTServer) handleVersion(w http.ResponseWriter, _ *http.Request) { - httpJSON(w, map[string]string{"version": version.String()}, 200) + httpJSON(w, map[string]string{"version": yaadversion.String()}, 200) } // handleLiveness responds to Kubernetes liveness probes (/healthz). diff --git a/internal/version/VERSION b/internal/version/VERSION deleted file mode 100644 index 6e8bf73..0000000 --- a/internal/version/VERSION +++ /dev/null @@ -1 +0,0 @@ -0.1.0 diff --git a/internal/version/version.go b/internal/version/version.go deleted file mode 100644 index 1a6ae42..0000000 --- a/internal/version/version.go +++ /dev/null @@ -1,52 +0,0 @@ -// Package version provides the canonical version string for yaad. -// -// Single source of truth: the VERSION file at the repo root, which is -// kept in sync with the co-located internal/version/VERSION via the -// `make tidy` target (or `release-please`, which bumps both atomically). -// -// The co-located VERSION file is embedded at compile time via `go:embed`, -// so `go build`, `go install`, and `go get` all see the correct version -// with zero ldflags wiring required. Release builds (goreleaser, or -// `make build` if a binary is ever added) may additionally inject a -// short commit + build date through ldflags: -// -// go build -ldflags " \ -// -X github.com/GrayCodeAI/yaad/internal/version.Commit=$(git rev-parse --short HEAD) \ -// -X github.com/GrayCodeAI/yaad/internal/version.Date=$(date -u +%Y-%m-%dT%H:%M:%SZ)" -// -// Do NOT edit Version directly — bump the root VERSION file and resync -// the co-located one (or let release-please/goreleaser do it). -package version - -import ( - _ "embed" - "fmt" - "runtime" - "strings" -) - -//go:embed VERSION -var versionFile string - -// Version is the current version of yaad, embedded from the VERSION file -// at compile time. -var Version = strings.TrimSpace(versionFile) - -// Commit is the git commit short SHA. Set via ldflags at release time; -// defaults to "none" for plain `go build`. -var Commit = "none" - -// Date is the build date in RFC3339. Set via ldflags at release time; -// defaults to "unknown" for plain `go build`. -var Date = "unknown" - -// String returns just the version string (kept for backwards compatibility -// with existing call sites that do `fmt.Printf("yaad v%s", version.String())`). -func String() string { return Version } - -// Full returns a verbose, human-readable version string suitable for -// `yaad --version` output. -func Full() string { - return fmt.Sprintf("yaad %s (commit: %s, built: %s, %s/%s)", - Version, Commit, Date, runtime.GOOS, runtime.GOARCH) -} diff --git a/internal/version/version_test.go b/internal/version/version_test.go deleted file mode 100644 index d2c9df6..0000000 --- a/internal/version/version_test.go +++ /dev/null @@ -1,84 +0,0 @@ -package version - -import ( - "runtime" - "strings" - "testing" -) - -func TestStringReturnsVersion(t *testing.T) { - t.Parallel() - got := String() - if got != Version { - t.Errorf("String() = %q, want %q", got, Version) - } -} - -func TestStringDefault(t *testing.T) { - t.Parallel() - // When built without ldflags, Version defaults to "dev". - // We only check it's non-empty. - if s := String(); s == "" { - t.Error("String() returned empty string") - } -} - -func TestFullFormat(t *testing.T) { - t.Parallel() - got := Full() - - // Should start with "yaad " - if !strings.HasPrefix(got, "yaad ") { - t.Errorf("Full() should start with 'yaad ', got %q", got) - } - - // Should contain the version - if !strings.Contains(got, Version) { - t.Errorf("Full() should contain version %q, got %q", Version, got) - } - - // Should contain the commit - if !strings.Contains(got, Commit) { - t.Errorf("Full() should contain commit %q, got %q", Commit, got) - } - - // Should contain the date - if !strings.Contains(got, Date) { - t.Errorf("Full() should contain date %q, got %q", Date, got) - } - - // Should contain GOOS and GOARCH - if !strings.Contains(got, runtime.GOOS) { - t.Errorf("Full() should contain GOOS %q, got %q", runtime.GOOS, got) - } - if !strings.Contains(got, runtime.GOARCH) { - t.Errorf("Full() should contain GOARCH %q, got %q", runtime.GOARCH, got) - } -} - -func TestFullContainsCommitLabel(t *testing.T) { - t.Parallel() - got := Full() - if !strings.Contains(got, "commit:") { - t.Errorf("Full() should contain 'commit:' label, got %q", got) - } - if !strings.Contains(got, "built:") { - t.Errorf("Full() should contain 'built:' label, got %q", got) - } -} - -func TestDefaultValues(t *testing.T) { - t.Parallel() - // Verify the default ldflags values are present when not overridden. - // These tests work for both dev and release builds because we check - // the variable values directly rather than hardcoding expected strings. - if Version == "" { - t.Error("Version should not be empty") - } - if Commit == "" { - t.Error("Commit should not be empty") - } - if Date == "" { - t.Error("Date should not be empty") - } -} diff --git a/version.go b/version.go new file mode 100644 index 0000000..982844c --- /dev/null +++ b/version.go @@ -0,0 +1,32 @@ +// Package yaad provides graph-based persistent memory for coding agents. +// +// The Version variable is sourced from the VERSION file at the repo root. +package yaad + +import ( + _ "embed" + "fmt" + "runtime" + "strings" +) + +//go:embed VERSION +var versionFile string + +// Version of the yaad library. Single source of truth: VERSION file. +var Version = strings.TrimSpace(versionFile) + +// Commit is the git commit short SHA. Set via ldflags at release time. +var Commit = "none" + +// Date is the build date in RFC3339. Set via ldflags at release time. +var Date = "unknown" + +// String returns just the version string. +func String() string { return Version } + +// Full returns a verbose version string suitable for `--version` output. +func Full() string { + return fmt.Sprintf("yaad %s (commit: %s, built: %s, %s/%s)", + Version, Commit, Date, runtime.GOOS, runtime.GOARCH) +} From fa1a1bca1fbc3bde80880d6db8c55902b34e22f1 Mon Sep 17 00:00:00 2001 From: Lakshman Patel Date: Wed, 8 Jul 2026 05:07:22 +0530 Subject: [PATCH 3/4] Security and quality hardening pass - Fix Strict-mode email regex to catch internationalized (SMTPUTF8/RFC 6531) addresses via Unicode letter/number classes, not just ASCII - CI, daemon, audit, memory, and MCP core hardening fixes from cross-repo consistency review - Update AGENTS.md to the shared hawk-eco extension-authoring format --- .github/workflows/ci.yml | 6 +- AGENTS.md | 273 +++++++++++++++++++++--------------- engine/audit.go | 7 +- engine/remember.go | 8 +- internal/daemon/daemon.go | 4 +- internal/server/mcp_core.go | 66 ++++++++- privacy/filter.go | 40 +++++- storage/sqlite.go | 29 ++++ 8 files changed, 299 insertions(+), 134 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 6a09a20..935d4c3 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -207,9 +207,9 @@ jobs: run: git clone --depth=1 https://github.com/GrayCodeAI/tok.git ../tok - name: Run fuzz targets run: | - go test -fuzz=FuzzContentHash -fuzztime=60s ./engine/... || true - go test -fuzz=FuzzExtractEntities -fuzztime=60s ./engine/... || true - go test -fuzz=FuzzRecallOpts -fuzztime=60s ./engine/... || true + go test -fuzz=FuzzContentHash -fuzztime=60s ./engine + go test -fuzz=FuzzExtractEntities -fuzztime=60s ./engine + go test -fuzz=FuzzRecallOpts -fuzztime=60s ./engine # ------------------------------------------------------------------------- # Cross-platform build matrix — only for repos that produce a binary. diff --git a/AGENTS.md b/AGENTS.md index bf835a3..4976a45 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -1,125 +1,168 @@ -# AGENTS.md — Yaad (याद) +--- +description: Extending hawk-eco — how to write AGENTS.md files, custom specialists, skills, hooks, MCP servers, and plugins. +globs: "*.go, *.js, *.md, *.json, *.toml, *.yaml, *.yml" +alwaysApply: false +--- -Graph-based persistent memory for coding agents. One config line. Works with any MCP agent. Zero setup. +# Extending hawk-eco -## Design Principles +hawk-eco is an open-source code intelligence platform. This document describes how to extend it with custom tools, skills, hooks, and integrations. -- **Zero setup** — single config line to enable persistent memory -- **MCP compatible** — works with any MCP-capable agent -- **Graph-based** — knowledge stored as a graph, not flat key-value -- **Self-healing** — memory consolidates and prunes automatically +## 1. Drop a project `AGENTS.md` -## Observability +When hawk-eco starts in a directory, it looks for project-level instructions and injects them into the system prompt. The lookup walks from your current working directory **up to the nearest git root** and reads the first matching file at each level — general rules at the repo root, more specific rules in sub-trees. Files are labeled with their directory in the prompt (e.g. `## Project guidelines (services/api/AGENTS.md)`). -See [hawk/docs/OTEL-CONVENTIONS.md](https://github.com/GrayCodeAI/hawk/blob/main/docs/OTEL-CONVENTIONS.md) for the shared OpenTelemetry attribute vocabulary (`gen_ai.*`, `cost.usd`, etc.) used across all GrayCodeAI repos. +Accepted file names, in priority order at each level: -## Build & Test +| Path | Notes | +| --- | --- | +| `./AGENTS.md` | The classic spot — committed to your repo, shared with the team. | +| `./ZERO.md` | Brand-specific alias. Same format, lower priority. | +| `./.zero/AGENTS.md` | Project-local, hidden, gitignored. Personal notes that stay out of git. | + +Matching is **case-insensitive** on the basename, so `AGENTS.md`, `Agents.md`, and `agents.md` resolve to the same file on Windows and macOS. The git-tracked filename in this repo is `AGENTS.md` — keep that on case-sensitive filesystems (Linux, the WSL filesystem, or a CI runner) to match what the loader looks for. + +Both files use the same format. YAML frontmatter is optional; the markdown body is loaded as instructions for the agent. hawk-eco reads the file once at session start, so changes take effect on the next launch — not mid-session. + +```markdown +# Project conventions for + +- Build with `make`, not `go build` directly. +- Tests live next to the source file (`foo_test.go` next to `foo.go`). +- Run `make lint` before opening a PR. +- Never edit files under `third_party/` — those are vendored. +``` + +Tips: + +- Keep each file under ~8 KiB. hawk-eco caps the **total** across all matched files at 32 KiB; everything past the cap is dropped. +- Re-state rules in the imperative voice: "Run `make lint`", not "you should consider running the linter". +- Don't put secrets, model IDs, or environment-specific paths in `AGENTS.md`. Use config files for those. +- In a monorepo, drop a narrower `AGENTS.md` in each sub-tree (e.g. `services/api/AGENTS.md`). hawk-eco picks those up automatically when you launch from inside the sub-tree. +- A YAML frontmatter block (`---\n...\n---`) at the top is preserved verbatim in the injected prompt but is not parsed for `globs:` or `alwaysApply:` scoping today — keep the body self-contained. + +### Personal guidelines, across every project + +For preferences that follow *you*, not a specific repo (tone, tooling habits, workflow), drop a `ZERO.md` in your user config directory: `~/.config/hawk-eco/ZERO.md` on Linux/macOS, `%AppData%\Roaming\hawk-eco\ZERO.md` on Windows — the same directory as config files and your personal specialists. Same format and 8 KiB cap as the project files above, and the same case-insensitive basename match. + +This file is injected as its own `## User guidelines` section, before the project's `AGENTS.md`/`ZERO.md`, and is labeled as personal preference in the prompt: project guidelines are the later, more specific instruction and take precedence over it when the two conflict. + +## 2. Custom specialists + +Specialists are hawk-eco's sub-agents. Three scopes, in priority order: + +| Scope | Path | Shared? | +| --- | --- | --- | +| Built-in | compiled into hawk-eco | yes | +| User | `~/.config/hawk-eco/specialists/*.md` | no — your machine only | +| Project | `./.zero/specialists/*.md` | yes — the repo team | + +Project overrides user overrides built-in when names collide. + +A specialist is a markdown manifest with frontmatter and a system prompt: + +```markdown +--- +description: Reviews API changes for breaking-change risk and missing tests. +tools: read-only,plan +--- + +You review API changes. For every changed hunk in `internal/api/` or any file +that ends in `_api.go`: + +1. Confirm the public signature is backward-compatible, or note the breaking + change explicitly with the migration path. +2. Confirm a corresponding test exists in `internal/api/*_test.go` and that + the new behaviour is exercised. +3. Flag any new exported symbol without a doc comment. + +Reply with one JSON object per finding: `{"file", "line", "severity", "message", "fix"}`. +``` + +CLI management: ```bash -go test ./... # Run all tests -go test -race ./... # Race detector -go test -coverprofile=c.out ./... # Coverage -go vet ./... # Static analysis -gofumpt -w . # Format +hawk-eco specialist list +hawk-eco specialist show api-reviewer +hawk-eco specialist create api-reviewer \ + --project \ + --description "Reviews API changes" \ + --tools read-only,plan \ + --prompt "$(cat api-reviewer.md)" +hawk-eco specialist edit api-reviewer --project +hawk-eco specialist delete api-reviewer --project +hawk-eco specialist path # prints the resolved specialists directory ``` -## Architecture - -- `memory.go` — Core memory graph (nodes, edges, queries) -- `server.go` — MCP server implementation -- `consolidator.go` — Automatic memory consolidation -- `recall.go` — Fused recall with atomic operations -- `rest.go` — REST API for non-MCP clients -- `internal/` — Storage, indexing, search internals - -## Conventions - -- Go 1.26+, pure Go, no CGO -- Table-driven tests -- Conventional Commits: `feat:`, `fix:`, `docs:`, `refactor:`, `test:` -- No `Co-authored-by:` trailers (auto-stripped by githook) -- `gofumpt` formatting enforced in CI -- SQLITE_BUSY prevention: use `settleSelfLink` waits in tests - -## Common Pitfalls - -- FusedRecall uses atomic loads for `NodesStored` — don't use regular reads -- SelfLink settling needs adequate wait time in tests (prevents SQLITE_BUSY) -- MCP prompts are defined in `internal/server/mcp_prompts.go` - -## Naming Conventions - -- **Engine is the core facade**: `engine.Engine` with methods `Remember()`, `Recall()`, `Context()`, `Feedback()`, `Forget()`, `Compact()` -- **Storage layer uses Store interface**: `storage.Store` with `GetNode()`, `UpdateNode()`, `ListSessions()`, `GetEdgesFrom()`, etc. -- **Graph layer**: `graph.New(store, git)` — wraps store with graph operations, community detection, drift search -- **Node types are lowercase strings**: `"convention"`, `"decision"`, `"task"`, `"bug"`, `"preference"`, `"spec"` — validated on input -- **Scopes are lowercase strings**: `"project"`, `"global"` — used for memory isolation -- **Hooks package**: `hooks.New(engine, dir)` returns a `Runner` with `SessionStart()`, `PostToolUse()`, `SessionEnd()` -- **Config uses TOML**: `config.Load(dir)` reads `.yaad/config.toml` — struct fields map to `[section] key` format -- **Error sentinels**: defined in `engine/errors.go` — `Err` prefix pattern -- **Skill types**: `skill.Skill` with `Name`, `Description`, `Steps` — stored as memory nodes, loaded by name - -## API Patterns - -- **Engine.Remember()**: takes `RememberInput` struct (Type, Content, Scope, Key, Project, Session, Agent, Pinned) — returns `(Node, error)` -- **Engine.Recall()**: takes `RecallOpts` struct (Query, Limit, Budget, Project) — returns result with Nodes slice -- **Engine.Context()**: takes project string — returns tiered context (hot/warm/cold) formatted as markdown -- **Engine.Feedback()**: takes node ID + action (`FeedbackApprove`, `FeedbackEdit`, `FeedbackDiscard`) — modifies confidence -- **Keyed upsert**: `Remember()` with `Key` field updates existing node instead of creating duplicate — same ID, incremented Version -- **Token budget enforcement**: `RecallOpts.Budget` limits total returned tokens — stops adding nodes when budget exceeded -- **Auto-decay on session start**: `hooks.SessionStart()` triggers `RunDecay()` — confidence decreases based on `HalfLifeDays` -- **Self-linking**: after storing a node, the engine finds related nodes via FTS and creates edges — best-effort, async -- **Entity extraction**: `Remember()` auto-extracts file paths, package names, function names — creates anchor nodes + `touches` edges - -## Testing Patterns - -- **Integration test package**: `package integration_test` in `yaad_test.go` — tests the full engine lifecycle -- **`setup(t)` helper**: returns `(engine, cleanup func)` — creates in-memory SQLite store, wires engine, returns cleanup -- **`ctx()` helper**: returns `context.Background()` — used throughout for brevity -- **Lifecycle tests**: `TestHawkFullSessionLifecycle` — session start, tool uses, session end, recall verification -- **Table-driven relevance tests**: `TestRelevanceFilterScoring` — struct slice with `tool, input, output, err, shouldCapture` fields -- **Concurrent safety test**: `TestConcurrentHawkOperations` — 3 goroutines doing remember/recall/context simultaneously -- **Edge case tests**: max content length (10000 chars), invalid node type, empty content — all should return errors -- **Config tests**: `TestConfigLoadDefaults`, `TestConfigLoadFromFile` — verify TOML parsing and default values -- **Compaction test**: create low-confidence nodes, run `Compact()`, verify merge count -- **Rollback test**: remember, edit, rollback to version 1 — verify content restored - -## Refactoring Guidelines - -- **Safe to refactor**: `engine/decay.go`, `engine/scoring.go`, `engine/rerank.go` — internal ranking logic -- **Safe to refactor**: `engine/query.go`, `engine/query_expand.go`, `engine/query_planner.go` — search internals -- **Safe to refactor**: `hooks/` package — relevance scoring, tool filtering — no external API contract -- **Do not touch**: `engine.Remember()`, `Recall()`, `Context()`, `Feedback()` signatures — core API contract -- **Do not touch**: `storage.Store` interface — implemented by SQLite, used by engine, graph, hooks -- **Do not touch**: `graph.Graph` interface — used by engine and hooks for relationship queries -- **Do not touch**: Node type strings (`"convention"`, `"decision"`, etc.) — used in MCP prompts and client code -- **Safe to extend**: add new node types, new edge types, new search strategies, new consolidation passes -- **When adding MCP tools**: add to `internal/server/` — each tool is a separate handler function - -## Key File Locations - -| What | Where | -|---|---| -| Engine core | `engine/engine_core.go` (facade with Remember, Recall, Context, Feedback, etc.) | -| Engine interface | `engine/interface.go` (if exists — defines Engine contract) | -| Remember implementation | `engine/remember.go` | -| Recall implementation | `engine/recall.go` | -| Fused recall (BM25 + vector + graph + temporal) | `engine/fused_recall.go` | -| Context generation | `engine/context.go`, `engine/context_pack.go` | -| Decay logic | `engine/decay.go` | -| Feedback handling | `engine/feedback.go` | -| Compaction | `engine/session_compress.go`, `engine/topic_consolidation.go` | -| Self-linking | `engine/selflink.go` | -| Entity extraction | `engine/entities.go`, `engine/entity_boost.go` | -| Proactive context | `engine/proactive.go` | -| Mental model | `engine/mental/` | -| Storage interface | `storage/interface.go` | -| SQLite storage | `storage/sqlite.go` | -| Graph operations | `graph/graph.go`, `graph/community.go`, `graph/drift_search.go` | -| Hooks (session lifecycle) | `hooks/` (SessionStart, PostToolUse, SessionEnd, ShouldCapture) | -| Skills | `skill/` (Store, Load, Replay) | -| Config loading | `config/` (TOML parsing, defaults) | -| MCP server | `internal/server/` | -| MCP prompts | `internal/server/mcp_prompts.go` | -| Integration tests | `yaad_test.go` (full lifecycle, concurrent, edge cases) | -| Linter config | `.golangci.yml` (errcheck, govet, staticcheck, gocritic, bodyclose, noctx) | +## 3. Skills + +Skills are markdown instruction files that extend agent capabilities. They can be: +- Project-scoped: dropped in `./.zero/skills/` or `./skills/` +- User-scoped: dropped in `~/.config/hawk-eco/skills/` + +A skill manifest: + +```markdown +--- +description: How to review Go code for security issues +globs: "*.go" +alwaysApply: true +--- + +When reviewing Go code for security: + +1. Check for SQL injection patterns +2. Verify error handling doesn't expose sensitive data +3. Confirm secrets are not hardcoded +4. Validate input sanitization +``` + +## 4. Hooks + +Hooks allow custom commands to run at specific lifecycle points: +- `beforeReview` — runs before code review starts +- `afterReview` — runs after code review completes +- `sessionStart` — runs at session initialization +- `sessionEnd` — runs at session teardown + +```bash +hawk-eco hook add beforeReview --command "lint-check" +hawk-eco hook remove beforeReview +hawk-eco hook list +``` + +## 5. MCP integration + +MCP (Model Context Protocol) servers can expose tools to hawk-eco: + +```bash +hawk-eco mcp add --name server --url http://localhost:8080 +hawk-eco mcp remove server +hawk-eco mcp list +``` + +## 6. Plugins + +Plugins extend hawk-eco with custom tools and capabilities: + +```bash +hawk-eco plugin add --name my-plugin --path ./my-plugin +hawk-eco plugin remove my-plugin +hawk-eco plugin list +``` + +## 7. Verification + +hawk-eco includes a self-verification system to validate local changes before contributing: + +```bash +hawk-eco verify +hawk-eco verify --fix +``` + +## Development + +```bash +make lint +hawk-eco verify +``` diff --git a/engine/audit.go b/engine/audit.go index efcf060..0d3afa5 100644 --- a/engine/audit.go +++ b/engine/audit.go @@ -32,9 +32,12 @@ type AuditEntry struct { // NewAuditLog creates or opens the audit log file. func NewAuditLog(yaadDir string) (*AuditLog, error) { path := filepath.Join(yaadDir, "audit.jsonl") - _ = os.MkdirAll(yaadDir, 0o755) + // The audit log records node IDs and operation details from memory + // content, so it belongs to the same class of sensitive data as the + // sqlite store — keep the directory and file owner-only. + _ = os.MkdirAll(yaadDir, 0o700) - f, err := os.OpenFile(path, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0o644) + f, err := os.OpenFile(path, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0o600) if err != nil { return nil, fmt.Errorf("open audit log: %w", err) } diff --git a/engine/remember.go b/engine/remember.go index a7b2fcb..6f1e123 100644 --- a/engine/remember.go +++ b/engine/remember.go @@ -9,7 +9,6 @@ import ( "time" "github.com/GrayCodeAI/yaad/internal/telemetry" - "github.com/GrayCodeAI/yaad/privacy" "github.com/GrayCodeAI/yaad/storage" "github.com/google/uuid" ) @@ -70,8 +69,11 @@ func (e *Engine) Remember(ctx context.Context, in RememberInput) (*storage.Node, // Steps 1-3: Privacy filtering, defaults, and content hashing are pure // computation with no shared state, so they run outside the lock. - content := stripPrivateTags(privacy.Filter(in.Content)) - summary := stripPrivateTags(privacy.Filter(in.Summary)) + // Use the engine's configured privacy level/StripPrivate setting (set via + // WithPrivacyLevel/WithPrivacyConfig) instead of the package-default filter, + // so a caller's configured privacy level is actually honored. + content := e.FilterContent(in.Content) + summary := e.FilterContent(in.Summary) if in.Scope == "" { in.Scope = "project" } diff --git a/internal/daemon/daemon.go b/internal/daemon/daemon.go index 9786202..45c189f 100644 --- a/internal/daemon/daemon.go +++ b/internal/daemon/daemon.go @@ -30,8 +30,10 @@ func PIDFile(projectDir string) string { // WritePID writes the current process PID to the PID file. func WritePID(projectDir string) error { + // .yaad also holds the sqlite store and audit log, both owner-only; + // keep the directory consistent rather than relying on creation order. dir := filepath.Join(projectDir, ".yaad") - if err := os.MkdirAll(dir, 0o755); err != nil { + if err := os.MkdirAll(dir, 0o700); err != nil { return err } return os.WriteFile(PIDFile(projectDir), []byte(strconv.Itoa(os.Getpid())), 0o644) diff --git a/internal/server/mcp_core.go b/internal/server/mcp_core.go index 68edeea..8702cc7 100644 --- a/internal/server/mcp_core.go +++ b/internal/server/mcp_core.go @@ -3,17 +3,34 @@ package server import ( "context" "encoding/json" + "net/http" "os" + "strings" + "time" "github.com/GrayCodeAI/yaad/engine" "github.com/mark3labs/mcp-go/mcp" mcpserver "github.com/mark3labs/mcp-go/server" ) +// maxMCPRequestBodySize caps the body of an MCP-over-HTTP request, matching +// the REST server's maxRequestBodySize so both HTTP surfaces have the same +// resource-exhaustion protection. +const maxMCPRequestBodySize = 1 << 20 // 1 MB + // MCPServer wraps the MCP protocol server for Hawk integration. type MCPServer struct { eng *engine.Engine server *mcpserver.MCPServer + apiKey string // if set, require Bearer / X-API-Key on HTTP requests +} + +// WithAPIKey sets the API key required for authenticated HTTP requests +// (Bearer / X-API-Key). Has no effect on ServeStdio, which is trusted by +// virtue of being a local subprocess pipe. +func (s *MCPServer) WithAPIKey(key string) *MCPServer { + s.apiKey = key + return s } // NewMCPServer creates an MCP server with all yaad tools registered. @@ -40,18 +57,61 @@ func (s *MCPServer) ServeStdio() error { // ServeHTTP starts the MCP server on a streamable HTTP endpoint. // The default endpoint path is /mcp. Clients connect to http:///mcp. func (s *MCPServer) ServeHTTP(addr string) error { - httpServer := mcpserver.NewStreamableHTTPServer(s.server) + httpServer, err := s.buildHTTPServer(addr) + if err != nil { + return err + } return httpServer.Start(addr) } // ServeHTTPWithShutdown starts the MCP server on a streamable HTTP endpoint // and returns the server so the caller can invoke Shutdown for graceful teardown. func (s *MCPServer) ServeHTTPWithShutdown(addr string) (*mcpserver.StreamableHTTPServer, error) { - httpServer := mcpserver.NewStreamableHTTPServer(s.server) - err := httpServer.Start(addr) + httpServer, err := s.buildHTTPServer(addr) + if err != nil { + return nil, err + } + err = httpServer.Start(addr) return httpServer, err } +// buildHTTPServer wires the streamable MCP transport behind the same minimum +// HTTP hardening the REST server gets: a body-size cap, an optional API-key +// check, and a slowloris-resistant header read timeout. It deliberately does +// NOT impose an overall request/response deadline (unlike the REST server's +// http.TimeoutHandler) because MCP's GET transport is a long-lived SSE +// stream; a blanket timeout would sever legitimate long-running sessions. +func (s *MCPServer) buildHTTPServer(addr string) (*mcpserver.StreamableHTTPServer, error) { + const endpointPath = "/mcp" + mux := http.NewServeMux() + httpServer := mcpserver.NewStreamableHTTPServer(s.server, mcpserver.WithStreamableHTTPServer(&http.Server{ + Addr: addr, + Handler: mux, + ReadHeaderTimeout: 10 * time.Second, + })) + mux.Handle(endpointPath, s.authMiddleware(httpServer)) + return httpServer, nil +} + +// authMiddleware caps the request body and, if an API key is configured, +// requires it via Bearer or X-API-Key before delegating to next. +func (s *MCPServer) authMiddleware(next http.Handler) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + r.Body = http.MaxBytesReader(w, r.Body, maxMCPRequestBodySize) + if s.apiKey != "" { + token := strings.TrimPrefix(r.Header.Get("Authorization"), "Bearer ") + if token == "" { + token = r.Header.Get("X-API-Key") + } + if !constantTimeEqual(token, s.apiKey) { + http.Error(w, `{"error":"unauthorized"}`, http.StatusUnauthorized) + return + } + } + next.ServeHTTP(w, r) + }) +} + func (s *MCPServer) registerTools() { add := func(tool mcp.Tool, handler mcpserver.ToolHandlerFunc) { s.server.AddTool(tool, handler) diff --git a/privacy/filter.go b/privacy/filter.go index 6dbc9e4..a6e13d2 100644 --- a/privacy/filter.go +++ b/privacy/filter.go @@ -60,8 +60,12 @@ var piiPatterns = []*regexp.Regexp{ // strictOnlyPatterns are only stripped in Strict mode. var strictOnlyPatterns = []*regexp.Regexp{ - // PII: email addresses (Moderate keeps work emails) - regexp.MustCompile(`[a-zA-Z0-9._%+\-]+@[a-zA-Z0-9.\-]+\.[a-zA-Z]{2,}`), + // PII: email addresses (Moderate keeps work emails). \p{L}/\p{N} (rather + // than a-zA-Z0-9) so internationalized (SMTPUTF8, RFC 6531) addresses with + // a literal non-ASCII local part or domain — e.g. "üser@müenchen.de" — + // are also redacted; punycode-encoded IDN domains were already ASCII and + // matched the narrower pattern. + regexp.MustCompile(`[\p{L}\p{N}._%+\-]+@[\p{L}\p{N}.\-]+\.[\p{L}]{2,}`), // PII: IPv4 addresses (Moderate keeps infrastructure IPs like 10.x, 192.168.x) regexp.MustCompile(`\b(?:(?:25[0-5]|2[0-4]\d|[01]?\d\d?)\.){3}(?:25[0-5]|2[0-4]\d|[01]?\d\d?)\b`), } @@ -94,15 +98,37 @@ func FilterWithLevel(content string, level FilterLevel) string { // Catch high-entropy tokens that regexes might miss. // Only target tokens that look like standalone secrets (no JSON, no code). - for _, word := range strings.Fields(content) { + // Replacement is done by byte offset (not a global substring Replace), so a + // short flagged token that happens to also appear as a substring of an + // earlier, unrelated word cannot cause the wrong location to be redacted + // while the actual flagged token is left untouched. + content = redactHighEntropyWords(content) + return content +} + +// wordPattern matches whitespace-delimited words, used to locate entropy +// candidates by exact byte offset rather than by substring search. +var wordPattern = regexp.MustCompile(`\S+`) + +func redactHighEntropyWords(content string) string { + var sb strings.Builder + lastIdx := 0 + for _, loc := range wordPattern.FindAllStringIndex(content, -1) { + wordStart, wordEnd := loc[0], loc[1] + word := content[wordStart:wordEnd] clean := strings.Trim(word, `"',:;{}[]()`) - // IsLikelySecret performs the entropy/shape checks internally, including - // the pure-alphanumeric high-entropy case (hex/base64/random keys). if len(clean) >= 24 && !strings.ContainsAny(clean, "{}[]():/ ") && IsLikelySecret(clean) { - content = strings.Replace(content, clean, "[REDACTED]", 1) + // clean is word with leading/trailing punctuation trimmed; locate + // its exact span within this word so surrounding punctuation is preserved. + cleanStart := wordStart + strings.Index(word, clean) + cleanEnd := cleanStart + len(clean) + sb.WriteString(content[lastIdx:cleanStart]) + sb.WriteString("[REDACTED]") + lastIdx = cleanEnd } } - return content + sb.WriteString(content[lastIdx:]) + return sb.String() } // hasHighEntropy returns true if a string has Shannon entropy above threshold. diff --git a/storage/sqlite.go b/storage/sqlite.go index e967fbe..5297518 100644 --- a/storage/sqlite.go +++ b/storage/sqlite.go @@ -8,6 +8,8 @@ import ( "database/sql" "errors" "fmt" + "os" + "path/filepath" "strings" "sync" "time" @@ -236,6 +238,18 @@ type Store struct { func (s *Store) DB() *sql.DB { return s.db } func NewStore(dbPath string) (*Store, error) { + // The store holds personal/agent memory content, so its directory and + // files must not be group/world-readable regardless of the process + // umask. In-memory DSNs (":memory:", "file::memory:...") have no + // directory to create or file to lock down. + if !isMemoryDSN(dbPath) { + if dir := filepath.Dir(dbPath); dir != "" && dir != "." { + if err := os.MkdirAll(dir, 0o700); err != nil { + return nil, fmt.Errorf("create db directory: %w", err) + } + } + } + // modernc.org/sqlite v1.34.4 only processes _pragma query parameters // (unlike mattn/go-sqlite3 which accepts _busy_timeout, _journal_mode, // and _foreign_keys as top-level keys). Without this fix, these DSN @@ -250,6 +264,15 @@ func NewStore(dbPath string) (*Store, error) { if err := db.PingContext(context.Background()); err != nil { return nil, err } + if !isMemoryDSN(dbPath) { + // Ping (and the WAL pragma above) may have just created the db/-wal/-shm + // files under the process umask (commonly 0644); tighten them to owner-only. + for _, suffix := range []string{"", "-wal", "-shm"} { + if err := os.Chmod(dbPath+suffix, 0o600); err != nil && !os.IsNotExist(err) { + return nil, fmt.Errorf("restrict db file permissions: %w", err) + } + } + } // WAL mode supports concurrent readers alongside a single writer. // A small pool (5) allows concurrent read operations while _busy_timeout // handles brief write contention. WAL mode ensures readers never block. @@ -265,6 +288,12 @@ func NewStore(dbPath string) (*Store, error) { return s, nil } +// isMemoryDSN reports whether dbPath refers to an in-memory SQLite database +// (no on-disk file/directory to create or lock down). +func isMemoryDSN(dbPath string) bool { + return dbPath == ":memory:" || strings.HasPrefix(dbPath, "file::memory:") +} + // withTimeout returns a child context with the store's query timeout applied. // If the parent context already has a shorter deadline, it is returned unchanged. func (s *Store) withTimeout(ctx context.Context) (context.Context, context.CancelFunc) { From 4711bfd5e248c23fe45254169c73bbdd5916f241 Mon Sep 17 00:00:00 2001 From: Lakshman Patel Date: Wed, 8 Jul 2026 07:51:36 +0530 Subject: [PATCH 4/4] Apply gofumpt formatting to satisfy CI format check --- internal/server/rest_handlers.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/server/rest_handlers.go b/internal/server/rest_handlers.go index 511fb96..2f50a84 100644 --- a/internal/server/rest_handlers.go +++ b/internal/server/rest_handlers.go @@ -11,12 +11,12 @@ import ( "net/http" "time" + yaadversion "github.com/GrayCodeAI/yaad" "github.com/GrayCodeAI/yaad/embeddings" "github.com/GrayCodeAI/yaad/engine" gitwatch "github.com/GrayCodeAI/yaad/git" "github.com/GrayCodeAI/yaad/graph" "github.com/GrayCodeAI/yaad/internal/telemetry" - yaadversion "github.com/GrayCodeAI/yaad" "github.com/GrayCodeAI/yaad/storage" "github.com/google/uuid" )