diff --git a/.claude-plugin/marketplace.json b/.claude-plugin/marketplace.json index 7524848..504d762 100644 --- a/.claude-plugin/marketplace.json +++ b/.claude-plugin/marketplace.json @@ -13,7 +13,7 @@ "name": "llm-wiki", "source": "./plugins/llm-wiki", "description": "Create and query project wikis with optional maintenance automation.", - "version": "0.3.1", + "version": "0.3.2", "author": { "name": "ivankuznetsov", "url": "https://github.com/ivankuznetsov" diff --git a/docs/agent-compatibility.md b/docs/agent-compatibility.md index 4539fe2..43e904c 100644 --- a/docs/agent-compatibility.md +++ b/docs/agent-compatibility.md @@ -35,7 +35,7 @@ plugin directory so a copied package does not depend on the repository root. | Agent Reviewer | `0.3.1` | Stable | `skills/agent-reviewer/SKILL.md` | agents, references, scripts, eval harness | | Agent SEO | `2.0.1` | Stable | `skills/seo/SKILL.md` | agents, context, data sources, hooks, scripts | | Agent Writing | `0.5.2` | Stable | `skills/writing/SKILL.md` | agents, voice/style context | -| LLM Wiki | `0.3.1` | Stable | five files under `skills/` | assets, consent-gated maintenance templates | +| LLM Wiki | `0.3.2` | Stable | five files under `skills/` | assets, consent-gated maintenance templates | | Screenote | `3.0.1` | Stable | `skills/{screenote,snapshot,feedback}/SKILL.md` | CLI launcher, references, evals | ## Plugin invocations diff --git a/plugin-surfaces.json b/plugin-surfaces.json index ad2dd57..bdeb98e 100644 --- a/plugin-surfaces.json +++ b/plugin-surfaces.json @@ -174,7 +174,7 @@ { "name": "llm-wiki", "path": "plugins/llm-wiki", - "version": "0.3.1", + "version": "0.3.2", "stability": "stable", "metadata": { "display_name": "LLM Wiki", diff --git a/plugin-surfaces.lock.json b/plugin-surfaces.lock.json index f45895c..c4c3257 100644 --- a/plugin-surfaces.lock.json +++ b/plugin-surfaces.lock.json @@ -636,11 +636,11 @@ } }, "llm-wiki": { - "version": "0.3.1", + "version": "0.3.2", "canonical": { "skills/bootstrap/SKILL.md": { - "sha256": "9b9597565a6020a70724efc0d274f1730c6fa32985648d638aaa9da378e9b839", - "semantic_sha256": "f953edb019bdc2c4dcf07fcc9d88dc252f1736a27d3240727d1bca7478397430", + "sha256": "6e446e783ac32d2f605f036891fcee29651aca14696065f2753cec13eab58a16", + "semantic_sha256": "f36ca66c3dbf57c57e2f13e89a9027580b727e35a4e58cc71db2e1564d47efdb", "sections": { "1:bootstrap llm wiki": "774f22f2954bef05dc6cdf5f125c0cf1125903dde6f81336c5ec061447386b9d", "2:preconditions": "82a221777c857eede5e97fddc34ce173709cba496b373154fac6410d2eaff30d", @@ -651,20 +651,20 @@ "2:[timestamp] bootstrap": "485c7edd8b497ca6654d20f1c139675f9e27cff72e4cf04f9f494dd44d393632", "2:step 5: add agent instructions": "fd40fbda54316fd3acbf2b0e22050f4f0682f459b34982e36b65bb6490b1e3de", "2:wiki": "1eda77f740be21744dbbea6153ed8d7e220fe6944aa55574ccf025fe46d2487e", - "2:step 6: hooks, scheduled automation, and qmd": "def8c592c6fcdf848c3b53cfd8d1811e87e18db4cd2b3b28a4caae95ed2b1a6f", + "2:step 6: hooks, scheduled automation, and qmd": "882e01f1a7be17ffab96fc34ae35cbbe8b79edee797f378b110a5671105fc131", "1:or": "d0ce4eb2166e22f45b8faf03f2a6c1ba2626fa52a30cb6998a606ccf74a274cc", "2:step 7: report": "5b7459747dc14c6c96ff58ca95b73c35d9995c2177c1ca828a94b27f5e3d37a0", "2:rules": "6535528b074874edb47973a9bfee3f0f209bebfca01486b8f11d2c59c8bc0092" } }, "skills/upgrade/SKILL.md": { - "sha256": "8f1005ef7997e28b1f656798767ee75347e1bc05710ac0adebcb82551f0a59fc", - "semantic_sha256": "b70ba03f836f68d4f2f2d37e7051441bfd16f95358b6ff49d43a54ea5fc9add3", + "sha256": "e5dcff373583573bcb7eb3375f1a7906b3ce32a572519b526f9d8e5be3ba5b31", + "semantic_sha256": "af583ec71a6c1e84e62c87859dfa0ab13e7890d3082db77fb16bddfc47dc067b", "sections": { - "1:upgrade llm wiki project structure": "9d93eacc79a1b3bd06492bfe8d6da41929ca7740e9d104bc57b19e4b4430085b", + "1:upgrade llm wiki project structure": "56556675486274c770f88e736ad65cff9e00009154adc8d76d93ba14c0677f8f", "2:preconditions": "4e05f4c754a0bc9a897ba43a7e303b90656aa67af9f67341c1da65a8cd48ffb7", - "2:run the upgrade": "d31f5a9f121318a7cd8312527640042b9aea42be02134091f063805298269bad", - "2:safety contract": "830b805bc6bbc985fe149e314a566b99d02c45468f3c795dfc7c443656a2ef32", + "2:run the upgrade": "f23e962fffe7efb8b4519c87cdc5b0e60cafb868e97531e47da47ddc1c07773f", + "2:safety contract": "27384894672abc0af96330fee4ee6ee983b57b255b04dc28268dc0dd812020b6", "2:report": "a9a4e2088a69fc85f0bc906ee30eec02a4287ef805e0a955983f414bae1adadd" } }, @@ -722,18 +722,20 @@ }, "templates": { "exists": true, - "sha256": "468cbaebd27e5db982c522211954d82d50013fadc652a41deb8639a79c643dc0", + "sha256": "264d01db93a76080b4ecb586a5cbe98608edc6358820a36f676a1c2683f3d2af", "files": [ "templates/compile-log.sh", - "templates/post-commit-refresh.sh" + "templates/install-systemd-scheduler.sh", + "templates/post-commit-refresh.sh", + "templates/refresh-wiki.sh" ] } }, "adapters": { "pi/skills/wiki-bootstrap/SKILL.md": { - "sha256": "30295c49c48525bd200c2ec5fd2bf8f3bd7e280ff00743d8ad18ae08ac9b4106", + "sha256": "35600a6d44693b492fb7b197866ad5052c3f00ee1262849776e36beb5889e8dc", "canonical": "skills/bootstrap/SKILL.md", - "canonical_semantic_sha256": "f953edb019bdc2c4dcf07fcc9d88dc252f1736a27d3240727d1bca7478397430", + "canonical_semantic_sha256": "f36ca66c3dbf57c57e2f13e89a9027580b727e35a4e58cc71db2e1564d47efdb", "overlays": [ "frontmatter", "invocation", @@ -741,9 +743,9 @@ ] }, "openclaw/skills/wiki-bootstrap/SKILL.md": { - "sha256": "92d45026922a865927e42b7ad880990624e4c11a26972da1d0a95d6d6cbe66f6", + "sha256": "ee96d7b8ecb513831c1ff954e72888bfd3dc9e8e7c9b3213e29e7c51f141694b", "canonical": "skills/bootstrap/SKILL.md", - "canonical_semantic_sha256": "f953edb019bdc2c4dcf07fcc9d88dc252f1736a27d3240727d1bca7478397430", + "canonical_semantic_sha256": "f36ca66c3dbf57c57e2f13e89a9027580b727e35a4e58cc71db2e1564d47efdb", "overlays": [ "frontmatter", "invocation", @@ -751,9 +753,9 @@ ] }, "pi/skills/wiki-upgrade/SKILL.md": { - "sha256": "5f6e360de9f1a5d8bf6c51b5d3083fc4aa2fc5b43695e629c79e595709e5297a", + "sha256": "ba41283db68305ac52bbee91ad1b570d9f449e8c9329d962490ad24ec4721d5b", "canonical": "skills/upgrade/SKILL.md", - "canonical_semantic_sha256": "b70ba03f836f68d4f2f2d37e7051441bfd16f95358b6ff49d43a54ea5fc9add3", + "canonical_semantic_sha256": "af583ec71a6c1e84e62c87859dfa0ab13e7890d3082db77fb16bddfc47dc067b", "overlays": [ "frontmatter", "invocation", @@ -761,9 +763,9 @@ ] }, "openclaw/skills/wiki-upgrade/SKILL.md": { - "sha256": "99651026489b7fd0a5cca9ac6306f4ef16033e695f5b5e5ae6724cf0f6f8fdf5", + "sha256": "39e2b2f1b6243b8efd0cc4c75dd9fea6b840d55ff77991e5d4c89652d54237f3", "canonical": "skills/upgrade/SKILL.md", - "canonical_semantic_sha256": "b70ba03f836f68d4f2f2d37e7051441bfd16f95358b6ff49d43a54ea5fc9add3", + "canonical_semantic_sha256": "af583ec71a6c1e84e62c87859dfa0ab13e7890d3082db77fb16bddfc47dc067b", "overlays": [ "frontmatter", "invocation", diff --git a/plugins/llm-wiki/.claude-plugin/plugin.json b/plugins/llm-wiki/.claude-plugin/plugin.json index ff010ab..91ab284 100644 --- a/plugins/llm-wiki/.claude-plugin/plugin.json +++ b/plugins/llm-wiki/.claude-plugin/plugin.json @@ -1,6 +1,6 @@ { "name": "llm-wiki", - "version": "0.3.1", + "version": "0.3.2", "description": "Create and query project wikis with optional maintenance automation.", "author": { "name": "ivankuznetsov", diff --git a/plugins/llm-wiki/.codex-plugin/plugin.json b/plugins/llm-wiki/.codex-plugin/plugin.json index 4de73c7..90b2252 100644 --- a/plugins/llm-wiki/.codex-plugin/plugin.json +++ b/plugins/llm-wiki/.codex-plugin/plugin.json @@ -1,6 +1,6 @@ { "name": "llm-wiki", - "version": "0.3.1", + "version": "0.3.2", "description": "Create and query project wikis with optional maintenance automation.", "author": { "name": "ivankuznetsov", diff --git a/plugins/llm-wiki/CHANGELOG.md b/plugins/llm-wiki/CHANGELOG.md index ca05b6b..488521c 100644 --- a/plugins/llm-wiki/CHANGELOG.md +++ b/plugins/llm-wiki/CHANGELOG.md @@ -4,6 +4,21 @@ All notable changes to **llm-wiki** are documented here. The format follows [Keep a Changelog](https://keepachangelog.com/), and the project adheres to [Semantic Versioning](https://semver.org/). +## [0.3.2] - 2026-07-22 + +### Fixed + +- Reconcile all linked worktrees to one non-persistent, memory-bounded systemd + timer per repository and stop obsolete managed units instead of multiplying + jobs across temporary checkouts. +- Drain queued commits under a machine-wide provider lock and publish wiki-only + results to `origin/llm-wiki/refresh`, leaving protected default checkouts + clean. +- Bound large source-pin migrations to 64 refs per Git transaction and recover + empty crash-left queue records when their source commit remains available. +- Preserve the four-agent consent gate and OpenClaw owner dispatch while + upgrading projects to the shared 0.1.16 transactional runtime. + ## [0.3.1] - 2026-07-20 ### Fixed diff --git a/plugins/llm-wiki/README.md b/plugins/llm-wiki/README.md index ed20d2a..ef6b8db 100644 --- a/plugins/llm-wiki/README.md +++ b/plugins/llm-wiki/README.md @@ -193,10 +193,13 @@ runner starts only when both `automation_enabled` and - Pi headless automation uses `pi -p --no-session --tools read,bash,edit,write,grep,find,ls ...` - OpenClaw headless automation uses `openclaw agent --local --agent --message ... --json --timeout 1800` without delivery, channel, reply, or recipient flags. Bootstrap records the agent whose configured workspace matches the project instead of guessing a default ID. - All automation paths search the project wiki and any detected main cross-project wiki. -- Scheduler and post-commit entries use managed markers and stable project slugs so repeated bootstraps do not create duplicate refresh jobs. +- Linux scheduler installation resolves the repository's primary checkout and + reconciles all linked worktrees to one non-persistent timer. Obsolete managed + timers are stopped and removed. Every repository shares one machine-wide + provider lock, and each service is limited to 4 GiB RAM with swap disabled. - Post-commit maintenance never writes into a user checkout. Relevant commits are coalesced in the shared Git directory and refreshed transactionally on the - local `llm-wiki/refresh` branch through a disposable managed worktree. A + `llm-wiki/refresh` branch through a disposable managed worktree. A canonical runner in that shared Git directory serves every linked worktree, with one canonical owner config, so upgrading once cannot leave older branches executing stale local scripts or selecting a stale provider. @@ -209,9 +212,12 @@ runner starts only when both `automation_enabled` and bounded changed-path context, so concurrent hooks cannot turn a historical backlog into an unbounded sequence of subscription runs. Override these defaults with `LLM_WIKI_MAX_AUTO_PENDING`, `LLM_WIKI_MAX_BATCH_SOURCES`, + `LLM_WIKI_MAX_SOURCE_PIN_BATCH`, `LLM_WIKI_MAX_PATHS_PER_SOURCE`, and `LLM_WIKI_MAX_PATH_BYTES`. - Queued commits are pinned under `refs/llm-wiki/sources/` until their durable - receipt is written. Sources that arrive outside a running batch open a visible + Queued commits are pinned under `refs/llm-wiki/sources/` in transactions of + at most 64 refs by default until their durable receipt is written. Empty + crash-left `..` queue files are reconstructed when the source commit + is still available. Sources that arrive outside a running batch open a visible `deferred:` circuit rather than remaining silently pending. Atomic source-SHA receipt refs make changed and no-op acknowledgement replay-safe, and a compare-and-swap Git ref makes stale-lock replacement single-winner. @@ -228,8 +234,10 @@ runner starts only when both `automation_enabled` and sources remain; only the final successful batch clears the circuit. A failed retry leaves it open. Pass a full source SHA instead of `all` to restore one quarantined record. -- The refresh branch is intentionally local and is never pushed automatically; - operators can inspect, merge, or open a PR from it on their normal schedule. +- Scheduled maintenance drains the same durable queue used by commit hooks. It + fetches, merges, and pushes only `origin/llm-wiki/refresh`; it never writes to + or pushes the protected default branch. A publication failure retains the + local refresh commit and queued state for a later retry. ## Update Status diff --git a/plugins/llm-wiki/openclaw.plugin.json b/plugins/llm-wiki/openclaw.plugin.json index d799093..02af868 100644 --- a/plugins/llm-wiki/openclaw.plugin.json +++ b/plugins/llm-wiki/openclaw.plugin.json @@ -2,7 +2,7 @@ "id": "llm-wiki", "name": "LLM Wiki", "description": "Create and query project wikis with optional maintenance automation.", - "version": "0.3.1", + "version": "0.3.2", "skills": [ "./openclaw/skills" ], diff --git a/plugins/llm-wiki/openclaw/skills/wiki-bootstrap/SKILL.md b/plugins/llm-wiki/openclaw/skills/wiki-bootstrap/SKILL.md index 82ef97b..3fd5047 100644 --- a/plugins/llm-wiki/openclaw/skills/wiki-bootstrap/SKILL.md +++ b/plugins/llm-wiki/openclaw/skills/wiki-bootstrap/SKILL.md @@ -248,6 +248,8 @@ OpenClaw context: - Ensure `AGENTS.md` contains the wiki section from Step 5. OpenClaw auto-injects the `AGENTS.md` in its configured agent workspace at the start of every session. - Confirm the project root is the active OpenClaw agent workspace before selecting `openclaw` as `headless_agent`. If it is not, report the workspace mismatch instead of claiming the project `AGENTS.md` will be injected. - Do not add channel-delivery flags to wiki maintenance commands. Scheduled and post-commit refreshes are local automation, not chat replies. +- Omit `--deliver`, `--channel`, `--reply-channel`, `--reply-to`, and `--to` + from every OpenClaw maintenance command. Do not install scheduled or post-commit automation unless the user separately approves it after seeing the provider, files, hooks, and activation commands. @@ -256,78 +258,52 @@ refresh commands instead. If the current tool is not the configured `headless_agent`, update session context for the current tool and validate/report the existing automation owner. Do not rewrite scheduler or post-commit ownership unless automation is missing, unsafe, or the user asks to repair or switch ownership. -The scheduler must use the configured `.llm-wiki/config.json` `headless_agent`. - -Create `.llm-wiki/refresh-wiki.sh` and make it executable. It should run the configured headless agent's CLI from the project root: - -- `headless_agent: "codex"`: use `codex exec -C "" ""`. -- `headless_agent: "claude"`: use `claude -p ""` with the same refresh intent. -- `headless_agent: "pi"`: use `pi -p --no-session --tools read,bash,edit,write,grep,find,ls ""` with the same refresh intent. -- `headless_agent: "openclaw"`: use `openclaw agent --local --agent "" --message "" --json --timeout 1800` from the configured OpenClaw project workspace. The explicit agent selector is required by the CLI and binds the turn to the verified project workspace. Omit `--deliver`, `--channel`, `--reply-channel`, `--reply-to`, and `--to` so automation cannot send the result to a channel. - -For Codex-owned headless automation, never write automation that shells out to `claude` or `claude -p`. Do not fall back from Codex automation to Claude if `codex` is missing; report the missing `codex` CLI instead. - -For Claude-owned headless automation, never write automation that shells out to `codex exec`. Do not fall back from Claude automation to Codex if `claude` is missing; report the missing `claude` CLI instead. - -For Pi-owned headless automation, never write automation that shells out to `codex exec` or `claude -p`. Do not fall back from Pi automation to Claude or Codex if `pi` is missing; report the missing `pi` CLI instead. - -For OpenClaw-owned headless automation, never write automation that shells out to `codex exec`, `claude -p`, or `pi`. Do not fall back from OpenClaw automation to another agent if `openclaw` is missing; report the missing `openclaw` CLI or project-workspace mismatch instead. - -Codex refresh script shape: +The scheduler and post-commit hook must use the configured +`.llm-wiki/config.json` `headless_agent`. Do not hand-write provider wrappers. +Resolve the installed `llm-wiki` package root by walking up from the active +skill until the ancestor containing both `templates/` and the package manifest +is found. This works for native and generated host skill paths. Copy these +canonical files from `/templates/`, make them executable, and keep +the same copies in `.llm-wiki/`: + +- `post-commit-refresh.sh` +- `refresh-wiki.sh` +- `compile-log.sh` +- `install-systemd-scheduler.sh` + +Copy the runner, compiler, and validated config into +`$(git rev-parse --git-common-dir)/llm-wiki/`. Wire the common `post-commit` +hook to call the shared runner with +`--project "$(git rev-parse --show-toplevel)"`, falling back to the checkout +copy only when the shared runner is absent. This makes the primary checkout's +installed runtime authoritative for every linked worktree, including older +branches with stale ignored `.llm-wiki` files or config. + +After the separate automation approval, on Linux run: ```bash -#!/usr/bin/env bash -set -euo pipefail -project_root="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" -cd "$project_root" - -codex exec -C "$project_root" "Refresh this project's LLM wiki. Read .llm-wiki/config.json, AGENTS.md, wiki/index.md, wiki/gaps.md, and recent wiki/log.md entries first. If .llm-wiki/config.json contains main_wiki_path, search that exact path before changing project pages. Also search default main cross-project wiki paths when they exist: ~/wikis/master/wiki/, ~/wikis/main/wiki/, ../wikis/master/wiki/, and ../wikis/main/wiki/. Inspect recent git history and changed source files. Update stale wiki pages, update wiki/index.md when page coverage changes, append wiki/log.md, and record uncertainty in wiki/gaps.md. Do not invent facts." +.llm-wiki/install-systemd-scheduler.sh --project "$(git rev-parse --show-toplevel)" ``` -Claude Code refresh script shape: - -```bash -#!/usr/bin/env bash -set -euo pipefail -project_root="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" -cd "$project_root" - -claude -p "Refresh this project's LLM wiki. Read .llm-wiki/config.json, CLAUDE.md, wiki/index.md, wiki/gaps.md, and recent wiki/log.md entries first. If .llm-wiki/config.json contains main_wiki_path, search that exact path before changing project pages. Also search default main cross-project wiki paths when they exist: ~/wikis/master/wiki/, ~/wikis/main/wiki/, ../wikis/master/wiki/, and ../wikis/main/wiki/. Inspect recent git history and changed source files. Update stale wiki pages, update wiki/index.md when page coverage changes, append wiki/log.md, and record uncertainty in wiki/gaps.md. Do not invent facts." --allowedTools "Bash,Read,Edit,Write" --max-budget-usd 0.50 -``` - -Pi refresh script shape: - -```bash -#!/usr/bin/env bash -set -euo pipefail -project_root="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" -cd "$project_root" - -pi -p --no-session --tools read,bash,edit,write,grep,find,ls "Refresh this project's LLM wiki. Read .llm-wiki/config.json, AGENTS.md, wiki/index.md, wiki/gaps.md, and recent wiki/log.md entries first. If .llm-wiki/config.json contains main_wiki_path, search that exact path before changing project pages. Also search default main cross-project wiki paths when they exist: ~/wikis/master/wiki/, ~/wikis/main/wiki/, ../wikis/master/wiki/, and ../wikis/main/wiki/. Inspect recent git history and changed source files. Update stale wiki pages, update wiki/index.md when page coverage changes, append wiki/log.md, and record uncertainty in wiki/gaps.md. Do not invent facts." -``` - -OpenClaw refresh script shape: - -```bash -#!/usr/bin/env bash -set -euo pipefail -project_root="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" -cd "$project_root" - -openclaw agent --local --agent "" --message "Refresh this project's LLM wiki. Read .llm-wiki/config.json, AGENTS.md, wiki/index.md, wiki/gaps.md, and recent wiki/log.md entries first. If .llm-wiki/config.json contains main_wiki_path, search that exact path before changing project pages. Also search default main cross-project wiki paths when they exist: ~/wikis/master/wiki/, ~/wikis/main/wiki/, ../wikis/master/wiki/, and ../wikis/main/wiki/. Inspect recent git history and changed source files. Update stale wiki pages, update wiki/index.md when page coverage changes, add a wiki/log.d/-.md fragment without editing compiled wiki/log.md, and record uncertainty in wiki/gaps.md. Do not invent facts." --json --timeout 1800 -``` - -After that separate approval, install the best available scheduler: - -- Linux with systemd user services: create `~/.config/systemd/user/llm-wiki-.service` and `.timer`, then run `systemctl --user daemon-reload` and `systemctl --user enable --now llm-wiki-.timer`. -- macOS with launchd: create `~/Library/LaunchAgents/com.llm-wiki..plist` with a 24 hour `StartInterval`, then run `launchctl load`. -- Other environments: install an equivalent cron entry that runs `.llm-wiki/refresh-wiki.sh` daily. - -Use a stable `` from the repository basename plus a short hash of the project root to avoid timer name collisions. Replace existing `llm-wiki-` scheduler files instead of adding duplicates. For cron, wrap the entry with `# BEGIN LLM WIKI ` and `# END LLM WIKI ` markers and replace that block on repeat bootstrap. If scheduler installation fails because the environment lacks systemd, launchd, cron, or permissions, keep `.llm-wiki/refresh-wiki.sh`, record the failure in `wiki/gaps.md`, and report the exact command the user can run. - -Also install post-commit wiki maintenance automation. Preserve existing hooks; do not overwrite unrelated hook logic. Install the canonical runtime in the shared Git directory and wire the common `post-commit` hook to pass the committing worktree explicitly. - -Install `.llm-wiki/post-commit-refresh.sh` AND `.llm-wiki/compile-log.sh` by copying the reference scripts bundled with this skill at `templates/post-commit-refresh.sh` and `templates/compile-log.sh` (resolve them relative to this SKILL.md), then `chmod +x` both. Also copy the same files verbatim to `$(git rev-parse --git-common-dir)/llm-wiki/post-commit-refresh.sh` and `compile-log.sh`, and copy the validated project config there as `config.json`. The hook must invoke that shared runner as `post-commit-refresh.sh --project "$(git rev-parse --show-toplevel)"`, falling back to the checkout-local runner only when the shared copy is absent. This makes one bootstrap or upgrade authoritative for every linked worktree, including older branches with stale ignored `.llm-wiki` files or config. `compile-log.sh` is the single source of truth for the changelog format: it regenerates `wiki/log.md` from the append-only `wiki/log.d/*.md` fragments, and the refresh runs it before committing. (Hive's `Hive::WikiLog` delegates here, so Ruby and shell callers share one implementation.) The bundled post-commit script reads the canonical shared `headless_agent` and dispatches to exactly one provider; never customize its provider function per project. Provider, QMD, and Git ref execution requires `timeout` or `gtimeout` so every potentially stuck command is bounded. When neither is available, the worker must fail before starting a provider. A repository-wide circuit stops automatic provider launches after two consecutive failed batches or when more than 25 sources are pending by default. A worker handles at most one batch of 10 sources with bounded path context; sources arriving outside that snapshot open a `deferred:` circuit. New sources continue queueing until an operator explicitly runs `.llm-wiki/post-commit-refresh.sh --retry-failed ` once per bounded batch. +The installer reconciles all linked worktrees to one non-persistent timer for +the repository's primary checkout, removes and stops obsolete managed units, +serializes all repositories through one machine-wide lock, and enforces a 4 GiB +memory limit with no swap. The timer drains already queued commits; it does not +launch a direct refresh against whichever checkout happened to install it. +When automation is not approved, do not enable a timer; the upgrade path uses +`--disabled` to reconcile old units without activating maintenance. On +non-systemd platforms, leave the scripts installed, record the scheduler gap in +`wiki/gaps.md`, and report the manual drain command. + +Preserve existing hooks and unrelated hook logic. The bundled runner reads the +canonical shared owner config and dispatches to exactly one of Claude Code, +Codex, Pi, or OpenClaw. It never falls back to a different provider. An +OpenClaw owner requires a validated `openclaw_agent_id` and never uses delivery, +channel, reply, or recipient flags. Provider, QMD, and Git ref execution is +time-bounded. A repository-wide circuit stops automatic provider launches after +two consecutive failed batches or more than 25 pending sources. Each worker +handles at most one batch of 10 sources, while the scheduler keeps draining +later batches without starting concurrent provider processes. The supported provider set is Claude Code, Codex, Pi, and OpenClaw. The copied config must preserve `openclaw_agent_id` when OpenClaw owns maintenance; the runtime rejects an unsupported owner or a missing/invalid OpenClaw agent ID @@ -336,13 +312,13 @@ instead of falling back to another provider. Transactional refresh contract (the bundled script implements all of these; any hand-edit must keep them): - **User checkouts are read-only inputs.** Queue each relevant source SHA under `$(git rev-parse --git-common-dir)/llm-wiki/pending/` before attempting the worker lock. Never use the committing checkout or first/main checkout as an agent workspace, log destination, QMD cache, staging area, or commit target. -- **Use one managed refresh branch and disposable worktree.** Drain queued commits on local branch `llm-wiki/refresh` in `/llm-wiki/refresh-worktree`, based/rebased on the current default branch. The agent inspects source commits with `git show`; it writes only under the managed worktree's `wiki/`. +- **Use one published refresh branch and disposable worktree.** Drain queued commits on `llm-wiki/refresh` in `/llm-wiki/refresh-worktree`, based/rebased on the current remote default branch. The agent inspects source commits with `git show`; it writes only under the managed worktree's `wiki/`. Publish only this branch to `origin`; never dirty or push the protected default branch. - **Seed ignored local wikis once.** When a new refresh branch has no tracked `wiki/`, copy the committing checkout's untracked local wiki into the disposable worktree before invoking the agent. Never replace an already-established refresh-branch wiki, follow a top-level `wiki` symlink, or copy any path outside `wiki/`. - **Serialize and coalesce.** Hold one stale-reclaimable compare-and-swap lock at `refs/llm-wiki/refresh-lock`. Its Git blob records PID, time, process-start identity, and nonce; `git update-ref ` makes stale replacement single-winner, while ownership-checked deletion prevents an old worker from releasing a successor's lock. Check the wait deadline and sleep after every failed acquisition attempt. One worker snapshots at most 10 queued SHAs and runs one refresh agent for that batch. A busy worker leaves new queue entries intact for a later worker instead of dropping them. - **Receipt completed sources.** Add one exact `LLM-Wiki-Source: ` commit-message paragraph when the batch creates a wiki commit, then atomically update `refs/llm-wiki/receipts/` for every successful source before queue deletion. Check receipt refs first and commit history as migration fallback before invoking the agent. This makes both changed and no-op batches replay-safe after a crash between completion and acknowledgement. -- **Pin queued commits.** Create `refs/llm-wiki/sources/` when a source is queued and delete it in the same ref transaction that writes its durable receipt. Backfill pins for pre-upgrade pending and quarantined records. Refuse to invoke or acknowledge a batch whose selected SHA is not an available commit. +- **Pin queued commits in bounded transactions.** Create `refs/llm-wiki/sources/` when a source is queued and delete it in the same ref transaction that writes its durable receipt. Backfill pins for pre-upgrade pending and quarantined records in batches of 64 by default (`LLM_WIKI_MAX_SOURCE_PIN_BATCH`). Reconstruct empty crash-left `..` queue files when their source commit is available; retain unavailable records for diagnosis. Refuse to invoke or acknowledge a batch whose selected SHA is not an available commit. - **Recover stale locks safely.** A live owner PID with the recorded process-start identity wins. Dead, PID-reused, or malformed owner blobs are replaceable only through the Git ref's compare-and-swap old-OID guard. -- **Validate before committing.** Reject any tracked, untracked, or ignored change outside `wiki/`. Compile `wiki/log.md`, force-stage only `wiki/` so intentionally ignored wikis persist, and commit with both recursion guards (`HIVE_SKIP_LLM_WIKI_POST_COMMIT=1` and `git -c core.hooksPath=/dev/null`). Never push automatically. +- **Validate before committing and publishing.** Reject any tracked, untracked, or ignored change outside `wiki/`. Compile `wiki/log.md`, force-stage only `wiki/` so intentionally ignored wikis persist, and commit with both recursion guards (`HIVE_SKIP_LLM_WIKI_POST_COMMIT=1` and `git -c core.hooksPath=/dev/null`). Fetch and merge the remote refresh branch, then push only `llm-wiki/refresh`; retain local work when fetch, merge, or push fails. - **Failure is clean and bounded.** If agent execution, wiki-only validation, compilation, staging, or commit fails, force-remove the disposable managed worktree. After two consecutive failed batches by default, move the active batch to `/llm-wiki/failed/` and open the repository-wide circuit. Continue queueing new sources without launching a provider. Never delete failed source data or automatically run a quarantined source again. User checkout bytes and branch refs must remain unchanged. - **Subscription use is bounded.** Run provider overrides, Codex, Claude Code, Pi, QMD, and Git ref operations through `timeout` or `gtimeout`. If no bounded diff --git a/plugins/llm-wiki/openclaw/skills/wiki-upgrade/SKILL.md b/plugins/llm-wiki/openclaw/skills/wiki-upgrade/SKILL.md index 95fa7f9..db84fdc 100644 --- a/plugins/llm-wiki/openclaw/skills/wiki-upgrade/SKILL.md +++ b/plugins/llm-wiki/openclaw/skills/wiki-upgrade/SKILL.md @@ -12,8 +12,9 @@ metadata: Upgrade the current project's managed llm-wiki files to the templates bundled with this installed release. This is a narrow, deterministic migration: do not -regenerate wiki pages, switch the headless owner, run a refresh agent, update -QMD, or reinstall the scheduler. +regenerate wiki pages, switch the headless owner, run a refresh agent, or update +QMD. It deterministically reconciles the managed scheduler and leaves it +disabled unless both automation consent flags are true. ## Preconditions @@ -45,10 +46,15 @@ not a migration failure. The script upgrades only managed structure: - `.llm-wiki/post-commit-refresh.sh` +- `.llm-wiki/refresh-wiki.sh` - `.llm-wiki/compile-log.sh` -- canonical copies of both scripts under +- `.llm-wiki/install-systemd-scheduler.sh` +- canonical copies of the runner and compiler under `$(git rev-parse --git-common-dir)/llm-wiki/` for all linked worktrees - canonical shared owner config and keepalive refs for existing queued commits +- bounded source-pin backfill, including recoverable interrupted queue writes +- one bounded repository-wide systemd timer, or the same reconciled units left + disabled when automation consent is absent - a missing `.llm-wiki/config.json` when live or historical evidence identifies one unambiguous legacy owner - `wiki/log.d/` and the compiled `wiki/log.md` layout @@ -64,7 +70,7 @@ checkout-local copy remains a manual entrypoint and compatibility fallback. ## Safety Contract - Preserve unrelated tracked and untracked project changes. -- Overwrite only the two llm-wiki-managed scripts listed above. +- Overwrite only the four llm-wiki-managed scripts listed above. - Replace or deduplicate only the marked post-commit hook block; preserve all unmarked hook logic and refuse unmatched markers. - Preserve all hand-written legacy changelog prose while introducing the diff --git a/plugins/llm-wiki/package.json b/plugins/llm-wiki/package.json index ad09f17..beca938 100644 --- a/plugins/llm-wiki/package.json +++ b/plugins/llm-wiki/package.json @@ -1,6 +1,6 @@ { "name": "llm-wiki", - "version": "0.3.1", + "version": "0.3.2", "type": "module", "description": "Create and query project wikis with optional maintenance automation.", "homepage": "https://github.com/ivankuznetsov/llm-wiki", diff --git a/plugins/llm-wiki/pi/skills/wiki-bootstrap/SKILL.md b/plugins/llm-wiki/pi/skills/wiki-bootstrap/SKILL.md index 5182469..8a31253 100644 --- a/plugins/llm-wiki/pi/skills/wiki-bootstrap/SKILL.md +++ b/plugins/llm-wiki/pi/skills/wiki-bootstrap/SKILL.md @@ -248,6 +248,8 @@ OpenClaw context: - Ensure `AGENTS.md` contains the wiki section from Step 5. OpenClaw auto-injects the `AGENTS.md` in its configured agent workspace at the start of every session. - Confirm the project root is the active OpenClaw agent workspace before selecting `openclaw` as `headless_agent`. If it is not, report the workspace mismatch instead of claiming the project `AGENTS.md` will be injected. - Do not add channel-delivery flags to wiki maintenance commands. Scheduled and post-commit refreshes are local automation, not chat replies. +- Omit `--deliver`, `--channel`, `--reply-channel`, `--reply-to`, and `--to` + from every OpenClaw maintenance command. Do not install scheduled or post-commit automation unless the user separately approves it after seeing the provider, files, hooks, and activation commands. @@ -256,78 +258,52 @@ refresh commands instead. If the current tool is not the configured `headless_agent`, update session context for the current tool and validate/report the existing automation owner. Do not rewrite scheduler or post-commit ownership unless automation is missing, unsafe, or the user asks to repair or switch ownership. -The scheduler must use the configured `.llm-wiki/config.json` `headless_agent`. - -Create `.llm-wiki/refresh-wiki.sh` and make it executable. It should run the configured headless agent's CLI from the project root: - -- `headless_agent: "codex"`: use `codex exec -C "" ""`. -- `headless_agent: "claude"`: use `claude -p ""` with the same refresh intent. -- `headless_agent: "pi"`: use `pi -p --no-session --tools read,bash,edit,write,grep,find,ls ""` with the same refresh intent. -- `headless_agent: "openclaw"`: use `openclaw agent --local --agent "" --message "" --json --timeout 1800` from the configured OpenClaw project workspace. The explicit agent selector is required by the CLI and binds the turn to the verified project workspace. Omit `--deliver`, `--channel`, `--reply-channel`, `--reply-to`, and `--to` so automation cannot send the result to a channel. - -For Codex-owned headless automation, never write automation that shells out to `claude` or `claude -p`. Do not fall back from Codex automation to Claude if `codex` is missing; report the missing `codex` CLI instead. - -For Claude-owned headless automation, never write automation that shells out to `codex exec`. Do not fall back from Claude automation to Codex if `claude` is missing; report the missing `claude` CLI instead. - -For Pi-owned headless automation, never write automation that shells out to `codex exec` or `claude -p`. Do not fall back from Pi automation to Claude or Codex if `pi` is missing; report the missing `pi` CLI instead. - -For OpenClaw-owned headless automation, never write automation that shells out to `codex exec`, `claude -p`, or `pi`. Do not fall back from OpenClaw automation to another agent if `openclaw` is missing; report the missing `openclaw` CLI or project-workspace mismatch instead. - -Codex refresh script shape: +The scheduler and post-commit hook must use the configured +`.llm-wiki/config.json` `headless_agent`. Do not hand-write provider wrappers. +Resolve the installed `llm-wiki` package root by walking up from the active +skill until the ancestor containing both `templates/` and the package manifest +is found. This works for native and generated host skill paths. Copy these +canonical files from `/templates/`, make them executable, and keep +the same copies in `.llm-wiki/`: + +- `post-commit-refresh.sh` +- `refresh-wiki.sh` +- `compile-log.sh` +- `install-systemd-scheduler.sh` + +Copy the runner, compiler, and validated config into +`$(git rev-parse --git-common-dir)/llm-wiki/`. Wire the common `post-commit` +hook to call the shared runner with +`--project "$(git rev-parse --show-toplevel)"`, falling back to the checkout +copy only when the shared runner is absent. This makes the primary checkout's +installed runtime authoritative for every linked worktree, including older +branches with stale ignored `.llm-wiki` files or config. + +After the separate automation approval, on Linux run: ```bash -#!/usr/bin/env bash -set -euo pipefail -project_root="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" -cd "$project_root" - -codex exec -C "$project_root" "Refresh this project's LLM wiki. Read .llm-wiki/config.json, AGENTS.md, wiki/index.md, wiki/gaps.md, and recent wiki/log.md entries first. If .llm-wiki/config.json contains main_wiki_path, search that exact path before changing project pages. Also search default main cross-project wiki paths when they exist: ~/wikis/master/wiki/, ~/wikis/main/wiki/, ../wikis/master/wiki/, and ../wikis/main/wiki/. Inspect recent git history and changed source files. Update stale wiki pages, update wiki/index.md when page coverage changes, append wiki/log.md, and record uncertainty in wiki/gaps.md. Do not invent facts." +.llm-wiki/install-systemd-scheduler.sh --project "$(git rev-parse --show-toplevel)" ``` -Claude Code refresh script shape: - -```bash -#!/usr/bin/env bash -set -euo pipefail -project_root="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" -cd "$project_root" - -claude -p "Refresh this project's LLM wiki. Read .llm-wiki/config.json, CLAUDE.md, wiki/index.md, wiki/gaps.md, and recent wiki/log.md entries first. If .llm-wiki/config.json contains main_wiki_path, search that exact path before changing project pages. Also search default main cross-project wiki paths when they exist: ~/wikis/master/wiki/, ~/wikis/main/wiki/, ../wikis/master/wiki/, and ../wikis/main/wiki/. Inspect recent git history and changed source files. Update stale wiki pages, update wiki/index.md when page coverage changes, append wiki/log.md, and record uncertainty in wiki/gaps.md. Do not invent facts." --allowedTools "Bash,Read,Edit,Write" --max-budget-usd 0.50 -``` - -Pi refresh script shape: - -```bash -#!/usr/bin/env bash -set -euo pipefail -project_root="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" -cd "$project_root" - -pi -p --no-session --tools read,bash,edit,write,grep,find,ls "Refresh this project's LLM wiki. Read .llm-wiki/config.json, AGENTS.md, wiki/index.md, wiki/gaps.md, and recent wiki/log.md entries first. If .llm-wiki/config.json contains main_wiki_path, search that exact path before changing project pages. Also search default main cross-project wiki paths when they exist: ~/wikis/master/wiki/, ~/wikis/main/wiki/, ../wikis/master/wiki/, and ../wikis/main/wiki/. Inspect recent git history and changed source files. Update stale wiki pages, update wiki/index.md when page coverage changes, append wiki/log.md, and record uncertainty in wiki/gaps.md. Do not invent facts." -``` - -OpenClaw refresh script shape: - -```bash -#!/usr/bin/env bash -set -euo pipefail -project_root="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" -cd "$project_root" - -openclaw agent --local --agent "" --message "Refresh this project's LLM wiki. Read .llm-wiki/config.json, AGENTS.md, wiki/index.md, wiki/gaps.md, and recent wiki/log.md entries first. If .llm-wiki/config.json contains main_wiki_path, search that exact path before changing project pages. Also search default main cross-project wiki paths when they exist: ~/wikis/master/wiki/, ~/wikis/main/wiki/, ../wikis/master/wiki/, and ../wikis/main/wiki/. Inspect recent git history and changed source files. Update stale wiki pages, update wiki/index.md when page coverage changes, add a wiki/log.d/-.md fragment without editing compiled wiki/log.md, and record uncertainty in wiki/gaps.md. Do not invent facts." --json --timeout 1800 -``` - -After that separate approval, install the best available scheduler: - -- Linux with systemd user services: create `~/.config/systemd/user/llm-wiki-.service` and `.timer`, then run `systemctl --user daemon-reload` and `systemctl --user enable --now llm-wiki-.timer`. -- macOS with launchd: create `~/Library/LaunchAgents/com.llm-wiki..plist` with a 24 hour `StartInterval`, then run `launchctl load`. -- Other environments: install an equivalent cron entry that runs `.llm-wiki/refresh-wiki.sh` daily. - -Use a stable `` from the repository basename plus a short hash of the project root to avoid timer name collisions. Replace existing `llm-wiki-` scheduler files instead of adding duplicates. For cron, wrap the entry with `# BEGIN LLM WIKI ` and `# END LLM WIKI ` markers and replace that block on repeat bootstrap. If scheduler installation fails because the environment lacks systemd, launchd, cron, or permissions, keep `.llm-wiki/refresh-wiki.sh`, record the failure in `wiki/gaps.md`, and report the exact command the user can run. - -Also install post-commit wiki maintenance automation. Preserve existing hooks; do not overwrite unrelated hook logic. Install the canonical runtime in the shared Git directory and wire the common `post-commit` hook to pass the committing worktree explicitly. - -Install `.llm-wiki/post-commit-refresh.sh` AND `.llm-wiki/compile-log.sh` by copying the reference scripts bundled with this skill at `templates/post-commit-refresh.sh` and `templates/compile-log.sh` (resolve them relative to this SKILL.md), then `chmod +x` both. Also copy the same files verbatim to `$(git rev-parse --git-common-dir)/llm-wiki/post-commit-refresh.sh` and `compile-log.sh`, and copy the validated project config there as `config.json`. The hook must invoke that shared runner as `post-commit-refresh.sh --project "$(git rev-parse --show-toplevel)"`, falling back to the checkout-local runner only when the shared copy is absent. This makes one bootstrap or upgrade authoritative for every linked worktree, including older branches with stale ignored `.llm-wiki` files or config. `compile-log.sh` is the single source of truth for the changelog format: it regenerates `wiki/log.md` from the append-only `wiki/log.d/*.md` fragments, and the refresh runs it before committing. (Hive's `Hive::WikiLog` delegates here, so Ruby and shell callers share one implementation.) The bundled post-commit script reads the canonical shared `headless_agent` and dispatches to exactly one provider; never customize its provider function per project. Provider, QMD, and Git ref execution requires `timeout` or `gtimeout` so every potentially stuck command is bounded. When neither is available, the worker must fail before starting a provider. A repository-wide circuit stops automatic provider launches after two consecutive failed batches or when more than 25 sources are pending by default. A worker handles at most one batch of 10 sources with bounded path context; sources arriving outside that snapshot open a `deferred:` circuit. New sources continue queueing until an operator explicitly runs `.llm-wiki/post-commit-refresh.sh --retry-failed ` once per bounded batch. +The installer reconciles all linked worktrees to one non-persistent timer for +the repository's primary checkout, removes and stops obsolete managed units, +serializes all repositories through one machine-wide lock, and enforces a 4 GiB +memory limit with no swap. The timer drains already queued commits; it does not +launch a direct refresh against whichever checkout happened to install it. +When automation is not approved, do not enable a timer; the upgrade path uses +`--disabled` to reconcile old units without activating maintenance. On +non-systemd platforms, leave the scripts installed, record the scheduler gap in +`wiki/gaps.md`, and report the manual drain command. + +Preserve existing hooks and unrelated hook logic. The bundled runner reads the +canonical shared owner config and dispatches to exactly one of Claude Code, +Codex, Pi, or OpenClaw. It never falls back to a different provider. An +OpenClaw owner requires a validated `openclaw_agent_id` and never uses delivery, +channel, reply, or recipient flags. Provider, QMD, and Git ref execution is +time-bounded. A repository-wide circuit stops automatic provider launches after +two consecutive failed batches or more than 25 pending sources. Each worker +handles at most one batch of 10 sources, while the scheduler keeps draining +later batches without starting concurrent provider processes. The supported provider set is Claude Code, Codex, Pi, and OpenClaw. The copied config must preserve `openclaw_agent_id` when OpenClaw owns maintenance; the runtime rejects an unsupported owner or a missing/invalid OpenClaw agent ID @@ -336,13 +312,13 @@ instead of falling back to another provider. Transactional refresh contract (the bundled script implements all of these; any hand-edit must keep them): - **User checkouts are read-only inputs.** Queue each relevant source SHA under `$(git rev-parse --git-common-dir)/llm-wiki/pending/` before attempting the worker lock. Never use the committing checkout or first/main checkout as an agent workspace, log destination, QMD cache, staging area, or commit target. -- **Use one managed refresh branch and disposable worktree.** Drain queued commits on local branch `llm-wiki/refresh` in `/llm-wiki/refresh-worktree`, based/rebased on the current default branch. The agent inspects source commits with `git show`; it writes only under the managed worktree's `wiki/`. +- **Use one published refresh branch and disposable worktree.** Drain queued commits on `llm-wiki/refresh` in `/llm-wiki/refresh-worktree`, based/rebased on the current remote default branch. The agent inspects source commits with `git show`; it writes only under the managed worktree's `wiki/`. Publish only this branch to `origin`; never dirty or push the protected default branch. - **Seed ignored local wikis once.** When a new refresh branch has no tracked `wiki/`, copy the committing checkout's untracked local wiki into the disposable worktree before invoking the agent. Never replace an already-established refresh-branch wiki, follow a top-level `wiki` symlink, or copy any path outside `wiki/`. - **Serialize and coalesce.** Hold one stale-reclaimable compare-and-swap lock at `refs/llm-wiki/refresh-lock`. Its Git blob records PID, time, process-start identity, and nonce; `git update-ref ` makes stale replacement single-winner, while ownership-checked deletion prevents an old worker from releasing a successor's lock. Check the wait deadline and sleep after every failed acquisition attempt. One worker snapshots at most 10 queued SHAs and runs one refresh agent for that batch. A busy worker leaves new queue entries intact for a later worker instead of dropping them. - **Receipt completed sources.** Add one exact `LLM-Wiki-Source: ` commit-message paragraph when the batch creates a wiki commit, then atomically update `refs/llm-wiki/receipts/` for every successful source before queue deletion. Check receipt refs first and commit history as migration fallback before invoking the agent. This makes both changed and no-op batches replay-safe after a crash between completion and acknowledgement. -- **Pin queued commits.** Create `refs/llm-wiki/sources/` when a source is queued and delete it in the same ref transaction that writes its durable receipt. Backfill pins for pre-upgrade pending and quarantined records. Refuse to invoke or acknowledge a batch whose selected SHA is not an available commit. +- **Pin queued commits in bounded transactions.** Create `refs/llm-wiki/sources/` when a source is queued and delete it in the same ref transaction that writes its durable receipt. Backfill pins for pre-upgrade pending and quarantined records in batches of 64 by default (`LLM_WIKI_MAX_SOURCE_PIN_BATCH`). Reconstruct empty crash-left `..` queue files when their source commit is available; retain unavailable records for diagnosis. Refuse to invoke or acknowledge a batch whose selected SHA is not an available commit. - **Recover stale locks safely.** A live owner PID with the recorded process-start identity wins. Dead, PID-reused, or malformed owner blobs are replaceable only through the Git ref's compare-and-swap old-OID guard. -- **Validate before committing.** Reject any tracked, untracked, or ignored change outside `wiki/`. Compile `wiki/log.md`, force-stage only `wiki/` so intentionally ignored wikis persist, and commit with both recursion guards (`HIVE_SKIP_LLM_WIKI_POST_COMMIT=1` and `git -c core.hooksPath=/dev/null`). Never push automatically. +- **Validate before committing and publishing.** Reject any tracked, untracked, or ignored change outside `wiki/`. Compile `wiki/log.md`, force-stage only `wiki/` so intentionally ignored wikis persist, and commit with both recursion guards (`HIVE_SKIP_LLM_WIKI_POST_COMMIT=1` and `git -c core.hooksPath=/dev/null`). Fetch and merge the remote refresh branch, then push only `llm-wiki/refresh`; retain local work when fetch, merge, or push fails. - **Failure is clean and bounded.** If agent execution, wiki-only validation, compilation, staging, or commit fails, force-remove the disposable managed worktree. After two consecutive failed batches by default, move the active batch to `/llm-wiki/failed/` and open the repository-wide circuit. Continue queueing new sources without launching a provider. Never delete failed source data or automatically run a quarantined source again. User checkout bytes and branch refs must remain unchanged. - **Subscription use is bounded.** Run provider overrides, Codex, Claude Code, Pi, QMD, and Git ref operations through `timeout` or `gtimeout`. If no bounded diff --git a/plugins/llm-wiki/pi/skills/wiki-upgrade/SKILL.md b/plugins/llm-wiki/pi/skills/wiki-upgrade/SKILL.md index e854e39..cfc6d4e 100644 --- a/plugins/llm-wiki/pi/skills/wiki-upgrade/SKILL.md +++ b/plugins/llm-wiki/pi/skills/wiki-upgrade/SKILL.md @@ -12,8 +12,9 @@ metadata: Upgrade the current project's managed llm-wiki files to the templates bundled with this installed release. This is a narrow, deterministic migration: do not -regenerate wiki pages, switch the headless owner, run a refresh agent, update -QMD, or reinstall the scheduler. +regenerate wiki pages, switch the headless owner, run a refresh agent, or update +QMD. It deterministically reconciles the managed scheduler and leaves it +disabled unless both automation consent flags are true. ## Preconditions @@ -45,10 +46,15 @@ not a migration failure. The script upgrades only managed structure: - `.llm-wiki/post-commit-refresh.sh` +- `.llm-wiki/refresh-wiki.sh` - `.llm-wiki/compile-log.sh` -- canonical copies of both scripts under +- `.llm-wiki/install-systemd-scheduler.sh` +- canonical copies of the runner and compiler under `$(git rev-parse --git-common-dir)/llm-wiki/` for all linked worktrees - canonical shared owner config and keepalive refs for existing queued commits +- bounded source-pin backfill, including recoverable interrupted queue writes +- one bounded repository-wide systemd timer, or the same reconciled units left + disabled when automation consent is absent - a missing `.llm-wiki/config.json` when live or historical evidence identifies one unambiguous legacy owner - `wiki/log.d/` and the compiled `wiki/log.md` layout @@ -64,7 +70,7 @@ checkout-local copy remains a manual entrypoint and compatibility fallback. ## Safety Contract - Preserve unrelated tracked and untracked project changes. -- Overwrite only the two llm-wiki-managed scripts listed above. +- Overwrite only the four llm-wiki-managed scripts listed above. - Replace or deduplicate only the marked post-commit hook block; preserve all unmarked hook logic and refuse unmatched markers. - Preserve all hand-written legacy changelog prose while introducing the diff --git a/plugins/llm-wiki/skills/bootstrap/SKILL.md b/plugins/llm-wiki/skills/bootstrap/SKILL.md index 810efc4..29f5a7d 100644 --- a/plugins/llm-wiki/skills/bootstrap/SKILL.md +++ b/plugins/llm-wiki/skills/bootstrap/SKILL.md @@ -243,6 +243,8 @@ OpenClaw context: - Ensure `AGENTS.md` contains the wiki section from Step 5. OpenClaw auto-injects the `AGENTS.md` in its configured agent workspace at the start of every session. - Confirm the project root is the active OpenClaw agent workspace before selecting `openclaw` as `headless_agent`. If it is not, report the workspace mismatch instead of claiming the project `AGENTS.md` will be injected. - Do not add channel-delivery flags to wiki maintenance commands. Scheduled and post-commit refreshes are local automation, not chat replies. +- Omit `--deliver`, `--channel`, `--reply-channel`, `--reply-to`, and `--to` + from every OpenClaw maintenance command. Do not install scheduled or post-commit automation unless the user separately approves it after seeing the provider, files, hooks, and activation commands. @@ -251,78 +253,52 @@ refresh commands instead. If the current tool is not the configured `headless_agent`, update session context for the current tool and validate/report the existing automation owner. Do not rewrite scheduler or post-commit ownership unless automation is missing, unsafe, or the user asks to repair or switch ownership. -The scheduler must use the configured `.llm-wiki/config.json` `headless_agent`. - -Create `.llm-wiki/refresh-wiki.sh` and make it executable. It should run the configured headless agent's CLI from the project root: - -- `headless_agent: "codex"`: use `codex exec -C "" ""`. -- `headless_agent: "claude"`: use `claude -p ""` with the same refresh intent. -- `headless_agent: "pi"`: use `pi -p --no-session --tools read,bash,edit,write,grep,find,ls ""` with the same refresh intent. -- `headless_agent: "openclaw"`: use `openclaw agent --local --agent "" --message "" --json --timeout 1800` from the configured OpenClaw project workspace. The explicit agent selector is required by the CLI and binds the turn to the verified project workspace. Omit `--deliver`, `--channel`, `--reply-channel`, `--reply-to`, and `--to` so automation cannot send the result to a channel. - -For Codex-owned headless automation, never write automation that shells out to `claude` or `claude -p`. Do not fall back from Codex automation to Claude if `codex` is missing; report the missing `codex` CLI instead. - -For Claude-owned headless automation, never write automation that shells out to `codex exec`. Do not fall back from Claude automation to Codex if `claude` is missing; report the missing `claude` CLI instead. - -For Pi-owned headless automation, never write automation that shells out to `codex exec` or `claude -p`. Do not fall back from Pi automation to Claude or Codex if `pi` is missing; report the missing `pi` CLI instead. - -For OpenClaw-owned headless automation, never write automation that shells out to `codex exec`, `claude -p`, or `pi`. Do not fall back from OpenClaw automation to another agent if `openclaw` is missing; report the missing `openclaw` CLI or project-workspace mismatch instead. - -Codex refresh script shape: +The scheduler and post-commit hook must use the configured +`.llm-wiki/config.json` `headless_agent`. Do not hand-write provider wrappers. +Resolve the installed `llm-wiki` package root by walking up from the active +skill until the ancestor containing both `templates/` and the package manifest +is found. This works for native and generated host skill paths. Copy these +canonical files from `/templates/`, make them executable, and keep +the same copies in `.llm-wiki/`: + +- `post-commit-refresh.sh` +- `refresh-wiki.sh` +- `compile-log.sh` +- `install-systemd-scheduler.sh` + +Copy the runner, compiler, and validated config into +`$(git rev-parse --git-common-dir)/llm-wiki/`. Wire the common `post-commit` +hook to call the shared runner with +`--project "$(git rev-parse --show-toplevel)"`, falling back to the checkout +copy only when the shared runner is absent. This makes the primary checkout's +installed runtime authoritative for every linked worktree, including older +branches with stale ignored `.llm-wiki` files or config. + +After the separate automation approval, on Linux run: ```bash -#!/usr/bin/env bash -set -euo pipefail -project_root="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" -cd "$project_root" - -codex exec -C "$project_root" "Refresh this project's LLM wiki. Read .llm-wiki/config.json, AGENTS.md, wiki/index.md, wiki/gaps.md, and recent wiki/log.md entries first. If .llm-wiki/config.json contains main_wiki_path, search that exact path before changing project pages. Also search default main cross-project wiki paths when they exist: ~/wikis/master/wiki/, ~/wikis/main/wiki/, ../wikis/master/wiki/, and ../wikis/main/wiki/. Inspect recent git history and changed source files. Update stale wiki pages, update wiki/index.md when page coverage changes, append wiki/log.md, and record uncertainty in wiki/gaps.md. Do not invent facts." +.llm-wiki/install-systemd-scheduler.sh --project "$(git rev-parse --show-toplevel)" ``` -Claude Code refresh script shape: - -```bash -#!/usr/bin/env bash -set -euo pipefail -project_root="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" -cd "$project_root" - -claude -p "Refresh this project's LLM wiki. Read .llm-wiki/config.json, CLAUDE.md, wiki/index.md, wiki/gaps.md, and recent wiki/log.md entries first. If .llm-wiki/config.json contains main_wiki_path, search that exact path before changing project pages. Also search default main cross-project wiki paths when they exist: ~/wikis/master/wiki/, ~/wikis/main/wiki/, ../wikis/master/wiki/, and ../wikis/main/wiki/. Inspect recent git history and changed source files. Update stale wiki pages, update wiki/index.md when page coverage changes, append wiki/log.md, and record uncertainty in wiki/gaps.md. Do not invent facts." --allowedTools "Bash,Read,Edit,Write" --max-budget-usd 0.50 -``` - -Pi refresh script shape: - -```bash -#!/usr/bin/env bash -set -euo pipefail -project_root="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" -cd "$project_root" - -pi -p --no-session --tools read,bash,edit,write,grep,find,ls "Refresh this project's LLM wiki. Read .llm-wiki/config.json, AGENTS.md, wiki/index.md, wiki/gaps.md, and recent wiki/log.md entries first. If .llm-wiki/config.json contains main_wiki_path, search that exact path before changing project pages. Also search default main cross-project wiki paths when they exist: ~/wikis/master/wiki/, ~/wikis/main/wiki/, ../wikis/master/wiki/, and ../wikis/main/wiki/. Inspect recent git history and changed source files. Update stale wiki pages, update wiki/index.md when page coverage changes, append wiki/log.md, and record uncertainty in wiki/gaps.md. Do not invent facts." -``` - -OpenClaw refresh script shape: - -```bash -#!/usr/bin/env bash -set -euo pipefail -project_root="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" -cd "$project_root" - -openclaw agent --local --agent "" --message "Refresh this project's LLM wiki. Read .llm-wiki/config.json, AGENTS.md, wiki/index.md, wiki/gaps.md, and recent wiki/log.md entries first. If .llm-wiki/config.json contains main_wiki_path, search that exact path before changing project pages. Also search default main cross-project wiki paths when they exist: ~/wikis/master/wiki/, ~/wikis/main/wiki/, ../wikis/master/wiki/, and ../wikis/main/wiki/. Inspect recent git history and changed source files. Update stale wiki pages, update wiki/index.md when page coverage changes, add a wiki/log.d/-.md fragment without editing compiled wiki/log.md, and record uncertainty in wiki/gaps.md. Do not invent facts." --json --timeout 1800 -``` - -After that separate approval, install the best available scheduler: - -- Linux with systemd user services: create `~/.config/systemd/user/llm-wiki-.service` and `.timer`, then run `systemctl --user daemon-reload` and `systemctl --user enable --now llm-wiki-.timer`. -- macOS with launchd: create `~/Library/LaunchAgents/com.llm-wiki..plist` with a 24 hour `StartInterval`, then run `launchctl load`. -- Other environments: install an equivalent cron entry that runs `.llm-wiki/refresh-wiki.sh` daily. - -Use a stable `` from the repository basename plus a short hash of the project root to avoid timer name collisions. Replace existing `llm-wiki-` scheduler files instead of adding duplicates. For cron, wrap the entry with `# BEGIN LLM WIKI ` and `# END LLM WIKI ` markers and replace that block on repeat bootstrap. If scheduler installation fails because the environment lacks systemd, launchd, cron, or permissions, keep `.llm-wiki/refresh-wiki.sh`, record the failure in `wiki/gaps.md`, and report the exact command the user can run. - -Also install post-commit wiki maintenance automation. Preserve existing hooks; do not overwrite unrelated hook logic. Install the canonical runtime in the shared Git directory and wire the common `post-commit` hook to pass the committing worktree explicitly. - -Install `.llm-wiki/post-commit-refresh.sh` AND `.llm-wiki/compile-log.sh` by copying the reference scripts bundled with this skill at `templates/post-commit-refresh.sh` and `templates/compile-log.sh` (resolve them relative to this SKILL.md), then `chmod +x` both. Also copy the same files verbatim to `$(git rev-parse --git-common-dir)/llm-wiki/post-commit-refresh.sh` and `compile-log.sh`, and copy the validated project config there as `config.json`. The hook must invoke that shared runner as `post-commit-refresh.sh --project "$(git rev-parse --show-toplevel)"`, falling back to the checkout-local runner only when the shared copy is absent. This makes one bootstrap or upgrade authoritative for every linked worktree, including older branches with stale ignored `.llm-wiki` files or config. `compile-log.sh` is the single source of truth for the changelog format: it regenerates `wiki/log.md` from the append-only `wiki/log.d/*.md` fragments, and the refresh runs it before committing. (Hive's `Hive::WikiLog` delegates here, so Ruby and shell callers share one implementation.) The bundled post-commit script reads the canonical shared `headless_agent` and dispatches to exactly one provider; never customize its provider function per project. Provider, QMD, and Git ref execution requires `timeout` or `gtimeout` so every potentially stuck command is bounded. When neither is available, the worker must fail before starting a provider. A repository-wide circuit stops automatic provider launches after two consecutive failed batches or when more than 25 sources are pending by default. A worker handles at most one batch of 10 sources with bounded path context; sources arriving outside that snapshot open a `deferred:` circuit. New sources continue queueing until an operator explicitly runs `.llm-wiki/post-commit-refresh.sh --retry-failed ` once per bounded batch. +The installer reconciles all linked worktrees to one non-persistent timer for +the repository's primary checkout, removes and stops obsolete managed units, +serializes all repositories through one machine-wide lock, and enforces a 4 GiB +memory limit with no swap. The timer drains already queued commits; it does not +launch a direct refresh against whichever checkout happened to install it. +When automation is not approved, do not enable a timer; the upgrade path uses +`--disabled` to reconcile old units without activating maintenance. On +non-systemd platforms, leave the scripts installed, record the scheduler gap in +`wiki/gaps.md`, and report the manual drain command. + +Preserve existing hooks and unrelated hook logic. The bundled runner reads the +canonical shared owner config and dispatches to exactly one of Claude Code, +Codex, Pi, or OpenClaw. It never falls back to a different provider. An +OpenClaw owner requires a validated `openclaw_agent_id` and never uses delivery, +channel, reply, or recipient flags. Provider, QMD, and Git ref execution is +time-bounded. A repository-wide circuit stops automatic provider launches after +two consecutive failed batches or more than 25 pending sources. Each worker +handles at most one batch of 10 sources, while the scheduler keeps draining +later batches without starting concurrent provider processes. The supported provider set is Claude Code, Codex, Pi, and OpenClaw. The copied config must preserve `openclaw_agent_id` when OpenClaw owns maintenance; the runtime rejects an unsupported owner or a missing/invalid OpenClaw agent ID @@ -331,13 +307,13 @@ instead of falling back to another provider. Transactional refresh contract (the bundled script implements all of these; any hand-edit must keep them): - **User checkouts are read-only inputs.** Queue each relevant source SHA under `$(git rev-parse --git-common-dir)/llm-wiki/pending/` before attempting the worker lock. Never use the committing checkout or first/main checkout as an agent workspace, log destination, QMD cache, staging area, or commit target. -- **Use one managed refresh branch and disposable worktree.** Drain queued commits on local branch `llm-wiki/refresh` in `/llm-wiki/refresh-worktree`, based/rebased on the current default branch. The agent inspects source commits with `git show`; it writes only under the managed worktree's `wiki/`. +- **Use one published refresh branch and disposable worktree.** Drain queued commits on `llm-wiki/refresh` in `/llm-wiki/refresh-worktree`, based/rebased on the current remote default branch. The agent inspects source commits with `git show`; it writes only under the managed worktree's `wiki/`. Publish only this branch to `origin`; never dirty or push the protected default branch. - **Seed ignored local wikis once.** When a new refresh branch has no tracked `wiki/`, copy the committing checkout's untracked local wiki into the disposable worktree before invoking the agent. Never replace an already-established refresh-branch wiki, follow a top-level `wiki` symlink, or copy any path outside `wiki/`. - **Serialize and coalesce.** Hold one stale-reclaimable compare-and-swap lock at `refs/llm-wiki/refresh-lock`. Its Git blob records PID, time, process-start identity, and nonce; `git update-ref ` makes stale replacement single-winner, while ownership-checked deletion prevents an old worker from releasing a successor's lock. Check the wait deadline and sleep after every failed acquisition attempt. One worker snapshots at most 10 queued SHAs and runs one refresh agent for that batch. A busy worker leaves new queue entries intact for a later worker instead of dropping them. - **Receipt completed sources.** Add one exact `LLM-Wiki-Source: ` commit-message paragraph when the batch creates a wiki commit, then atomically update `refs/llm-wiki/receipts/` for every successful source before queue deletion. Check receipt refs first and commit history as migration fallback before invoking the agent. This makes both changed and no-op batches replay-safe after a crash between completion and acknowledgement. -- **Pin queued commits.** Create `refs/llm-wiki/sources/` when a source is queued and delete it in the same ref transaction that writes its durable receipt. Backfill pins for pre-upgrade pending and quarantined records. Refuse to invoke or acknowledge a batch whose selected SHA is not an available commit. +- **Pin queued commits in bounded transactions.** Create `refs/llm-wiki/sources/` when a source is queued and delete it in the same ref transaction that writes its durable receipt. Backfill pins for pre-upgrade pending and quarantined records in batches of 64 by default (`LLM_WIKI_MAX_SOURCE_PIN_BATCH`). Reconstruct empty crash-left `..` queue files when their source commit is available; retain unavailable records for diagnosis. Refuse to invoke or acknowledge a batch whose selected SHA is not an available commit. - **Recover stale locks safely.** A live owner PID with the recorded process-start identity wins. Dead, PID-reused, or malformed owner blobs are replaceable only through the Git ref's compare-and-swap old-OID guard. -- **Validate before committing.** Reject any tracked, untracked, or ignored change outside `wiki/`. Compile `wiki/log.md`, force-stage only `wiki/` so intentionally ignored wikis persist, and commit with both recursion guards (`HIVE_SKIP_LLM_WIKI_POST_COMMIT=1` and `git -c core.hooksPath=/dev/null`). Never push automatically. +- **Validate before committing and publishing.** Reject any tracked, untracked, or ignored change outside `wiki/`. Compile `wiki/log.md`, force-stage only `wiki/` so intentionally ignored wikis persist, and commit with both recursion guards (`HIVE_SKIP_LLM_WIKI_POST_COMMIT=1` and `git -c core.hooksPath=/dev/null`). Fetch and merge the remote refresh branch, then push only `llm-wiki/refresh`; retain local work when fetch, merge, or push fails. - **Failure is clean and bounded.** If agent execution, wiki-only validation, compilation, staging, or commit fails, force-remove the disposable managed worktree. After two consecutive failed batches by default, move the active batch to `/llm-wiki/failed/` and open the repository-wide circuit. Continue queueing new sources without launching a provider. Never delete failed source data or automatically run a quarantined source again. User checkout bytes and branch refs must remain unchanged. - **Subscription use is bounded.** Run provider overrides, Codex, Claude Code, Pi, QMD, and Git ref operations through `timeout` or `gtimeout`. If no bounded diff --git a/plugins/llm-wiki/skills/upgrade/SKILL.md b/plugins/llm-wiki/skills/upgrade/SKILL.md index 0f2bbff..b297159 100644 --- a/plugins/llm-wiki/skills/upgrade/SKILL.md +++ b/plugins/llm-wiki/skills/upgrade/SKILL.md @@ -7,8 +7,9 @@ description: Upgrade an existing project's managed llm-wiki structure without re Upgrade the current project's managed llm-wiki files to the templates bundled with this installed release. This is a narrow, deterministic migration: do not -regenerate wiki pages, switch the headless owner, run a refresh agent, update -QMD, or reinstall the scheduler. +regenerate wiki pages, switch the headless owner, run a refresh agent, or update +QMD. It deterministically reconciles the managed scheduler and leaves it +disabled unless both automation consent flags are true. ## Preconditions @@ -40,10 +41,15 @@ not a migration failure. The script upgrades only managed structure: - `.llm-wiki/post-commit-refresh.sh` +- `.llm-wiki/refresh-wiki.sh` - `.llm-wiki/compile-log.sh` -- canonical copies of both scripts under +- `.llm-wiki/install-systemd-scheduler.sh` +- canonical copies of the runner and compiler under `$(git rev-parse --git-common-dir)/llm-wiki/` for all linked worktrees - canonical shared owner config and keepalive refs for existing queued commits +- bounded source-pin backfill, including recoverable interrupted queue writes +- one bounded repository-wide systemd timer, or the same reconciled units left + disabled when automation consent is absent - a missing `.llm-wiki/config.json` when live or historical evidence identifies one unambiguous legacy owner - `wiki/log.d/` and the compiled `wiki/log.md` layout @@ -59,7 +65,7 @@ checkout-local copy remains a manual entrypoint and compatibility fallback. ## Safety Contract - Preserve unrelated tracked and untracked project changes. -- Overwrite only the two llm-wiki-managed scripts listed above. +- Overwrite only the four llm-wiki-managed scripts listed above. - Replace or deduplicate only the marked post-commit hook block; preserve all unmarked hook logic and refuse unmatched markers. - Preserve all hand-written legacy changelog prose while introducing the diff --git a/plugins/llm-wiki/skills/upgrade/scripts/upgrade-project.sh b/plugins/llm-wiki/skills/upgrade/scripts/upgrade-project.sh index bf2b75c..6ca3154 100755 --- a/plugins/llm-wiki/skills/upgrade/scripts/upgrade-project.sh +++ b/plugins/llm-wiki/skills/upgrade/scripts/upgrade-project.sh @@ -197,8 +197,10 @@ fi script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" plugin_root="$(cd "$script_dir/../../.." && pwd)" post_template="$plugin_root/templates/post-commit-refresh.sh" +refresh_template="$plugin_root/templates/refresh-wiki.sh" compile_template="$plugin_root/templates/compile-log.sh" -for template in "$post_template" "$compile_template"; do +scheduler_template="$plugin_root/templates/install-systemd-scheduler.sh" +for template in "$post_template" "$refresh_template" "$compile_template" "$scheduler_template"; do if [ ! -f "$template" ]; then printf 'llm-wiki: bundled template missing: %s\n' "$template" >&2 exit 1 @@ -377,8 +379,10 @@ render_migrated_log() { } pin_existing_queued_sources() { - local path name queued_sha queued_branch update_line + local path name queued_sha update_line batch_size batch_count=0 local updates="$tmp_dir/source-pin-updates" transaction="$tmp_dir/source-pins" + batch_size="${LLM_WIKI_MAX_SOURCE_PIN_BATCH:-64}" + [[ "$batch_size" =~ ^[1-9][0-9]*$ ]] || batch_size=64 { shopt -s nullglob for path in \ @@ -388,8 +392,7 @@ pin_existing_queued_sources() { [ -f "$path" ] || continue name="$(basename "$path")" queued_sha="" - queued_branch="" - IFS=$'\t' read -r queued_sha queued_branch <"$path" || true + IFS=$'\t' read -r queued_sha _ <"$path" || true # A pre-upgrade worker may have stopped after writing the atomic temp but # before renaming it into the visible queue. Pin only temps that the # runtime can recover: a strict .. name whose record starts with @@ -410,14 +413,23 @@ pin_existing_queued_sources() { done shopt -u nullglob } | LC_ALL=C sort -u >"$updates" - { - printf 'start\n' - while IFS= read -r update_line; do - printf '%s\n' "$update_line" - done <"$updates" - printf 'commit\n' - } >"$transaction" - git -C "$root" update-ref --stdin <"$transaction" + : >"$transaction" + while IFS= read -r update_line; do + if [ "$batch_count" -eq 0 ]; then + printf 'start\n' >"$transaction" + fi + printf '%s\n' "$update_line" >>"$transaction" + batch_count=$((batch_count + 1)) + if [ "$batch_count" -ge "$batch_size" ]; then + printf 'commit\n' >>"$transaction" + git -C "$root" update-ref --stdin <"$transaction" || return 1 + batch_count=0 + fi + done <"$updates" + if [ "$batch_count" -gt 0 ]; then + printf 'commit\n' >>"$transaction" + git -C "$root" update-ref --stdin <"$transaction" + fi } process_identity() { @@ -495,6 +507,18 @@ if [ ! -x "$root/.llm-wiki/post-commit-refresh.sh" ] || \ post_needs_upgrade=1 changes+=(".llm-wiki/post-commit-refresh.sh") fi +refresh_needs_upgrade=0 +if [ ! -x "$root/.llm-wiki/refresh-wiki.sh" ] || \ + ! cmp -s "$refresh_template" "$root/.llm-wiki/refresh-wiki.sh"; then + refresh_needs_upgrade=1 + changes+=(".llm-wiki/refresh-wiki.sh") +fi +scheduler_script_needs_upgrade=0 +if [ ! -x "$root/.llm-wiki/install-systemd-scheduler.sh" ] || \ + ! cmp -s "$scheduler_template" "$root/.llm-wiki/install-systemd-scheduler.sh"; then + scheduler_script_needs_upgrade=1 + changes+=(".llm-wiki/install-systemd-scheduler.sh") +fi compile_needs_upgrade=0 if [ ! -x "$root/.llm-wiki/compile-log.sh" ] || \ ! cmp -s "$compile_template" "$root/.llm-wiki/compile-log.sh"; then @@ -527,6 +551,22 @@ if [ ! -x "$hook_path" ] || ! cmp -s "$rendered_hook" "$hook_path"; then hook_needs_upgrade=1 changes+=("post-commit hook") fi +scheduler_args=(--project "$root") +if ! grep -Eq '"automation_enabled"[[:space:]]*:[[:space:]]*true' "$canonical_config_source" 2>/dev/null || \ + ! grep -Eq '"external_provider_access_approved"[[:space:]]*:[[:space:]]*true' "$canonical_config_source" 2>/dev/null; then + scheduler_args+=(--disabled) +fi +scheduler_status=0 +"$scheduler_template" --check "${scheduler_args[@]}" >/dev/null 2>&1 || scheduler_status=$? +case "$scheduler_status" in + 0) ;; + 10) changes+=("bounded repository-wide systemd scheduler") ;; + 20) changes+=("bounded repository-wide systemd scheduler (blocked: flock is required)") ;; + *) + printf 'llm-wiki: could not inspect the systemd scheduler\n' >&2 + exit 1 + ;; +esac if [ "$mode" = check ]; then if [ "${#changes[@]}" -eq 0 ]; then @@ -561,6 +601,12 @@ fi if [ "$post_needs_upgrade" -eq 1 ]; then install -m 0755 "$post_template" "$root/.llm-wiki/post-commit-refresh.sh" fi +if [ "$refresh_needs_upgrade" -eq 1 ]; then + install -m 0755 "$refresh_template" "$root/.llm-wiki/refresh-wiki.sh" +fi +if [ "$scheduler_script_needs_upgrade" -eq 1 ]; then + install -m 0755 "$scheduler_template" "$root/.llm-wiki/install-systemd-scheduler.sh" +fi if [ "$compile_needs_upgrade" -eq 1 ]; then install -m 0755 "$compile_template" "$root/.llm-wiki/compile-log.sh" fi @@ -573,6 +619,14 @@ fi if [ "$shared_config_needs_upgrade" -eq 1 ]; then install -m 0644 "$canonical_config_source" "$shared_config_path" fi +scheduler_apply_status=0 +"$root/.llm-wiki/install-systemd-scheduler.sh" "${scheduler_args[@]}" || scheduler_apply_status=$? +if [ "$scheduler_apply_status" -eq 20 ]; then + printf 'llm-wiki: warning: managed files were upgraded, but flock is required before the bounded systemd scheduler can be installed\n' >&2 +elif [ "$scheduler_apply_status" -ne 0 ]; then + printf 'llm-wiki: could not install the bounded systemd scheduler\n' >&2 + exit "$scheduler_apply_status" +fi if [ "$log_needs_migration" -eq 1 ]; then install -m 0644 "$compiled_log" "$root/wiki/log.md" fi diff --git a/plugins/llm-wiki/templates/install-systemd-scheduler.sh b/plugins/llm-wiki/templates/install-systemd-scheduler.sh new file mode 100755 index 0000000..e6d44ab --- /dev/null +++ b/plugins/llm-wiki/templates/install-systemd-scheduler.sh @@ -0,0 +1,187 @@ +#!/usr/bin/env bash +set -euo pipefail + +mode=apply +project=. +force_disabled=0 +while [ "$#" -gt 0 ]; do + case "$1" in + --check) mode=check; shift ;; + --disabled) force_disabled=1; shift ;; + --project) + [ "$#" -ge 2 ] || { printf 'Usage: %s [--check] [--disabled] [--project ]\n' "$0" >&2; exit 2; } + project="$2" + shift 2 + ;; + *) printf 'Usage: %s [--check] [--disabled] [--project ]\n' "$0" >&2; exit 2 ;; + esac +done + +[ "$(uname -s 2>/dev/null || true)" = Linux ] || exit 0 +root="$(git -C "$project" rev-parse --show-toplevel 2>/dev/null || true)" +[ -n "$root" ] || { printf 'llm-wiki: scheduler project is not a Git worktree\n' >&2; exit 1; } +primary_root="$( + git -C "$root" worktree list --porcelain -z 2>/dev/null | + while IFS= read -r -d '' field; do + case "$field" in + worktree\ *) printf '%s\n' "${field#worktree }"; break ;; + esac + done +)" +[ -n "$primary_root" ] || { printf 'llm-wiki: could not resolve the repository primary worktree\n' >&2; exit 1; } +common_dir="$(git -C "$primary_root" rev-parse --path-format=absolute --git-common-dir)" +shared_runner="$common_dir/llm-wiki/post-commit-refresh.sh" + +user_dir="${LLM_WIKI_SYSTEMD_USER_DIR:-${XDG_CONFIG_HOME:-$HOME/.config}/systemd/user}" +flock_path="${LLM_WIKI_FLOCK_PATH:-$(command -v flock 2>/dev/null || true)}" +if [ -z "$flock_path" ] || [ ! -x "$flock_path" ]; then + printf 'llm-wiki: flock is required for the bounded systemd scheduler\n' >&2 + exit 20 +fi + +digest() { + if command -v sha256sum >/dev/null 2>&1; then + printf '%s' "$1" | sha256sum | awk '{print substr($1,1,8)}' + else + printf '%s' "$1" | shasum -a 256 | awk '{print substr($1,1,8)}' + fi +} + +systemd_path() { + printf '%s' "$1" | + sed -e 's/\\/\\x5c/g' -e 's/ /\\x20/g' -e 's/"/\\x22/g' +} + +decode_systemd_path() { + printf '%s' "$1" | + sed -e 's/\\x22/"/g' -e 's/\\x20/ /g' -e 's/\\x5c/\\/g' +} + +base="$(basename "$primary_root" | sed 's/[^A-Za-z0-9_.-]/-/g')" +slug="$base-$(digest "$primary_root")" +service_name="llm-wiki-$slug.service" +timer_name="llm-wiki-$slug.timer" +service_path="$user_dir/$service_name" +timer_path="$user_dir/$timer_name" +wants_dir="$user_dir/timers.target.wants" +wants_path="$wants_dir/$timer_name" +encoded_root="$(systemd_path "$primary_root")" +encoded_runner="$(systemd_path "$shared_runner")" +encoded_flock="$(systemd_path "$flock_path")" + +tmp_dir="$(mktemp -d)" +trap 'rm -rf "$tmp_dir"' EXIT +service_rendered="$tmp_dir/service" +timer_rendered="$tmp_dir/timer" +cat >"$service_rendered" <"$timer_rendered" </dev/null || true)" + configured="$(printf '%s\n' "$contents" | sed -n 's/^WorkingDirectory=//p' | head -n 1)" + [ -n "$configured" ] || continue + configured="$(decode_systemd_path "$configured")" + candidate_common="$(git -C "$configured" rev-parse --path-format=absolute --git-common-dir 2>/dev/null || true)" + [ "$candidate_common" = "$common_dir" ] || continue + if ! printf '%s\n' "$contents" | grep -Fqx 'X-LLMWikiManaged=yes' && \ + ! printf '%s\n' "$contents" | grep -Eq '^Description=Refresh LLM wiki for llm-wiki-'; then + continue + fi + had_managed_unit=1 + candidate_timer="${candidate%.service}.timer" + [ -L "$wants_dir/$(basename "$candidate_timer")" ] && had_enabled_unit=1 + if [ "$candidate" != "$service_path" ]; then + obsolete+=("$candidate" "$candidate_timer" "$wants_dir/$(basename "$candidate_timer")") + fi +done +shopt -u nullglob + +enable_timer=1 +if [ "$force_disabled" -eq 1 ]; then + enable_timer=0 +elif [ "$had_managed_unit" -eq 1 ] && [ "$had_enabled_unit" -eq 0 ] && [ ! -L "$wants_path" ]; then + enable_timer=0 +fi + +drift=0 +[ -f "$service_path" ] && cmp -s "$service_rendered" "$service_path" || drift=1 +[ -f "$timer_path" ] && cmp -s "$timer_rendered" "$timer_path" || drift=1 +[ -f "$common_dir/llm-wiki/scheduler-service" ] && \ + [ "$(sed -n '1p' "$common_dir/llm-wiki/scheduler-service")" = "$service_name" ] || drift=1 +[ "${#obsolete[@]}" -eq 0 ] || drift=1 +if [ "$enable_timer" -eq 1 ]; then + [ -L "$wants_path" ] && [ "$(readlink "$wants_path")" = "../$timer_name" ] || drift=1 +elif [ -L "$wants_path" ]; then + drift=1 +fi + +if [ "$mode" = check ]; then + [ "$drift" -eq 0 ] && exit 0 + printf 'llm-wiki: systemd scheduler upgrade available: %s\n' "$primary_root" + exit 10 +fi +[ "$drift" -eq 1 ] || exit 0 + +mkdir -p "$user_dir" "$wants_dir" "$common_dir/llm-wiki" +install -m 0644 "$service_rendered" "$service_path" +install -m 0644 "$timer_rendered" "$timer_path" +printf '%s\n' "$service_name" >"$common_dir/llm-wiki/scheduler-service" +for candidate in "${obsolete[@]}"; do + [ -e "$candidate" ] || [ -L "$candidate" ] || continue + if [ "${LLM_WIKI_SKIP_SYSTEMCTL:-${HIVE_SKIP_LLM_WIKI_SYSTEMCTL:-}}" != 1 ] && \ + command -v systemctl >/dev/null 2>&1; then + case "$candidate" in + *.service|*.timer) + systemctl --user stop "$(basename "$candidate")" >/dev/null 2>&1 || true + ;; + esac + fi + rm -f -- "$candidate" +done +if [ "$enable_timer" -eq 1 ]; then + ln -sfn "../$timer_name" "$wants_path" +else + rm -f -- "$wants_path" +fi + +if [ "${LLM_WIKI_SKIP_SYSTEMCTL:-${HIVE_SKIP_LLM_WIKI_SYSTEMCTL:-}}" != 1 ] && \ + command -v systemctl >/dev/null 2>&1; then + systemctl --user daemon-reload + systemctl --user stop "$service_name" >/dev/null 2>&1 || true + systemctl --user stop "$timer_name" >/dev/null 2>&1 || true + if [ "$enable_timer" -eq 1 ]; then + systemctl --user start "$timer_name" + fi +fi +printf 'llm-wiki: installed one bounded scheduler for repository: %s\n' "$primary_root" diff --git a/plugins/llm-wiki/templates/post-commit-refresh.sh b/plugins/llm-wiki/templates/post-commit-refresh.sh index 9bf6f71..91786f1 100755 --- a/plugins/llm-wiki/templates/post-commit-refresh.sh +++ b/plugins/llm-wiki/templates/post-commit-refresh.sh @@ -1,6 +1,8 @@ #!/usr/bin/env bash set -euo pipefail +# LLM_WIKI_RUNNER_CAPABILITIES: drain + # Transactional LLM-wiki post-commit refresh. # # Every relevant commit is queued in the shared Git directory. One worker @@ -11,30 +13,44 @@ set -euo pipefail project_override="" retry_selector="" +drain_mode=0 + +usage() { + printf 'Usage: %s [--project ] [--drain | --retry-failed ]\n' "$0" +} + while [ "$#" -gt 0 ]; do case "$1" in --project) [ "$#" -ge 2 ] || { - printf 'Usage: %s [--project ] [--retry-failed ]\n' "$0" >&2 + usage >&2 exit 2 } project_override="$2" shift 2 ;; + --drain) + drain_mode=1 + shift + ;; --retry-failed) [ "$#" -ge 2 ] || { - printf 'Usage: %s [--project ] [--retry-failed ]\n' "$0" >&2 + usage >&2 exit 2 } retry_selector="$2" shift 2 ;; *) - printf 'Usage: %s [--project ] [--retry-failed ]\n' "$0" >&2 + usage >&2 exit 2 ;; esac done +if [ "$drain_mode" -eq 1 ] && [ -n "$retry_selector" ]; then + printf 'llm-wiki: --drain and --retry-failed are mutually exclusive\n' >&2 + exit 2 +fi if [ "$retry_selector" != all ] && [ -n "$retry_selector" ] && \ [[ ! "$retry_selector" =~ ^[0-9a-fA-F]{40,64}$ ]]; then printf 'llm-wiki: retry selector must be a full source SHA or all\n' >&2 @@ -103,13 +119,19 @@ pending_dir="$state_dir/pending" failed_dir="$state_dir/failed" failure_count_file="$state_dir/consecutive-failures" breaker_file="$state_dir/refresh-disabled" +publication_blocked_file="$state_dir/publication-blocked" canonical_config="$state_dir/config.json" lock_ref="${LLM_WIKI_LOCK_REF:-refs/llm-wiki/refresh-lock}" source_ref_prefix="refs/llm-wiki/sources" refresh_root="$state_dir/refresh-worktree" log_file="$state_dir/post-commit-refresh.log" refresh_branch="${LLM_WIKI_REFRESH_BRANCH:-llm-wiki/refresh}" +refresh_remote="${LLM_WIKI_REFRESH_REMOTE:-origin}" +remote_refresh_ref="refs/llm-wiki/remotes/refresh" +remote_base_ref="refs/llm-wiki/remotes/base" lock_owner_oid="" +global_lock_fd="" +global_lock_keeper_pid="" owner_config="$canonical_config" [ -f "$owner_config" ] || owner_config="$committing_tree/.llm-wiki/config.json" if ! grep -Eq '"automation_enabled"[[:space:]]*:[[:space:]]*true' "$owner_config" 2>/dev/null || \ @@ -132,6 +154,7 @@ positive_integer_or_default() { max_auto_pending="$(positive_integer_or_default "${LLM_WIKI_MAX_AUTO_PENDING:-25}" 25)" max_batch_sources="$(positive_integer_or_default "${LLM_WIKI_MAX_BATCH_SOURCES:-10}" 10)" +max_source_pin_batch="$(positive_integer_or_default "${LLM_WIKI_MAX_SOURCE_PIN_BATCH:-64}" 64)" max_paths_per_source="$(positive_integer_or_default "${LLM_WIKI_MAX_PATHS_PER_SOURCE:-20}" 20)" max_path_bytes="$(positive_integer_or_default "${LLM_WIKI_MAX_PATH_BYTES:-200}" 200)" @@ -158,7 +181,7 @@ run_qmd() { sha="$(git rev-parse HEAD 2>/dev/null || true)" [ -n "$sha" ] || exit 0 -if [ -z "$retry_selector" ]; then +if [ "$drain_mode" -eq 0 ] && [ -z "$retry_selector" ]; then changed_files="$(git diff-tree --no-commit-id --name-only -r HEAD 2>/dev/null || true)" [ -n "$changed_files" ] || exit 0 @@ -193,6 +216,21 @@ if [ -z "$retry_selector" ]; then log_line "automatic refresh circuit is open; source $sha queued" exit 0 fi + + scheduler_service="$(sed -n '1p' "$state_dir/scheduler-service" 2>/dev/null || true)" + if [ "${LLM_WIKI_SKIP_SYSTEMCTL:-${HIVE_SKIP_LLM_WIKI_SYSTEMCTL:-}}" != "1" ] && \ + [[ "$scheduler_service" =~ ^llm-wiki-[A-Za-z0-9_.-]+\.service$ ]]; then + if command -v systemctl >/dev/null 2>&1 && \ + run_with_timeout "${LLM_WIKI_SYSTEMCTL_TIMEOUT:-10}" \ + systemctl --user start --no-block "$scheduler_service" \ + >>"$log_file" 2>&1; then + log_line "queued source $sha for memory-bounded systemd worker $scheduler_service" + exit 0 + else + rm -f -- "$state_dir/scheduler-service" + log_line "WARN: memory-bounded systemd worker $scheduler_service could not be started; using machine-wide serialized fallback" + fi + fi fi # From this point onward every Git command is nested maintenance work against @@ -208,6 +246,46 @@ process_identity() { fi } +acquire_global_provider_lock() { + local uid runtime_dir lock_file lock_status + [ "${LLM_WIKI_GLOBAL_LOCK_HELD:-}" = "1" ] && return 0 + + uid="${UID:-$(id -u)}" + runtime_dir="${XDG_RUNTIME_DIR:-}" + if [ -z "$runtime_dir" ] && [ -d "/run/user/$uid" ] && [ -w "/run/user/$uid" ]; then + runtime_dir="/run/user/$uid" + fi + runtime_dir="${runtime_dir:-${TMPDIR:-/tmp}}" + lock_file="$runtime_dir/llm-wiki-refresh.lock" + + if [ "${LLM_WIKI_DISABLE_FLOCK:-}" != "1" ] && command -v flock >/dev/null 2>&1; then + exec {global_lock_fd}>"$lock_file" + flock --nonblock "$global_lock_fd" + return + fi + + command -v ruby >/dev/null 2>&1 || return 1 + coproc LLM_WIKI_GLOBAL_LOCK_KEEPER { + # Hooks can inherit Bundler/Coverage loader state even when PATH resolves a + # different system Ruby. This helper only needs Ruby's core File API. + unset RUBYOPT RUBYLIB BUNDLER_SETUP BUNDLE_GEMFILE BUNDLE_BIN_PATH \ + GEM_HOME GEM_PATH RUBYGEMS_GEMDEPS + ruby -e ' + lock = File.open(ARGV.fetch(0), File::RDWR | File::CREAT, 0o600) + if lock.flock(File::LOCK_EX | File::LOCK_NB) + STDOUT.puts("locked") + STDOUT.flush + STDIN.read + else + STDOUT.puts("busy") + end + ' "$lock_file" + } + global_lock_keeper_pid="$LLM_WIKI_GLOBAL_LOCK_KEEPER_PID" + IFS= read -r lock_status <&"${LLM_WIKI_GLOBAL_LOCK_KEEPER[0]}" || lock_status="" + [ "$lock_status" = "locked" ] +} + lock_owner_stale() { local owner_oid="$1" owner pid identity current_identity owner="$(git cat-file -p "$owner_oid" 2>/dev/null || true)" @@ -266,17 +344,23 @@ pending_sources_present() { [ "$(pending_source_count)" -gt 0 ] } +queue_temp_source_sha() { + local name + name="$(basename "$1")" + [[ "$name" =~ ^\.([0-9a-fA-F]{40,64})\.[0-9]+$ ]] || return 1 + printf '%s\n' "${BASH_REMATCH[1]}" +} + recover_queue_temps() { - local path name queued_sha queued_sha_in_file queued_branch target + local path name queued_sha queued_sha_in_file queued_branch target recovery_tmp shopt -s nullglob for path in "$pending_dir"/.[0-9a-fA-F]*.*; do [ -f "$path" ] || continue name="$(basename "$path")" - if [[ ! "$name" =~ ^\.([0-9a-fA-F]{40,64})\.[0-9]+$ ]]; then + queued_sha="$(queue_temp_source_sha "$path")" || { log_line "WARN: unrecognized queue temp retained: $name" continue - fi - queued_sha="${BASH_REMATCH[1]}" + } target="$pending_dir/$queued_sha" if [ -f "$target" ]; then rm -f -- "$path" @@ -285,7 +369,21 @@ recover_queue_temps() { fi IFS=$'\t' read -r queued_sha_in_file queued_branch <"$path" || true if [ "$queued_sha_in_file" != "$queued_sha" ]; then - log_line "WARN: incomplete queue temp retained: $name" + if ! git cat-file -e "${queued_sha}^{commit}" 2>/dev/null; then + log_line "WARN: incomplete queue temp retained because its source commit is unavailable: $name" + continue + fi + recovery_tmp="$state_dir/.queue-recovery.${queued_sha}.$$" + printf '%s\tinterrupted-write\n' "$queued_sha" >"$recovery_tmp" + if ! git diff-tree --no-commit-id --name-only -r "$queued_sha" \ + >>"$recovery_tmp" 2>/dev/null; then + rm -f -- "$recovery_tmp" + log_line "WARN: incomplete queue temp retained because its source paths could not be read: $name" + continue + fi + mv -f -- "$recovery_tmp" "$target" + rm -f -- "$path" + log_line "reconstructed interrupted queue write for source $queued_sha" continue fi mv -f -- "$path" "$target" @@ -305,29 +403,52 @@ open_backlog_circuit_if_needed() { return 0 } -pin_queued_sources() { - local path queued_sha queued_branch +run_update_ref_transaction() { + [ "$#" -gt 0 ] || return 0 { printf 'start\n' - shopt -s nullglob - for path in "$pending_dir"/* "$failed_dir"/*; do - [ -f "$path" ] || continue - queued_sha="" - queued_branch="" - IFS=$'\t' read -r queued_sha queued_branch <"$path" || true - if [[ "$queued_sha" =~ ^[0-9a-fA-F]{40,64}$ ]] && \ - git cat-file -e "${queued_sha}^{commit}" 2>/dev/null; then - printf 'update %s/%s %s\n' "$source_ref_prefix" "$queued_sha" "$queued_sha" - else - log_line "ERROR: queued source is not an available commit: $queued_sha" - fi - done - shopt -u nullglob + printf '%s\n' "$@" printf 'commit\n' } | run_with_timeout "${LLM_WIKI_GIT_REF_TIMEOUT:-5}" \ git update-ref --stdin >>"$log_file" 2>&1 } +pin_source_ref_batch() { + local queued_sha + local commands=() + for queued_sha in "$@"; do + commands+=("update $source_ref_prefix/$queued_sha $queued_sha") + done + run_update_ref_transaction "${commands[@]}" +} + +pin_queued_sources() { + local path queued_sha queued_branch + local source_shas=() + shopt -s nullglob + for path in "$pending_dir"/* "$failed_dir"/*; do + [ -f "$path" ] || continue + queued_sha="" + queued_branch="" + IFS=$'\t' read -r queued_sha queued_branch <"$path" || true + if [[ "$queued_sha" =~ ^[0-9a-fA-F]{40,64}$ ]] && \ + git cat-file -e "${queued_sha}^{commit}" 2>/dev/null; then + source_shas+=("$queued_sha") + if [ "${#source_shas[@]}" -ge "$max_source_pin_batch" ]; then + pin_source_ref_batch "${source_shas[@]}" || { + shopt -u nullglob + return 1 + } + source_shas=() + fi + else + log_line "ERROR: queued source is not an available commit: $queued_sha" + fi + done + shopt -u nullglob + pin_source_ref_batch "${source_shas[@]}" +} + open_source_pin_circuit() { local breaker_tmp="$state_dir/.refresh-disabled.$$" printf 'source-pin:batch\n' >"$breaker_tmp" @@ -350,6 +471,8 @@ reconcile_circuit_after_success() { printf 'quarantined:%s\n' "$failed_count" >"$breaker_tmp" mv -f "$breaker_tmp" "$breaker_file" log_line "refresh succeeded, but $failed_count quarantined source(s) remain; automatic refresh stays disabled" + elif [ "$pending_count" -gt 0 ] && [ "$drain_mode" -eq 1 ]; then + log_line "scheduled refresh left $pending_count newly arrived source(s) queued for the next bounded drain" elif [ "$pending_count" -gt 0 ]; then breaker_tmp="$state_dir/.refresh-disabled.$$" printf 'deferred:%s\n' "$pending_count" >"$breaker_tmp" @@ -400,19 +523,6 @@ restore_failed_sources() { printf 'llm-wiki: retrying %s restored source(s) with queued work\n' "$restored" } -failed_sources_present() { - local file - shopt -s nullglob - for file in "$failed_dir"/*; do - if [ -f "$file" ]; then - shopt -u nullglob - return 0 - fi - done - shopt -u nullglob - return 1 -} - failed_source_count() { local path count=0 shopt -s nullglob @@ -423,6 +533,36 @@ failed_source_count() { printf '%s\n' "$count" } +recoverable_queue_temps_present() { + local path queued_sha queued_sha_in_file + shopt -s nullglob + for path in "$pending_dir"/.[0-9a-fA-F]*.*; do + [ -f "$path" ] || continue + queued_sha="$(queue_temp_source_sha "$path")" || continue + queued_sha_in_file="" + IFS=$'\t' read -r queued_sha_in_file _ <"$path" || true + if [ "$queued_sha_in_file" = "$queued_sha" ] || \ + git cat-file -e "${queued_sha}^{commit}" 2>/dev/null; then + shopt -u nullglob + return 0 + fi + done + shopt -u nullglob + return 1 +} + +# Scheduled runs are queue consumers. Avoid taking the shared Git-ref lock or +# preparing cache state when there is provably nothing they can consume. A +# queued source must still be recovered and pinned while the circuit is open, +# so the breaker is checked only after the worker owns the lock. The no-work +# condition is checked again under the lock below for race safety. +if [ "$drain_mode" -eq 1 ]; then + if ! pending_sources_present && ! recoverable_queue_temps_present; then + log_line "scheduled drain skipped; no queued sources" + exit 0 + fi +fi + if ! acquire_lock; then log_line "refresh remains queued; worker lock was busy" if [ -n "$retry_selector" ]; then @@ -444,6 +584,8 @@ cleanup_refresh_worktree() { # shellcheck disable=SC2329 cleanup() { cleanup_refresh_worktree + [ -n "$global_lock_keeper_pid" ] && kill "$global_lock_keeper_pid" 2>/dev/null || true + [ -n "$global_lock_keeper_pid" ] && wait "$global_lock_keeper_pid" 2>/dev/null || true [ -n "$lock_owner_oid" ] && \ run_with_timeout "${LLM_WIKI_GIT_REF_TIMEOUT:-5}" \ git update-ref -d "$lock_ref" "$lock_owner_oid" \ @@ -454,7 +596,7 @@ trap cleanup EXIT default_base_ref() { local candidate candidate="$(git symbolic-ref -q refs/remotes/origin/HEAD 2>/dev/null || true)" - for candidate in "$candidate" refs/remotes/origin/main refs/remotes/origin/master refs/heads/main refs/heads/master; do + for candidate in "$remote_base_ref" "$candidate" refs/remotes/origin/main refs/remotes/origin/master refs/heads/main refs/heads/master; do [ -n "$candidate" ] || continue if git rev-parse --verify --quiet "${candidate}^{commit}" >/dev/null; then printf '%s\n' "$candidate" @@ -464,24 +606,138 @@ default_base_ref() { printf '%s\n' "$sha" } +sync_remote_base_ref() { + local symref_output branch_ref candidate + git update-ref -d "$remote_base_ref" >/dev/null 2>&1 || true + publication_required || return 0 + + symref_output="$( + run_with_timeout "${LLM_WIKI_GIT_FETCH_TIMEOUT:-120}" \ + git ls-remote --symref "$refresh_remote" HEAD 2>>"$log_file" || true + )" + branch_ref="$(printf '%s\n' "$symref_output" | awk '$1 == "ref:" && $3 == "HEAD" { print $2; exit }')" + if [ -z "$branch_ref" ]; then + for candidate in main master; do + if run_with_timeout "${LLM_WIKI_GIT_FETCH_TIMEOUT:-120}" \ + git ls-remote --exit-code --heads "$refresh_remote" "refs/heads/$candidate" \ + >>"$log_file" 2>&1; then + branch_ref="refs/heads/$candidate" + break + fi + done + fi + if [ -z "$branch_ref" ]; then + log_line "ERROR: could not resolve the default branch on $refresh_remote; queue retained" + return 1 + fi + + run_with_timeout "${LLM_WIKI_GIT_FETCH_TIMEOUT:-120}" \ + git fetch --no-tags "$refresh_remote" "$branch_ref:$remote_base_ref" \ + >>"$log_file" 2>&1 || { + log_line "ERROR: could not fetch $refresh_remote default branch; queue retained" + return 1 + } +} + +publication_required() { + [ "${LLM_WIKI_SKIP_PUSH:-}" != "1" ] && \ + git remote get-url "$refresh_remote" >/dev/null 2>&1 +} + +sync_remote_refresh_ref() { + local remote_head status + git update-ref -d "$remote_refresh_ref" >/dev/null 2>&1 || true + publication_required || return 0 + + if remote_head="$( + run_with_timeout "${LLM_WIKI_GIT_FETCH_TIMEOUT:-120}" \ + git ls-remote --exit-code --heads "$refresh_remote" \ + "refs/heads/$refresh_branch" 2>>"$log_file" + )"; then + [ -n "$remote_head" ] || return 0 + else + status=$? + if [ "$status" -eq 2 ]; then + return 0 + fi + log_line "ERROR: could not inspect $refresh_remote/$refresh_branch; queue retained" + return 1 + fi + + run_with_timeout "${LLM_WIKI_GIT_FETCH_TIMEOUT:-120}" \ + git fetch --no-tags "$refresh_remote" \ + "refs/heads/$refresh_branch:$remote_refresh_ref" \ + >>"$log_file" 2>&1 || { + log_line "ERROR: could not fetch $refresh_remote/$refresh_branch; queue retained" + return 1 + } +} + +merge_refresh_ref() { + local merge_ref="$1" description="$2" blocked_tmp + git -C "$refresh_root" merge-base --is-ancestor "$merge_ref" HEAD 2>/dev/null && return 0 + if HIVE_SKIP_LLM_WIKI_POST_COMMIT=1 \ + git -C "$refresh_root" -c core.hooksPath=/dev/null \ + merge --no-edit "$merge_ref" >>"$log_file" 2>&1; then + return 0 + fi + + HIVE_SKIP_LLM_WIKI_POST_COMMIT=1 \ + git -C "$refresh_root" -c core.hooksPath=/dev/null \ + merge --abort >>"$log_file" 2>&1 || true + blocked_tmp="$state_dir/.publication-blocked.$$" + { + printf 'ref=%s\n' "$merge_ref" + printf 'description=%s\n' "$description" + printf 'recorded_at=%s\n' "$(date -u +%Y-%m-%dT%H:%M:%SZ)" + } >"$blocked_tmp" + mv -f "$blocked_tmp" "$publication_blocked_file" + log_line "ERROR: could not merge $description into $refresh_branch; queue retained" + return 1 +} + prepare_refresh_worktree() { local base_ref - base_ref="$(default_base_ref)" cleanup_refresh_worktree mkdir -p "$state_dir" + sync_remote_refresh_ref || return 1 + sync_remote_base_ref || return 1 + base_ref="$(default_base_ref)" if git show-ref --verify --quiet "refs/heads/$refresh_branch"; then git worktree add "$refresh_root" "$refresh_branch" >>"$log_file" 2>&1 || return 1 - if ! HIVE_SKIP_LLM_WIKI_POST_COMMIT=1 \ - git -C "$refresh_root" -c core.hooksPath=/dev/null rebase "$base_ref" >>"$log_file" 2>&1; then - HIVE_SKIP_LLM_WIKI_POST_COMMIT=1 \ - git -C "$refresh_root" -c core.hooksPath=/dev/null rebase --abort >>"$log_file" 2>&1 || true - log_line "ERROR: could not rebase $refresh_branch onto $base_ref; queue retained" - return 1 - fi + elif git show-ref --verify --quiet "$remote_refresh_ref"; then + git worktree add -b "$refresh_branch" "$refresh_root" "$remote_refresh_ref" \ + >>"$log_file" 2>&1 || return 1 else git worktree add -b "$refresh_branch" "$refresh_root" "$base_ref" >>"$log_file" 2>&1 || return 1 fi + + if git show-ref --verify --quiet "$remote_refresh_ref"; then + merge_refresh_ref "$remote_refresh_ref" "$refresh_remote/$refresh_branch" || return 1 + fi + merge_refresh_ref "$base_ref" "$base_ref" || return 1 + rm -f -- "$publication_blocked_file" +} + +publish_refresh_branch() { + if [ "${LLM_WIKI_SKIP_PUSH:-}" = "1" ]; then + log_line "refresh branch push skipped by LLM_WIKI_SKIP_PUSH" + return 0 + fi + if ! git remote get-url "$refresh_remote" >/dev/null 2>&1; then + log_line "refresh branch kept locally; remote $refresh_remote is not configured" + return 0 + fi + + if ! HIVE_SKIP_LLM_WIKI_POST_COMMIT=1 \ + run_with_timeout "${LLM_WIKI_GIT_PUSH_TIMEOUT:-120}" \ + git -C "$refresh_root" -c core.hooksPath=/dev/null \ + push "$refresh_remote" "HEAD:refs/heads/$refresh_branch" \ + >>"$log_file" 2>&1; then + return 2 + fi + git update-ref "$remote_refresh_ref" "$(git -C "$refresh_root" rev-parse HEAD)" } seed_untracked_local_wiki() { @@ -592,26 +848,71 @@ snapshot_queue() { [ "${#QUEUE_FILES[@]}" -gt 0 ] } -source_receipted() { +local_source_commit() { local queued_sha="$1" - git show-ref --verify --quiet "refs/llm-wiki/receipts/$queued_sha" && return 0 - git -C "$refresh_root" log --format=%B 2>/dev/null | \ - grep -Fqx "LLM-Wiki-Source: $queued_sha" + git -C "$refresh_root" log -n 1 --format=%H --fixed-strings \ + --grep="LLM-Wiki-Source: $queued_sha" 2>/dev/null || true +} + +source_receipted() { + local queued_sha="$1" receipt_commit + receipt_commit="$(git rev-parse --verify "refs/llm-wiki/receipts/$queued_sha^{commit}" 2>/dev/null || true)" + if [ -n "$receipt_commit" ]; then + publication_required || return 0 + if git show-ref --verify --quiet "$remote_refresh_ref" && \ + git merge-base --is-ancestor "$receipt_commit" "$remote_refresh_ref"; then + return 0 + fi + fi + receipt_commit="$(local_source_commit "$queued_sha")" + [ -n "$receipt_commit" ] || return 1 + publication_required || return 0 + git show-ref --verify --quiet "$remote_refresh_ref" || return 1 + git merge-base --is-ancestor "$receipt_commit" "$remote_refresh_ref" +} + +publish_retained_sources() { + local file queued_sha queued_branch status + local original=("${QUEUE_FILES[@]}") retained=() remaining=() + for file in "${QUEUE_FILES[@]}"; do + IFS=$'\t' read -r queued_sha queued_branch <"$file" + if [ -n "$(local_source_commit "$queued_sha")" ]; then + retained+=("$file") + else + remaining+=("$file") + fi + done + [ "${#retained[@]}" -gt 0 ] || return 0 + + QUEUE_FILES=("${retained[@]}") + if publish_refresh_branch; then + status=0 + else + status=$? + QUEUE_FILES=("${original[@]}") + log_line "WARN: could not publish retained $refresh_branch commit; queue retained" + return "$status" + fi + if ! write_source_receipts; then + QUEUE_FILES=("${original[@]}") + log_line "ERROR: could not write retained source receipts; queue retained" + return 1 + fi + rm -f -- "${retained[@]}" + QUEUE_FILES=("${remaining[@]}") + log_line "published ${#retained[@]} retained source(s) without another agent run" } write_source_receipts() { local refresh_head file queued_sha queued_branch + local commands=() refresh_head="$(git -C "$refresh_root" rev-parse HEAD)" - { - printf 'start\n' - for file in "${QUEUE_FILES[@]}"; do - IFS=$'\t' read -r queued_sha queued_branch <"$file" - printf 'update refs/llm-wiki/receipts/%s %s\n' "$queued_sha" "$refresh_head" - printf 'delete %s/%s\n' "$source_ref_prefix" "$queued_sha" - done - printf 'commit\n' - } | run_with_timeout "${LLM_WIKI_GIT_REF_TIMEOUT:-5}" \ - git update-ref --stdin >>"$log_file" 2>&1 + for file in "${QUEUE_FILES[@]}"; do + IFS=$'\t' read -r queued_sha queued_branch <"$file" + commands+=("update refs/llm-wiki/receipts/$queued_sha $refresh_head") + commands+=("delete $source_ref_prefix/$queued_sha") + done + run_update_ref_transaction "${commands[@]}" } prune_receipted_queue_files() { @@ -633,7 +934,7 @@ prune_receipted_queue_files() { } record_batch_failure() { - local file queued_sha queued_branch count max_attempts count_tmp remaining=0 + local file queued_sha queued_branch count max_attempts count_tmp unreceipted=0 max_attempts="${LLM_WIKI_MAX_REFRESH_ATTEMPTS:-2}" [[ "$max_attempts" =~ ^[1-9][0-9]*$ ]] || max_attempts=2 @@ -648,9 +949,9 @@ record_batch_failure() { log_line "acknowledged committed source $queued_sha after receipt write failure" continue fi - remaining=1 + unreceipted=1 done - if [ "$remaining" -eq 0 ]; then + if [ "$unreceipted" -eq 0 ]; then rm -f -- "$failure_count_file" reconcile_circuit_after_success return 0 @@ -686,6 +987,13 @@ process_queue_batch() { rm -f -- "$failure_count_file" return 0 fi + retained_status=0 + publish_retained_sources || retained_status=$? + [ "$retained_status" -eq 0 ] || return "$retained_status" + if [ "${#QUEUE_FILES[@]}" -eq 0 ]; then + rm -f -- "$failure_count_file" + return 0 + fi for file in "${QUEUE_FILES[@]}"; do IFS=$'\t' read -r queued_sha queued_branch <"$file" if ! git cat-file -e "${queued_sha}^{commit}" 2>/dev/null; then @@ -767,6 +1075,21 @@ PROMPT log_line "ERROR: refresh commit failed; queue retained" return 1 fi + elif publication_required; then + short_source="$(basename "${QUEUE_FILES[0]}")" + if ! HIVE_SKIP_LLM_WIKI_POST_COMMIT=1 \ + git -C "$refresh_root" -c core.hooksPath=/dev/null \ + commit --allow-empty \ + -m "docs(wiki): acknowledge ${#QUEUE_FILES[@]} commit(s) at ${short_source:0:12}" \ + "${commit_args[@]}" >>"$log_file" 2>&1; then + log_line "ERROR: empty refresh acknowledgement commit failed; queue retained" + return 1 + fi + fi + + if ! publish_refresh_branch; then + log_line "WARN: could not push $refresh_branch to $refresh_remote; generated commit and queue retained" + return 2 fi if ! write_source_receipts; then @@ -780,8 +1103,19 @@ PROMPT return 0 } -configure_qmd_environment recover_queue_temps + +# A scheduled drain is a queue consumer, never a source-discovery trigger. +# Check for work before pinning sources or preparing the managed worktree so an +# idle timer cannot mutate refresh state. Circuit-blocked work is deliberately +# pinned first, preserving queued commits until a later bounded retry. +if [ "$drain_mode" -eq 1 ]; then + if ! pending_sources_present; then + log_line "scheduled drain skipped; no queued sources" + exit 0 + fi +fi + if ! pin_queued_sources; then open_source_pin_circuit if [ -n "$retry_selector" ]; then @@ -794,14 +1128,30 @@ if [ -n "$retry_selector" ]; then if ! restore_failed_sources; then exit 1 fi + rm -f -- "$publication_blocked_file" # A worker can pass the pre-lock check and then wait while its predecessor opens # the circuit. Re-check under the lock before any worktree or provider action. elif [ -f "$breaker_file" ]; then log_line "automatic refresh circuit remains open; queued sources retained" exit 0 +elif [ -f "$publication_blocked_file" ]; then + log_line "automatic refresh publication remains blocked by a merge conflict; queued sources retained; run --retry-failed all after resolving the branch" + exit 0 elif open_backlog_circuit_if_needed; then exit 0 fi +if ! pending_sources_present; then + exit 0 +fi +if ! acquire_global_provider_lock; then + log_line "refresh remains queued; machine-wide provider lock was busy" + if [ -n "$retry_selector" ]; then + printf 'llm-wiki: retry deferred; machine-wide refresh worker is busy\n' >&2 + exit 1 + fi + exit 0 +fi +configure_qmd_environment if ! prepare_refresh_worktree; then log_line "ERROR: could not prepare managed refresh worktree; queue retained" if [ -n "$retry_selector" ]; then @@ -818,8 +1168,19 @@ if ! seed_untracked_local_wiki; then exit 0 fi -if snapshot_queue; then - if ! process_queue_batch; then +drain_batch_count=0 +max_drain_batches="$(positive_integer_or_default "${LLM_WIKI_MAX_DRAIN_BATCHES:-3}" 3)" +while snapshot_queue; do + batch_status=0 + process_queue_batch || batch_status=$? + if [ "$batch_status" -eq 2 ]; then + log_line "refresh publication deferred; it will retry without opening the provider circuit" + if [ -n "$retry_selector" ]; then + printf 'llm-wiki: retry generated the refresh but publication is still deferred; see %s\n' "$log_file" >&2 + exit 1 + fi + exit 0 + elif [ "$batch_status" -ne 0 ]; then record_batch_failure if [ -n "$retry_selector" ]; then printf 'llm-wiki: retry failed; see %s\n' "$log_file" >&2 @@ -827,7 +1188,12 @@ if snapshot_queue; then fi exit 0 fi -fi + drain_batch_count=$((drain_batch_count + 1)) + if [ "$drain_mode" -ne 1 ] || [ "$drain_batch_count" -ge "$max_drain_batches" ]; then + break + fi + sleep "${LLM_WIKI_DRAIN_SETTLE_SECONDS:-1}" +done reconcile_circuit_after_success diff --git a/plugins/llm-wiki/templates/refresh-wiki.sh b/plugins/llm-wiki/templates/refresh-wiki.sh new file mode 100755 index 0000000..a6af646 --- /dev/null +++ b/plugins/llm-wiki/templates/refresh-wiki.sh @@ -0,0 +1,32 @@ +#!/usr/bin/env bash +set -euo pipefail + +# Compatibility entrypoint for daily schedulers. Scheduled runs only drain +# work already queued by post-commit hooks; provider selection and all writes +# remain inside the canonical transactional runner's managed worktree. +project_root="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +project_root="$(git -C "$project_root" rev-parse --show-toplevel 2>/dev/null || true)" +if [ -z "$project_root" ]; then + printf 'llm-wiki: refresh-wiki.sh must be installed inside a Git worktree\n' >&2 + exit 1 +fi + +common_dir="$(git -C "$project_root" rev-parse --path-format=absolute --git-common-dir 2>/dev/null || true)" +shared_runner="$common_dir/llm-wiki/post-commit-refresh.sh" +local_runner="$project_root/.llm-wiki/post-commit-refresh.sh" + +supports_drain() { + [ -x "$1" ] && \ + grep -Fqx '# LLM_WIKI_RUNNER_CAPABILITIES: drain' "$1" 2>/dev/null +} + +if supports_drain "$shared_runner"; then + runner="$shared_runner" +elif supports_drain "$local_runner"; then + runner="$local_runner" +else + printf 'llm-wiki: no drain-capable transactional refresh runner was found; run the llm-wiki upgrade command\n' >&2 + exit 1 +fi + +exec "$runner" --project "$project_root" --drain diff --git a/tests/test_llm_wiki_openclaw.py b/tests/test_llm_wiki_openclaw.py index c9f8643..4b111ab 100644 --- a/tests/test_llm_wiki_openclaw.py +++ b/tests/test_llm_wiki_openclaw.py @@ -10,6 +10,7 @@ REPO_ROOT = Path(__file__).resolve().parents[1] PLUGIN_ROOT = REPO_ROOT / "plugins" / "llm-wiki" POST_COMMIT_TEMPLATE = PLUGIN_ROOT / "templates" / "post-commit-refresh.sh" +SCHEDULER_TEMPLATE = PLUGIN_ROOT / "templates" / "install-systemd-scheduler.sh" UPGRADE_SCRIPT = PLUGIN_ROOT / "skills" / "upgrade" / "scripts" / "upgrade-project.sh" @@ -106,6 +107,114 @@ def test_post_commit_template_requires_explicit_automation_consent(self): self.assertEqual([], arguments) self.assertIn("automatic refresh disabled", result.stderr) + def test_post_commit_runtime_bounds_recovery_and_publishes_only_refresh_branch(self): + template = POST_COMMIT_TEMPLATE.read_text(encoding="utf-8") + + self.assertIn("LLM_WIKI_MAX_SOURCE_PIN_BATCH", template) + self.assertIn("reconstructed interrupted queue write", template) + self.assertIn('refresh_branch="${LLM_WIKI_REFRESH_BRANCH:-llm-wiki/refresh}"', template) + self.assertIn('push "$refresh_remote" "HEAD:refs/heads/$refresh_branch"', template) + self.assertNotIn('push "$refresh_remote" "HEAD:refs/heads/$base_branch"', template) + + def test_scheduler_reconciles_linked_worktrees_and_stops_disabled_units(self): + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) / "project" + linked = Path(directory) / "linked" + unit_dir = Path(directory) / "systemd" + bin_dir = Path(directory) / "bin" + systemctl_log = Path(directory) / "systemctl.log" + root.mkdir() + unit_dir.mkdir() + bin_dir.mkdir() + (root / ".llm-wiki").mkdir() + (root / "wiki").mkdir() + (root / ".llm-wiki" / "config.json").write_text("{}\n", encoding="utf-8") + (root / "wiki" / "index.md").write_text("# Wiki\n", encoding="utf-8") + self.run_git(root, "init", "-b", "main") + self.run_git(root, "config", "user.email", "llm-wiki-test@example.com") + self.run_git(root, "config", "user.name", "LLM Wiki Test") + self.run_git(root, "add", ".") + self.run_git(root, "commit", "-m", "initial") + self.run_git(root, "worktree", "add", "-b", "feature", str(linked)) + + common_dir = Path( + subprocess.run( + ["git", "rev-parse", "--path-format=absolute", "--git-common-dir"], + cwd=root, + text=True, + capture_output=True, + check=True, + ).stdout.strip() + ) + shared_dir = common_dir / "llm-wiki" + shared_dir.mkdir() + shutil.copy2(POST_COMMIT_TEMPLATE, shared_dir / "post-commit-refresh.sh") + (shared_dir / "post-commit-refresh.sh").chmod(0o755) + + fake_systemctl = bin_dir / "systemctl" + fake_systemctl.write_text( + '#!/usr/bin/env bash\nprintf "%s\\n" "$*" >>"$LLM_WIKI_SYSTEMCTL_LOG"\n', + encoding="utf-8", + ) + fake_systemctl.chmod(0o755) + environment = os.environ.copy() + environment.update( + { + "PATH": f"{bin_dir}:/usr/bin:/bin", + "LLM_WIKI_SYSTEMD_USER_DIR": str(unit_dir), + "LLM_WIKI_FLOCK_PATH": shutil.which("flock") or "/usr/bin/flock", + "LLM_WIKI_SYSTEMCTL_LOG": str(systemctl_log), + } + ) + + for project in (root, linked): + result = subprocess.run( + [str(SCHEDULER_TEMPLATE), "--project", str(project)], + env=environment, + text=True, + capture_output=True, + check=False, + ) + self.assertEqual(0, result.returncode, result.stdout + result.stderr) + + services = list(unit_dir.glob("llm-wiki-*.service")) + timers = list(unit_dir.glob("llm-wiki-*.timer")) + self.assertEqual(1, len(services)) + self.assertEqual(1, len(timers)) + service = services[0] + timer = timers[0] + service_text = service.read_text(encoding="utf-8") + timer_text = timer.read_text(encoding="utf-8") + self.assertIn("MemoryMax=4G", service_text) + self.assertIn("MemorySwapMax=0", service_text) + self.assertIn("%t/llm-wiki-refresh.lock", service_text) + self.assertNotIn("Persistent=", timer_text) + + legacy_service = unit_dir / "llm-wiki-linked-legacy.service" + legacy_timer = unit_dir / "llm-wiki-linked-legacy.timer" + legacy_service.write_text( + "[Unit]\nDescription=Refresh LLM wiki for llm-wiki-linked-legacy\n" + f"[Service]\nWorkingDirectory={linked}\n", + encoding="utf-8", + ) + legacy_timer.write_text("[Timer]\nPersistent=true\n", encoding="utf-8") + + result = subprocess.run( + [str(SCHEDULER_TEMPLATE), "--disabled", "--project", str(linked)], + env=environment, + text=True, + capture_output=True, + check=False, + ) + self.assertEqual(0, result.returncode, result.stdout + result.stderr) + self.assertFalse(legacy_service.exists()) + self.assertFalse(legacy_timer.exists()) + self.assertFalse((unit_dir / "timers.target.wants" / timer.name).exists()) + systemctl_calls = systemctl_log.read_text(encoding="utf-8") + self.assertIn("stop llm-wiki-linked-legacy.service", systemctl_calls) + self.assertIn("stop llm-wiki-linked-legacy.timer", systemctl_calls) + self.assertIn(f"stop {timer.name}", systemctl_calls) + def test_post_commit_template_requires_explicit_openclaw_agent_id(self): result, arguments, log = self.run_template("openclaw", openclaw_agent_id=None) self.assertEqual(0, result.returncode, result.stdout + result.stderr) @@ -155,6 +264,76 @@ def test_project_upgrade_accepts_preserved_openclaw_owner(self): self.assertEqual(10, result.returncode, result.stdout + result.stderr) self.assertIn("upgrade available", result.stdout) + def test_project_upgrade_installs_all_runtime_files_without_enabling_unapproved_timer(self): + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) / "project" + unit_dir = Path(directory) / "systemd" + root.mkdir() + unit_dir.mkdir() + (root / ".llm-wiki").mkdir() + (root / "wiki").mkdir() + (root / ".llm-wiki" / "config.json").write_text( + json.dumps( + { + "headless_agent": "codex", + "automation_enabled": False, + "external_provider_access_approved": False, + } + ) + + "\n", + encoding="utf-8", + ) + (root / "wiki" / "index.md").write_text("# Wiki\n", encoding="utf-8") + self.run_git(root, "init", "-b", "main") + self.run_git(root, "config", "user.email", "llm-wiki-test@example.com") + self.run_git(root, "config", "user.name", "LLM Wiki Test") + self.run_git(root, "add", ".") + self.run_git(root, "commit", "-m", "initial") + + environment = os.environ.copy() + environment.update( + { + "LLM_WIKI_SYSTEMD_USER_DIR": str(unit_dir), + "LLM_WIKI_SKIP_SYSTEMCTL": "1", + "LLM_WIKI_FLOCK_PATH": shutil.which("flock") or "/usr/bin/flock", + } + ) + result = subprocess.run( + [str(UPGRADE_SCRIPT), "--project", str(root)], + env=environment, + text=True, + capture_output=True, + check=False, + ) + self.assertEqual(0, result.returncode, result.stdout + result.stderr) + for name in ( + "post-commit-refresh.sh", + "refresh-wiki.sh", + "compile-log.sh", + "install-systemd-scheduler.sh", + ): + self.assertTrue((root / ".llm-wiki" / name).is_file(), name) + self.assertEqual(1, len(list(unit_dir.glob("llm-wiki-*.service")))) + self.assertEqual(1, len(list(unit_dir.glob("llm-wiki-*.timer")))) + self.assertEqual( + [], + list((unit_dir / "timers.target.wants").glob("llm-wiki-*.timer")), + ) + + check_result = subprocess.run( + [str(UPGRADE_SCRIPT), "--check", "--project", str(root)], + env=environment, + text=True, + capture_output=True, + check=False, + ) + self.assertEqual( + 0, + check_result.returncode, + check_result.stdout + check_result.stderr, + ) + self.assertIn("project structure is current", check_result.stdout) + def run_template(self, headless_agent, openclaw_agent_id="main", consent=True): with tempfile.TemporaryDirectory() as directory: root = Path(directory) / "project" diff --git a/wiki/architecture.md b/wiki/architecture.md index 5a76407..76aa241 100644 --- a/wiki/architecture.md +++ b/wiki/architecture.md @@ -38,7 +38,13 @@ generated contract artifact, while a shipped workflow runtime validates JSON collections, pagination, and identifiers before canonical skills act on them. LLM Wiki's shared transactional refresh runner dispatches exactly one configured owner, including a validated OpenClaw workspace agent, from a disposable refresh -worktree. +worktree. The primary checkout owns one repository-wide non-persistent systemd +timer; linked worktrees cannot install independent timers. Services share a +machine-wide provider lock, run with a 4 GiB memory ceiling and no swap, and +drain durable commit-hook queues. Successful wiki-only batches are merged with +and pushed only to `origin/llm-wiki/refresh`, never the protected default branch. +Large queued-source pin sets are processed in bounded Git transactions, and +recoverable interrupted queue files are rebuilt from their source commits. Agent SEO writes new project artifacts by default. Its legacy `scrub` surface is a read-only formatting audit, while live analytics access requires an diff --git a/wiki/gaps.md b/wiki/gaps.md index 37d7105..c427af6 100644 --- a/wiki/gaps.md +++ b/wiki/gaps.md @@ -9,9 +9,10 @@ knowledge remains authoritative here. - Automatic Screenote annotation resolution remains deferred until the mutation is explicitly approved in the CLI contract. -- LLM Wiki 0.3.0 upstream release/tag publication remains a release-management - step. The ClawHub replacement is not verified until the exact merged source - is published and publicly visible. +- LLM Wiki's standalone and four-host marketplace packages intentionally use + separate version lines (`0.1.x` upstream runtime and `0.3.x` consent-gated + marketplace package). A marketplace release must preserve the consent and + OpenClaw adaptations while vendoring the released runtime. - Compatibility with OpenClaw releases older than `2026.7.1-beta.2` remains unverified. ClawHub packages therefore declare that tested version as their conservative plugin API floor while leaving the broader host minimum diff --git a/wiki/log.d/20260722-llm-wiki-032-runtime-safety.md b/wiki/log.d/20260722-llm-wiki-032-runtime-safety.md new file mode 100644 index 0000000..c99497e --- /dev/null +++ b/wiki/log.d/20260722-llm-wiki-032-runtime-safety.md @@ -0,0 +1,11 @@ +# 2026-07-22 — LLM Wiki 0.3.2 runtime safety + +- Vendored the standalone 0.1.16 queue, scheduler, and publication runtime into + the four-host package while preserving 0.3 consent gates and OpenClaw owner + dispatch. +- Replaced checkout-local timer proliferation with one non-persistent, + memory-bounded timer owned by the repository primary checkout. +- Added machine-wide provider serialization, wiki-only refresh-branch + publication, bounded source-pin transactions, and interrupted queue recovery. +- Upgrades now reconcile and stop obsolete units, leaving the timer disabled + unless both automation consent flags are explicitly true. diff --git a/wiki/plugins.md b/wiki/plugins.md index fc76f63..5483435 100644 --- a/wiki/plugins.md +++ b/wiki/plugins.md @@ -8,7 +8,7 @@ OpenClaw. | `agent-reviewer` | `0.3.0` | `agent-reviewer` | agents, references, scripts, eval | | `agent-seo` | `2.0.0` | `seo` | agents, context, data sources, hooks, scripts | | `agent-writing` | `0.5.1` | `writing` | agents, context | -| `llm-wiki` | `0.3.0` | `bootstrap`, `upgrade`, `research`, `wiki-plan`, `wiki-status` | assets, consent-gated templates | +| `llm-wiki` | `0.3.2` | `bootstrap`, `upgrade`, `research`, `wiki-plan`, `wiki-status` | assets, consent-gated templates | | `screenote` | `3.0.0` | `screenote`, `snapshot`, `feedback` | CLI launcher, references, evals | Claude and Codex install through their root marketplaces. Pi and OpenClaw @@ -25,4 +25,7 @@ Version 2.0 makes the removed mutation contract explicit and documents the LLM Wiki bootstrap creates the requested project wiki. Scheduler, managed-hook, shared-Git, and provider-backed maintenance are a separate opt-in; 0.2.x configs without both consent flags are automation-disabled under the 0.3 -runtime. +runtime. Version 0.3.2 reconciles linked checkouts to one non-persistent, +memory-bounded timer per repository, serializes providers across repositories, +publishes wiki-only output to `origin/llm-wiki/refresh`, and bounds source-ref +recovery transactions.