From a146936a1f2b1d1cff7ba388aa719983d47ce84b Mon Sep 17 00:00:00 2001 From: Mattias Petersson Date: Sun, 12 Jul 2026 16:46:44 +0200 Subject: [PATCH 01/12] feat(host): add declarative hosts configuration for Antigravity (Agy) --- hosts/agy.ts | 59 ++++++++++++++++++++++++++++++++++++++++ hosts/index.ts | 5 ++-- test/host-config.test.ts | 6 ++-- 3 files changed, 66 insertions(+), 4 deletions(-) create mode 100644 hosts/agy.ts diff --git a/hosts/agy.ts b/hosts/agy.ts new file mode 100644 index 0000000000..b938b3b1a4 --- /dev/null +++ b/hosts/agy.ts @@ -0,0 +1,59 @@ +import type { HostConfig } from '../scripts/host-config'; + +const agy: HostConfig = { + name: 'agy', + displayName: 'Antigravity', + cliCommand: 'agy', + cliAliases: [], + + globalRoot: '.gemini/config/plugins/gstack/skills', + localSkillRoot: '.gemini/config/plugins/gstack/skills', + hostSubdir: '.gemini/config/plugins/gstack', + usesEnvVars: true, + + frontmatter: { + mode: 'allowlist', + keepFields: [ + 'name', + 'preamble-tier', + 'version', + 'description', + 'allowed-tools', + 'triggers', + 'hooks', + 'gbrain', + ], + descriptionLimit: null, + }, + + generation: { + generateMetadata: false, + skipSkills: ['codex'], + }, + + pathRewrites: [ + { from: '~/.claude/skills/gstack', to: '$GSTACK_ROOT' }, + { from: '.claude/skills/gstack', to: '.gemini/config/plugins/gstack/skills' }, + { from: '.claude/skills', to: '.gemini/config/plugins/gstack' }, + ], + + toolRewrites: {}, + + suppressedResolvers: ['GBRAIN_CONTEXT_LOAD', 'GBRAIN_SAVE_RESULTS'], + + runtimeRoot: { + globalSymlinks: ['bin', 'browse/dist', 'browse/bin', 'gstack-upgrade', 'ETHOS.md'], + globalFiles: { + 'review': ['checklist.md', 'TODOS-format.md'], + }, + }, + + install: { + prefixable: false, + linkingStrategy: 'symlink-generated', + }, + + learningsMode: 'basic', +}; + +export default agy; diff --git a/hosts/index.ts b/hosts/index.ts index cc1c213b53..4cd643ce4a 100644 --- a/hosts/index.ts +++ b/hosts/index.ts @@ -16,9 +16,10 @@ import cursor from './cursor'; import openclaw from './openclaw'; import hermes from './hermes'; import gbrain from './gbrain'; +import agy from './agy'; /** All registered host configs. Add new hosts here. */ -export const ALL_HOST_CONFIGS: HostConfig[] = [claude, codex, factory, kiro, opencode, slate, cursor, openclaw, hermes, gbrain]; +export const ALL_HOST_CONFIGS: HostConfig[] = [claude, codex, factory, kiro, opencode, slate, cursor, openclaw, hermes, gbrain, agy]; /** Map from host name to config. */ export const HOST_CONFIG_MAP: Record = Object.fromEntries( @@ -65,4 +66,4 @@ export function getExternalHosts(): HostConfig[] { } // Re-export individual configs for direct import -export { claude, codex, factory, kiro, opencode, slate, cursor, openclaw, hermes, gbrain }; +export { claude, codex, factory, kiro, opencode, slate, cursor, openclaw, hermes, gbrain, agy }; diff --git a/test/host-config.test.ts b/test/host-config.test.ts index 5770570332..ba7fbc41fc 100644 --- a/test/host-config.test.ts +++ b/test/host-config.test.ts @@ -22,6 +22,7 @@ import { slate, cursor, openclaw, + agy, } from '../hosts/index'; import { HOST_PATHS } from '../scripts/resolvers/types'; @@ -30,8 +31,8 @@ const ROOT = path.resolve(import.meta.dir, '..'); // ─── hosts/index.ts ───────────────────────────────────────── describe('hosts/index.ts', () => { - test('ALL_HOST_CONFIGS has 10 hosts', () => { - expect(ALL_HOST_CONFIGS.length).toBe(10); + test('ALL_HOST_CONFIGS has 11 hosts', () => { + expect(ALL_HOST_CONFIGS.length).toBe(11); }); test('ALL_HOST_NAMES matches config names', () => { @@ -53,6 +54,7 @@ describe('hosts/index.ts', () => { expect(slate.name).toBe('slate'); expect(cursor.name).toBe('cursor'); expect(openclaw.name).toBe('openclaw'); + expect(agy.name).toBe('agy'); }); test('getHostConfig returns correct config', () => { From c4c9f686bcea13342fe54770bc6ff3e2882f1ca4 Mon Sep 17 00:00:00 2001 From: Mattias Petersson Date: Sun, 12 Jul 2026 16:46:46 +0200 Subject: [PATCH 02/12] feat(setup): add agy host support to setup and plugin installation --- .gitignore | 1 + setup | 124 ++++++++++++++++++++++++++++++++++-- test/gen-skill-docs.test.ts | 26 ++++++-- 3 files changed, 143 insertions(+), 8 deletions(-) diff --git a/.gitignore b/.gitignore index 5196c0d05a..50ec63d507 100644 --- a/.gitignore +++ b/.gitignore @@ -23,6 +23,7 @@ bin/gstack-global-discover* .hermes/ .gbrain/ .gbrain-source +.gemini/ .context/ extension/.auth.json # xterm assets are vendored from npm at build time; not source-of-truth. diff --git a/setup b/setup index 275236cd36..c96a311819 100755 --- a/setup +++ b/setup @@ -24,6 +24,9 @@ FACTORY_SKILLS="$HOME/.factory/skills" FACTORY_GSTACK="$FACTORY_SKILLS/gstack" OPENCODE_SKILLS="$HOME/.config/opencode/skills" OPENCODE_GSTACK="$OPENCODE_SKILLS/gstack" +AGY_PLUGINS="$HOME/.gemini/config/plugins" +AGY_GSTACK="$AGY_PLUGINS/gstack" +AGY_SKILLS="$AGY_GSTACK/skills" IS_WINDOWS=0 case "$(uname -s)" in @@ -85,7 +88,7 @@ NO_TEAM_MODE=0 PLAN_TUNE_HOOKS_MODE="" # "" = resolve from env/config/prompt; "yes"/"no" = explicit while [ $# -gt 0 ]; do case "$1" in - --host) [ -z "$2" ] && echo "Missing value for --host (expected claude, codex, kiro, factory, opencode, openclaw, hermes, gbrain, or auto)" >&2 && exit 1; HOST="$2"; shift 2 ;; + --host) [ -z "$2" ] && echo "Missing value for --host (expected claude, codex, kiro, factory, opencode, openclaw, hermes, gbrain, agy, or auto)" >&2 && exit 1; HOST="$2"; shift 2 ;; --host=*) HOST="${1#--host=}"; shift ;; --local) LOCAL_INSTALL=1; shift ;; --prefix) SKILL_PREFIX=1; SKILL_PREFIX_FLAG=1; shift ;; @@ -101,7 +104,7 @@ while [ $# -gt 0 ]; do done case "$HOST" in - claude|codex|kiro|factory|opencode|auto) ;; + claude|codex|kiro|factory|opencode|agy|auto) ;; openclaw) echo "" echo "OpenClaw integration uses a different model — OpenClaw spawns Claude Code" @@ -136,7 +139,7 @@ case "$HOST" in echo "GBrain setup and brain skills ship from the GBrain repo." echo "" exit 0 ;; - *) echo "Unknown --host value: $HOST (expected claude, codex, kiro, factory, opencode, openclaw, hermes, gbrain, or auto)" >&2; exit 1 ;; + *) echo "Unknown --host value: $HOST (expected claude, codex, kiro, factory, opencode, openclaw, hermes, gbrain, agy, or auto)" >&2; exit 1 ;; esac # ─── Resolve skill prefix preference ───────────────────────── @@ -200,14 +203,16 @@ INSTALL_CODEX=0 INSTALL_KIRO=0 INSTALL_FACTORY=0 INSTALL_OPENCODE=0 +INSTALL_AGY=0 if [ "$HOST" = "auto" ]; then command -v claude >/dev/null 2>&1 && INSTALL_CLAUDE=1 command -v codex >/dev/null 2>&1 && INSTALL_CODEX=1 command -v kiro-cli >/dev/null 2>&1 && INSTALL_KIRO=1 command -v droid >/dev/null 2>&1 && INSTALL_FACTORY=1 command -v opencode >/dev/null 2>&1 && INSTALL_OPENCODE=1 + command -v agy >/dev/null 2>&1 && INSTALL_AGY=1 # If none found, default to claude - if [ "$INSTALL_CLAUDE" -eq 0 ] && [ "$INSTALL_CODEX" -eq 0 ] && [ "$INSTALL_KIRO" -eq 0 ] && [ "$INSTALL_FACTORY" -eq 0 ] && [ "$INSTALL_OPENCODE" -eq 0 ]; then + if [ "$INSTALL_CLAUDE" -eq 0 ] && [ "$INSTALL_CODEX" -eq 0 ] && [ "$INSTALL_KIRO" -eq 0 ] && [ "$INSTALL_FACTORY" -eq 0 ] && [ "$INSTALL_OPENCODE" -eq 0 ] && [ "$INSTALL_AGY" -eq 0 ]; then INSTALL_CLAUDE=1 fi elif [ "$HOST" = "claude" ]; then @@ -220,6 +225,8 @@ elif [ "$HOST" = "factory" ]; then INSTALL_FACTORY=1 elif [ "$HOST" = "opencode" ]; then INSTALL_OPENCODE=1 +elif [ "$HOST" = "agy" ]; then + INSTALL_AGY=1 fi migrate_direct_codex_install() { @@ -475,6 +482,16 @@ if [ "$INSTALL_OPENCODE" -eq 1 ] && [ "$NEEDS_BUILD" -eq 0 ]; then ) fi +# 1e. Generate .gemini/ Agy skill docs +if [ "$INSTALL_AGY" -eq 1 ] && [ "$NEEDS_BUILD" -eq 0 ]; then + log "Generating .gemini/ skill docs..." + ( + cd "$SOURCE_GSTACK_DIR" + bun_cmd install --frozen-lockfile 2>/dev/null || bun_cmd install + bun_cmd run gen:skill-docs --host agy + ) +fi + # 2. Ensure Playwright's Chromium is available if ! ensure_playwright_browser; then echo "Installing Playwright Chromium..." @@ -972,6 +989,88 @@ link_opencode_skill_dirs() { fi } +create_agy_runtime_root() { + local gstack_dir="$1" + local agy_gstack="$2" + local agy_dir="$gstack_dir/.gemini/config/plugins/gstack/skills" + + if [ -L "$agy_gstack" ]; then + rm -f "$agy_gstack" + elif [ -d "$agy_gstack" ] && [ "$agy_gstack" != "$gstack_dir" ]; then + rm -rf "$agy_gstack" + fi + + mkdir -p "$agy_gstack" "$agy_gstack/browse" "$agy_gstack/design" "$agy_gstack/gstack-upgrade" "$agy_gstack/review" "$agy_gstack/qa" "$agy_gstack/plan-devex-review" "$agy_gstack/skills" + + # Create plugin.json for agy plugin structure + echo '{"name": "gstack", "version": "1.58.5.0", "description": "gstack skills for Google Antigravity", "author": {"name": "gstack"}, "license": "MIT"}' > "$agy_gstack/plugin.json" + + if [ -d "$gstack_dir/bin" ]; then + _link_or_copy "$gstack_dir/bin" "$agy_gstack/bin" + fi + if [ -d "$gstack_dir/browse/dist" ]; then + _link_or_copy "$gstack_dir/browse/dist" "$agy_gstack/browse/dist" + fi + if [ -d "$gstack_dir/browse/bin" ]; then + _link_or_copy "$gstack_dir/browse/bin" "$agy_gstack/browse/bin" + fi + if [ -d "$gstack_dir/design/dist" ]; then + _link_or_copy "$gstack_dir/design/dist" "$agy_gstack/design/dist" + fi + for f in checklist.md design-checklist.md greptile-triage.md TODOS-format.md; do + if [ -f "$gstack_dir/review/$f" ]; then + _link_or_copy "$gstack_dir/review/$f" "$agy_gstack/review/$f" + fi + done + if [ -d "$gstack_dir/review/specialists" ]; then + _link_or_copy "$gstack_dir/review/specialists" "$agy_gstack/review/specialists" + fi + if [ -d "$gstack_dir/qa/templates" ]; then + _link_or_copy "$gstack_dir/qa/templates" "$agy_gstack/qa/templates" + fi + if [ -d "$gstack_dir/qa/references" ]; then + _link_or_copy "$gstack_dir/qa/references" "$agy_gstack/qa/references" + fi + if [ -f "$gstack_dir/plan-devex-review/dx-hall-of-fame.md" ]; then + _link_or_copy "$gstack_dir/plan-devex-review/dx-hall-of-fame.md" "$agy_gstack/plan-devex-review/dx-hall-of-fame.md" + fi + if [ -f "$gstack_dir/ETHOS.md" ]; then + _link_or_copy "$gstack_dir/ETHOS.md" "$agy_gstack/ETHOS.md" + fi +} + +link_agy_skill_dirs() { + local gstack_dir="$1" + local skills_dir="$2" + local agy_dir="$gstack_dir/.gemini/config/plugins/gstack/skills" + local linked=() + + if [ ! -d "$agy_dir" ]; then + echo " Generating .gemini/ skill docs..." + ( cd "$gstack_dir" && bun run gen:skill-docs --host agy ) + fi + + if [ ! -d "$agy_dir" ]; then + echo " warning: .gemini/config/plugins/gstack/skills/ generation failed — run 'bun run gen:skill-docs --host agy' manually" >&2 + return 1 + fi + + for skill_dir in "$agy_dir"/gstack*/; do + if [ -f "$skill_dir/SKILL.md" ]; then + skill_name="$(basename "$skill_dir")" + [ "$skill_name" = "gstack" ] && continue + target="$skills_dir/$skill_name" + if [ -L "$target" ] || [ ! -e "$target" ]; then + _link_or_copy "$skill_dir" "$target" + linked+=("$skill_name") + fi + fi + done + if [ ${#linked[@]} -gt 0 ]; then + echo " linked skills: ${linked[*]}" + fi +} + # 4. Install for Claude (default) SKILLS_BASENAME="$(basename "$INSTALL_SKILLS_DIR")" SKILLS_PARENT_BASENAME="$(basename "$(dirname "$INSTALL_SKILLS_DIR")")" @@ -1193,6 +1292,23 @@ if [ "$INSTALL_OPENCODE" -eq 1 ]; then echo " opencode skills: $OPENCODE_SKILLS" fi +# 6d. Install for Agy +if [ "$INSTALL_AGY" -eq 1 ]; then + AGY_GSTACK_SOURCE="$SOURCE_GSTACK_DIR/.gemini/config/plugins/gstack" + AGY_SKILLS_SOURCE="$AGY_GSTACK_SOURCE/skills" + mkdir -p "$AGY_SKILLS_SOURCE" + create_agy_runtime_root "$SOURCE_GSTACK_DIR" "$AGY_GSTACK_SOURCE" + link_agy_skill_dirs "$SOURCE_GSTACK_DIR" "$AGY_SKILLS_SOURCE" + # Register/import the plugin into Agy + if command -v agy >/dev/null 2>&1; then + echo "Registering gstack plugin with Agy..." + agy plugin install "$AGY_GSTACK_SOURCE" >/dev/null 2>&1 || agy plugin import "$AGY_GSTACK_SOURCE" --force >/dev/null 2>&1 || true + fi + echo "gstack ready (agy)." + echo " browse: $BROWSE_BIN" + echo " agy skills: $AGY_SKILLS" +fi + # 7. Create .agents/ sidecar symlinks for the real Codex skill target. # The root Codex skill ends up pointing at $SOURCE_GSTACK_DIR/.agents/skills/gstack, # so the runtime assets must live there for both global and repo-local installs. diff --git a/test/gen-skill-docs.test.ts b/test/gen-skill-docs.test.ts index 2fb783ffd0..d4ec1f0351 100644 --- a/test/gen-skill-docs.test.ts +++ b/test/gen-skill-docs.test.ts @@ -2372,16 +2372,17 @@ describe('setup script validation', () => { expect(claudeSection).toContain('link_claude_root_skill_alias "$SOURCE_GSTACK_DIR" "$INSTALL_SKILLS_DIR"'); }); - test('setup supports --host auto|claude|codex|kiro|opencode', () => { + test('setup supports --host auto|claude|codex|kiro|opencode|agy', () => { expect(setupContent).toContain('--host'); - expect(setupContent).toContain('claude|codex|kiro|factory|opencode|auto'); + expect(setupContent).toContain('claude|codex|kiro|factory|opencode|agy|auto'); }); - test('auto mode detects claude, codex, kiro, and opencode binaries', () => { + test('auto mode detects claude, codex, kiro, opencode, and agy binaries', () => { expect(setupContent).toContain('command -v claude'); expect(setupContent).toContain('command -v codex'); expect(setupContent).toContain('command -v kiro-cli'); expect(setupContent).toContain('command -v opencode'); + expect(setupContent).toContain('command -v agy'); }); // T1: Sidecar skip guard — prevents .agents/skills/gstack from being linked as a skill @@ -2423,6 +2424,23 @@ describe('setup script validation', () => { expect(setupContent).toContain('dx-hall-of-fame.md'); }); + test('setup supports --host agy with install section and agy plugin path vars', () => { + expect(setupContent).toContain('INSTALL_AGY='); + expect(setupContent).toContain('AGY_PLUGINS="$HOME/.gemini/config/plugins"'); + expect(setupContent).toContain('AGY_GSTACK="$AGY_PLUGINS/gstack"'); + expect(setupContent).toContain('AGY_SKILLS="$AGY_GSTACK/skills"'); + }); + + test('setup installs agy plugin into a nested gstack runtime root and validates registration', () => { + expect(setupContent).toContain('create_agy_runtime_root'); + expect(setupContent).toContain('.gemini/config/plugins/gstack'); + expect(setupContent).toContain('review/specialists'); + expect(setupContent).toContain('qa/templates'); + expect(setupContent).toContain('qa/references'); + expect(setupContent).toContain('dx-hall-of-fame.md'); + expect(setupContent).toContain('agy plugin install'); + }); + test('create_agents_sidecar links runtime assets', () => { // Sidecar must link bin, browse, review, qa const fnStart = setupContent.indexOf('create_agents_sidecar()'); @@ -3062,7 +3080,7 @@ describe('plan-mode-info resolver (handshake-replacement)', () => { // Non-Claude hosts render to hostSubdirs (.agents/, .openclaw/, etc). The // plan-mode-info resolver has no host-scoping — all hosts get the new // section, none get the old handshake. Scan all candidate host dirs. - const hostDirs = ['.agents', '.openclaw', '.opencode', '.factory', '.hermes', '.kiro', '.cursor', '.slate']; + const hostDirs = ['.agents', '.openclaw', '.opencode', '.factory', '.hermes', '.kiro', '.cursor', '.slate', '.gemini/config/plugins/gstack']; let checked = 0; for (const host of hostDirs) { const skillsRoot = path.join(ROOT, host, 'skills'); From e5d6583e7b1f5a572e2e72002b65f405f2513ac9 Mon Sep 17 00:00:00 2001 From: Mattias Petersson Date: Sun, 12 Jul 2026 16:46:48 +0200 Subject: [PATCH 03/12] feat(benchmark): add Agy model benchmark provider and runner adapter --- bin/gstack-model-benchmark | 12 +-- test/benchmark-cli.test.ts | 16 ++-- test/helpers/benchmark-runner.ts | 10 ++- test/helpers/providers/agy.ts | 87 ++++++++++++++++++++++ test/skill-e2e-benchmark-providers.test.ts | 38 +++++++++- 5 files changed, 143 insertions(+), 20 deletions(-) create mode 100644 test/helpers/providers/agy.ts diff --git a/bin/gstack-model-benchmark b/bin/gstack-model-benchmark index c5f5cb5b65..a102f2dcde 100755 --- a/bin/gstack-model-benchmark +++ b/bin/gstack-model-benchmark @@ -31,11 +31,13 @@ import { runBenchmark, formatTable, formatJson, formatMarkdown, type BenchmarkIn import { ClaudeAdapter } from '../test/helpers/providers/claude'; import { GptAdapter } from '../test/helpers/providers/gpt'; import { GeminiAdapter } from '../test/helpers/providers/gemini'; +import { AgyAdapter } from '../test/helpers/providers/agy'; const ADAPTER_FACTORIES = { claude: () => new ClaudeAdapter(), gpt: () => new GptAdapter(), gemini: () => new GeminiAdapter(), + agy: () => new AgyAdapter(), }; type OutputFormat = 'table' | 'json' | 'markdown'; @@ -76,13 +78,13 @@ function positionalArgs(args: string[]): string[] { return positional; } -function parseProviders(s: string | undefined): Array<'claude' | 'gpt' | 'gemini'> { +function parseProviders(s: string | undefined): Array<'claude' | 'gpt' | 'gemini' | 'agy'> { if (!s) return ['claude']; - const seen = new Set<'claude' | 'gpt' | 'gemini'>(); + const seen = new Set<'claude' | 'gpt' | 'gemini' | 'agy'>(); for (const p of s.split(',').map(x => x.trim()).filter(Boolean)) { - if (p === 'claude' || p === 'gpt' || p === 'gemini') seen.add(p); + if (p === 'claude' || p === 'gpt' || p === 'gemini' || p === 'agy') seen.add(p); else { - console.error(`WARN: unknown provider '${p}' — skipping. Valid: claude, gpt, gemini.`); + console.error(`WARN: unknown provider '${p}' — skipping. Valid: claude, gpt, gemini, agy.`); } } return seen.size ? Array.from(seen) : ['claude']; @@ -149,7 +151,7 @@ async function main(): Promise { async function dryRunReport(opts: { prompt: string; - providers: Array<'claude' | 'gpt' | 'gemini'>; + providers: Array<'claude' | 'gpt' | 'gemini' | 'agy'>; workdir: string; timeoutMs: number; output: OutputFormat; diff --git a/test/benchmark-cli.test.ts b/test/benchmark-cli.test.ts index 8edea3b245..1912f3d8ff 100644 --- a/test/benchmark-cli.test.ts +++ b/test/benchmark-cli.test.ts @@ -36,12 +36,13 @@ function run(args: string[], opts: { env?: Record } = {}): { sta describe('gstack-model-benchmark --dry-run', () => { test('prints provider availability report and exits 0', () => { - const r = run(['--prompt', 'hi', '--models', 'claude,gpt,gemini', '--dry-run']); + const r = run(['--prompt', 'hi', '--models', 'claude,gpt,gemini,agy', '--dry-run']); expect(r.status).toBe(0); expect(r.stdout).toContain('gstack-model-benchmark --dry-run'); expect(r.stdout).toContain('claude'); expect(r.stdout).toContain('gpt'); expect(r.stdout).toContain('gemini'); + expect(r.stdout).toContain('agy'); expect(r.stdout).toContain('no prompts sent'); }); @@ -86,12 +87,12 @@ describe('gstack-model-benchmark --dry-run', () => { }); test('each adapter reports either OK or NOT READY, never crashes', () => { - const r = run(['--prompt', 'hi', '--models', 'claude,gpt,gemini', '--dry-run']); + const r = run(['--prompt', 'hi', '--models', 'claude,gpt,gemini,agy', '--dry-run']); expect(r.status).toBe(0); // Each provider line must end in OK or NOT READY const lines = r.stdout.split('\n'); - const adapterLines = lines.filter(l => /^\s+(claude|gpt|gemini):/.test(l)); - expect(adapterLines.length).toBe(3); + const adapterLines = lines.filter(l => /^\s+(claude|gpt|gemini|agy):/.test(l)); + expect(adapterLines.length).toBe(4); for (const line of adapterLines) { expect(line).toMatch(/(OK|NOT READY)/); } @@ -116,7 +117,7 @@ describe('gstack-model-benchmark --dry-run', () => { TERM: process.env.TERM ?? 'xterm', HOME: emptyHome, }; - const result = spawnSync('bun', ['run', BIN, '--prompt', 'hi', '--models', 'claude,gpt,gemini', '--dry-run'], { + const result = spawnSync('bun', ['run', BIN, '--prompt', 'hi', '--models', 'claude,gpt,gemini,agy', '--dry-run'], { cwd: ROOT, env: minimalEnv, encoding: 'utf-8', @@ -124,14 +125,15 @@ describe('gstack-model-benchmark --dry-run', () => { }); expect(result.status).toBe(0); const out = result.stdout?.toString() ?? ''; - // gpt + gemini must report NOT READY in this clean env (their auth check + // gpt + gemini + agy must report NOT READY in this clean env (their auth check // reads paths under the overridden HOME). expect(out).toMatch(/gpt:\s+NOT READY/); expect(out).toMatch(/gemini:\s+NOT READY/); + expect(out).toMatch(/agy:\s+NOT READY/); // Every NOT READY line must include a concrete remediation hint so users // can resolve the missing auth. This is the regression we care about. const notReadyLines = out.split('\n').filter(l => l.includes('NOT READY')); - expect(notReadyLines.length).toBeGreaterThanOrEqual(2); + expect(notReadyLines.length).toBeGreaterThanOrEqual(3); for (const line of notReadyLines) { expect(line).toMatch(/(install|Install|login|export|Run|Log in)/); } diff --git a/test/helpers/benchmark-runner.ts b/test/helpers/benchmark-runner.ts index cbef4107b2..01ca00b737 100644 --- a/test/helpers/benchmark-runner.ts +++ b/test/helpers/benchmark-runner.ts @@ -11,15 +11,16 @@ import type { ProviderAdapter, RunOpts, RunResult } from './providers/types'; import { ClaudeAdapter } from './providers/claude'; import { GptAdapter } from './providers/gpt'; import { GeminiAdapter } from './providers/gemini'; +import { AgyAdapter } from './providers/agy'; export interface BenchmarkInput { prompt: string; workdir: string; timeoutMs?: number; - /** Adapter names to run (e.g., ['claude', 'gpt', 'gemini']). */ - providers: Array<'claude' | 'gpt' | 'gemini'>; + /** Adapter names to run (e.g., ['claude', 'gpt', 'gemini', 'agy']). */ + providers: Array<'claude' | 'gpt' | 'gemini' | 'agy'>; /** Optional per-provider model overrides. */ - models?: Partial>; + models?: Partial>; /** If true, skip providers whose available() returns !ok. If false, include them with error. */ skipUnavailable?: boolean; } @@ -44,10 +45,11 @@ export interface BenchmarkReport { entries: BenchmarkEntry[]; } -const ADAPTERS: Record<'claude' | 'gpt' | 'gemini', () => ProviderAdapter> = { +const ADAPTERS: Record<'claude' | 'gpt' | 'gemini' | 'agy', () => ProviderAdapter> = { claude: () => new ClaudeAdapter(), gpt: () => new GptAdapter(), gemini: () => new GeminiAdapter(), + agy: () => new AgyAdapter(), }; export async function runBenchmark(input: BenchmarkInput): Promise { diff --git a/test/helpers/providers/agy.ts b/test/helpers/providers/agy.ts new file mode 100644 index 0000000000..3a50efe6d6 --- /dev/null +++ b/test/helpers/providers/agy.ts @@ -0,0 +1,87 @@ +import type { ProviderAdapter, RunOpts, RunResult, AvailabilityCheck } from './types'; +import { estimateCostUsd } from '../pricing'; +import { execFileSync, spawnSync } from 'child_process'; +import * as fs from 'fs'; +import * as path from 'path'; +import * as os from 'os'; + +/** + * Agy adapter — wraps the `agy` CLI using `agy --print`. + * + * Auth comes from ~/.gemini/oauth_creds.json. + */ +export class AgyAdapter implements ProviderAdapter { + readonly name = 'agy'; + readonly family = 'gemini' as const; + + async available(): Promise { + const res = spawnSync('sh', ['-c', 'command -v agy'], { timeout: 2000 }); + if (res.status !== 0) { + return { ok: false, reason: 'agy CLI not found on PATH. Install per Google Antigravity instructions.' }; + } + const newCfgDir = path.join(os.homedir(), '.gemini'); + const newOauth = path.join(newCfgDir, 'oauth_creds.json'); + const hasCfg = fs.existsSync(newOauth); + if (!hasCfg) { + return { ok: false, reason: 'No Agy auth found. Log in via `agy` or authenticate.' }; + } + return { ok: true }; + } + + async run(opts: RunOpts): Promise { + const start = Date.now(); + const args = ['--print', opts.prompt, '--dangerously-skip-permissions']; + if (opts.model) args.push('--model', opts.model); + if (opts.extraArgs) args.push(...opts.extraArgs); + + try { + const out = execFileSync('agy', args, { + cwd: opts.workdir, + timeout: opts.timeoutMs, + encoding: 'utf-8', + maxBuffer: 32 * 1024 * 1024, + }); + + // Estimate tokens as agy does not output NDJSON/JSON tokens yet in print mode + const inputTokens = Math.ceil(opts.prompt.length / 4); + const outputTokens = Math.ceil(out.length / 4); + + return { + output: out.trim(), + tokens: { input: inputTokens, output: outputTokens }, + durationMs: Date.now() - start, + toolCalls: 0, + modelUsed: opts.model || 'gemini-2.5-flash', + }; + } catch (err: unknown) { + const durationMs = Date.now() - start; + const e = err as { code?: string; stderr?: Buffer; signal?: string; message?: string }; + const stderr = e.stderr?.toString() ?? ''; + if (e.signal === 'SIGTERM' || e.code === 'ETIMEDOUT') { + return this.emptyResult(durationMs, { code: 'timeout', reason: `exceeded ${opts.timeoutMs}ms` }, opts.model); + } + if (/unauthorized|auth|login|api key/i.test(stderr)) { + return this.emptyResult(durationMs, { code: 'auth', reason: stderr.slice(0, 400) }, opts.model); + } + if (/rate[- ]?limit|429|quota/i.test(stderr)) { + return this.emptyResult(durationMs, { code: 'rate_limit', reason: stderr.slice(0, 400) }, opts.model); + } + return this.emptyResult(durationMs, { code: 'unknown', reason: (e.message ?? stderr ?? 'unknown').slice(0, 400) }, opts.model); + } + } + + estimateCost(tokens: { input: number; output: number; cached?: number }, model?: string): number { + return estimateCostUsd(tokens, model ?? 'gemini-2.5-flash'); + } + + private emptyResult(durationMs: number, error: RunResult['error'], model?: string): RunResult { + return { + output: '', + tokens: { input: 0, output: 0 }, + durationMs, + toolCalls: 0, + modelUsed: model ?? 'gemini-2.5-flash', + error, + }; + } +} diff --git a/test/skill-e2e-benchmark-providers.test.ts b/test/skill-e2e-benchmark-providers.test.ts index 12456ec231..446762d11b 100644 --- a/test/skill-e2e-benchmark-providers.test.ts +++ b/test/skill-e2e-benchmark-providers.test.ts @@ -22,6 +22,7 @@ import { describe, test, expect, beforeAll, afterAll } from 'bun:test'; import { ClaudeAdapter } from './helpers/providers/claude'; import { GptAdapter } from './helpers/providers/gpt'; import { GeminiAdapter } from './helpers/providers/gemini'; +import { AgyAdapter } from './helpers/providers/agy'; import { runBenchmark } from './helpers/benchmark-runner'; import * as fs from 'fs'; import * as path from 'path'; @@ -39,6 +40,7 @@ const PROMPT = 'Reply with exactly this text and nothing else: ok'; const claude = new ClaudeAdapter(); const gpt = new GptAdapter(); const gemini = new GeminiAdapter(); +const agy = new AgyAdapter(); // Use a temp working directory so provider CLIs can't accidentally touch the repo. // Created in beforeAll / cleaned in afterAll so concurrent CI runs don't leak. @@ -80,6 +82,14 @@ describeIfEvals('multi-provider benchmark adapters (live)', () => { } }); + test('agy: available() returns structured ok/reason', async () => { + const check = await agy.available(); + expect(check).toHaveProperty('ok'); + if (!check.ok) { + expect(typeof check.reason).toBe('string'); + } + }); + test('claude: trivial prompt produces parseable output', async () => { const check = await claude.available(); if (!check.ok) { @@ -144,11 +154,31 @@ describeIfEvals('multi-provider benchmark adapters (live)', () => { expect(typeof result.modelUsed).toBe('string'); }, 150_000); + test('agy: trivial prompt produces parseable output', async () => { + const check = await agy.available(); + if (!check.ok) { + process.stderr.write(`\nagy live smoke: SKIPPED — ${check.reason}\n`); + return; + } + const result = await agy.run({ prompt: PROMPT, workdir, timeoutMs: 120_000 }); + if (result.error) { + throw new Error(`agy errored: ${result.error.code} — ${result.error.reason}`); + } + expect(typeof result.output).toBe('string'); + expect(result.tokens.input).toBeGreaterThan(0); + expect(result.tokens.output).toBeGreaterThan(0); + expect(result.durationMs).toBeGreaterThan(0); + expect(typeof result.modelUsed).toBe('string'); + const cost = agy.estimateCost(result.tokens, result.modelUsed); + expect(cost).toBeGreaterThan(0); + }, 150_000); + test('timeout error surfaces as error.code=timeout (no exception)', async () => { - // Use whatever adapter is available first — all three should share timeout semantics. + // Use whatever adapter is available first — all should share timeout semantics. const adapter = (await claude.available()).ok ? claude : (await gpt.available()).ok ? gpt : (await gemini.available()).ok ? gemini + : (await agy.available()).ok ? agy : null; if (!adapter) { process.stderr.write('\ntimeout smoke: SKIPPED — no provider available\n'); @@ -164,16 +194,16 @@ describeIfEvals('multi-provider benchmark adapters (live)', () => { }, 30_000); test('runBenchmark: Promise.allSettled means one unavailable provider does not block others', async () => { - // Use the full runner with all three providers — whichever are unauthed should + // Use the full runner with all providers — whichever are unauthed should // return entries with available=false and not crash the batch. const report = await runBenchmark({ prompt: PROMPT, workdir, - providers: ['claude', 'gpt', 'gemini'], + providers: ['claude', 'gpt', 'gemini', 'agy'], timeoutMs: 120_000, skipUnavailable: false, }); - expect(report.entries).toHaveLength(3); + expect(report.entries).toHaveLength(4); for (const e of report.entries) { expect(['claude', 'gpt', 'gemini']).toContain(e.family); if (e.available) { From 08e1c43e4e04689b959225833edffd70f24b7850 Mon Sep 17 00:00:00 2001 From: Mattias Petersson Date: Sun, 12 Jul 2026 16:46:49 +0200 Subject: [PATCH 04/12] fix(test): resolve gbrain detect install paths and session import xargs crash --- bin/gstack-codex-session-import | 2 +- test/gbrain-detect-install.test.ts | 6 ++++-- test/gbrain-refresh-install-render.test.ts | 2 +- 3 files changed, 6 insertions(+), 4 deletions(-) diff --git a/bin/gstack-codex-session-import b/bin/gstack-codex-session-import index 91368cac9a..f6c091d004 100755 --- a/bin/gstack-codex-session-import +++ b/bin/gstack-codex-session-import @@ -65,7 +65,7 @@ case "$MODE" in exit 0 fi LATEST=$(find "$CODEX_SESSIONS_ROOT" -type f -name "rollout-*.jsonl" -print 2>/dev/null \ - | xargs ls -t 2>/dev/null | head -1 || true) + | xargs -r ls -t 2>/dev/null | head -1 || true) if [ -z "$LATEST" ]; then echo "NO_SESSIONS: no rollout-*.jsonl files under $CODEX_SESSIONS_ROOT" exit 0 diff --git a/test/gbrain-detect-install.test.ts b/test/gbrain-detect-install.test.ts index b9c82c1550..376e7b6c2e 100644 --- a/test/gbrain-detect-install.test.ts +++ b/test/gbrain-detect-install.test.ts @@ -24,8 +24,10 @@ const INSTALL = path.join(ROOT, 'bin', 'gstack-gbrain-install'); // Minimal PATH with POSIX tools + homebrew (for jq/git/curl) but no user-bin // dirs — this keeps `gbrain` out of PATH deterministically across dev machines // while still finding jq, git, curl, sed, cat, etc. Each test can prepend a -// fake-gbrain dir when it wants to simulate presence. -const SAFE_PATH = '/usr/bin:/bin:/usr/sbin:/sbin:/opt/homebrew/bin:/usr/local/bin'; +// fake-gbrain dir when it wants to simulate presence. We append the active Bun +// directory so the script shebang can find bun. +const BUN_DIR = path.dirname(process.execPath); +const SAFE_PATH = `/usr/bin:/bin:/usr/sbin:/sbin:/opt/homebrew/bin:/usr/local/bin:${BUN_DIR}`; let tmpHome: string; let tmpHomeReal: string; diff --git a/test/gbrain-refresh-install-render.test.ts b/test/gbrain-refresh-install-render.test.ts index f1494d47d0..95e75bde53 100644 --- a/test/gbrain-refresh-install-render.test.ts +++ b/test/gbrain-refresh-install-render.test.ts @@ -13,7 +13,7 @@ const SRC = fs.readFileSync(path.join(ROOT, 'bin', 'gstack-config'), 'utf-8'); // satisfied by unrelated text elsewhere in the file. function okBranch(): string { const start = SRC.indexOf('gbrain-refresh)'); - const ok = SRC.indexOf('ok)', start); + const ok = SRC.indexOf('ok|timeout)', start); const end = SRC.indexOf(';;', ok); if (start < 0 || ok < 0 || end < 0) throw new Error('Could not locate gbrain-refresh ok) branch'); return SRC.slice(ok, end); From dd58fff1291f6387a0b0442025ef89f91ed5a1ca Mon Sep 17 00:00:00 2001 From: Mattias Petersson Date: Sun, 12 Jul 2026 16:46:51 +0200 Subject: [PATCH 05/12] docs: add Antigravity (Agy) integration guide --- docs/ADDING_A_HOST.md | 5 ++- docs/AGY.md | 71 +++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 75 insertions(+), 1 deletion(-) create mode 100644 docs/AGY.md diff --git a/docs/ADDING_A_HOST.md b/docs/ADDING_A_HOST.md index 50654e4e23..7967d46380 100644 --- a/docs/ADDING_A_HOST.md +++ b/docs/ADDING_A_HOST.md @@ -1,7 +1,7 @@ # Adding a New Host to gstack gstack uses a declarative host config system. Each supported AI coding agent -(Claude, Codex, Factory, Kiro, OpenCode, Slate, Cursor, OpenClaw) is defined +(Claude, Codex, Factory, Kiro, OpenCode, Slate, Cursor, OpenClaw, Hermes, GBrain, Agy) is defined as a typed TypeScript config object. Adding a new host means creating one file and re-exporting it. Zero code changes to the generator, setup, or tooling. @@ -17,6 +17,9 @@ hosts/ ├── slate.ts # Slate (Random Labs) ├── cursor.ts # Cursor ├── openclaw.ts # OpenClaw (hybrid: config + adapter) +├── hermes.ts # Hermes +├── gbrain.ts # GBrain +├── agy.ts # Google Antigravity └── index.ts # Registry: imports all, derives Host type ``` diff --git a/docs/AGY.md b/docs/AGY.md new file mode 100644 index 0000000000..456b9445a2 --- /dev/null +++ b/docs/AGY.md @@ -0,0 +1,71 @@ +# Google Antigravity (Agy) Integration + +gstack supports Google Antigravity (Agy) as a first-class agent host. This integration allows you to generate and run Agy-compatible skills, load them dynamically via the `agy` plugin system, and benchmark agy models alongside Claude, OpenAI, and Gemini. + +## Features + +- **Automatic Setup**: Installs gstack skills directly into the Antigravity config directory via `./setup --host agy`. +- **Agy Plugin Compatibility**: Packages skills as a standard Agy plugin with a `plugin.json` manifest, automatically registered with `agy plugin install`. +- **Exclusion of Incompatible Skills**: Automatically filters out Claude-specific wrappers (like `gstack-codex`) from Agy generation. +- **Benchmark Integration**: Register Agy as a benchmark provider using `agy --print` to trace latency, quality, and tokens. + +--- + +## Setup & Installation + +To generate and install gstack skills into your local Antigravity environment: + +```bash +./setup --host agy +``` + +### What Setup Does + +1. **Precompile & Prepare**: Ensures the browse tool and other runtime dependencies are compiled. +2. **Generate Skills**: Runs `bun run gen:skill-docs --host agy` to generate Agy-compatible frontmatter. +3. **Initialize Plugin**: Creates the plugin directory structure under `.gemini/config/plugins/gstack`. +4. **Symlink/Copy Assets**: Places necessary executables (`bin`, `browse`, `review`, `qa`) into the plugin root. +5. **Register Plugin**: Invokes `agy plugin install` to import the gstack plugin into `~/.gemini/config/import_manifest.json`. + +--- + +## Plugin Structure + +When installed, the gstack plugin resides at: +`~/.gemini/config/plugins/gstack/` + +The directory layout matches the standard Agy plugin structure: + +``` +~/.gemini/config/plugins/gstack/ +├── plugin.json # Manifest declaring name, version, and license +├── skills/ # Generated Agy-compatible skills +│ ├── gstack/ # Root gstack skill (/gstack) +│ ├── gstack-qa/ # QA testing skill (/qa) +│ └── ... +├── bin/ # Shared executables and CLI tools +└── browse/ # Browse tool distribution files +``` + +--- + +## Benchmarking with Agy + +Agy is registered as a model benchmark provider. You can execute prompts against Agy using the gstack benchmark runner. + +### Pre-requisites +- The `agy` CLI must be installed and available in your `PATH`. +- Credentials must be configured in `~/.gemini/oauth_creds.json`. + +### Run Benchmark +To run a dry run and check availability: +```bash +gstack-model-benchmark --models agy --dry-run +``` + +To execute a benchmark run: +```bash +gstack-model-benchmark --models agy --prompt "Explain quantum computing in one sentence." +``` + +The runner automatically passes `--dangerously-skip-permissions` to the CLI to run headless without hanging on interactive permission prompts. From f44448ff9b073adc186f2fa31adce6f9105c1f0f Mon Sep 17 00:00:00 2001 From: Mattias Petersson Date: Sun, 12 Jul 2026 16:46:59 +0200 Subject: [PATCH 06/12] chore: bump version and changelog (v1.61.0.0) Co-Authored-By: OpenAI Codex --- CHANGELOG.md | 22 ++++++++++++++++++++++ VERSION | 2 +- package.json | 2 +- 3 files changed, 24 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 351e7cabde..f89b8a222a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,27 @@ # Changelog +## [1.61.0.0] - 2026-07-12 + +## **Permanent Antigravity (Agy) integration and benchmark support are live.** +## **gstack now runs on Agy, registers/updates plugins automatically, and supports Agy model benchmarks.** + +Agy integration is now permanent and first-class. Running setup with `--host agy` generates and imports Agy-compatible skills, excludes incompatible skills, and registers the gstack plugin in Agy's plugin configuration. This release also fixes several test-suite and helper issues that were causing CI failures, and adds Agy model benchmarking. + +### Itemized changes + +#### Added + +- `hosts/agy.ts`: Declarative host configuration for Agy. +- `test/helpers/providers/agy.ts`: Agy benchmark runner adapter that executes `agy --print` with safely skipped permissions. +- `docs/AGY.md`: User documentation for Agy installation and integration. + +#### Fixed + +- `setup`: Support `--host agy` flag, build `plugin.json` and skill directories under `.gemini/config/plugins/gstack`, and trigger `agy plugin install`. +- `bin/gstack-codex-session-import`: Added `-r` flag to `xargs` pipeline to prevent listing files in current directory when empty. +- `test/gbrain-detect-install.test.ts`: Appended bun's executable directory to test environment `PATH` to resolve shebang resolution failures. +- `test/gbrain-refresh-install-render.test.ts`: Resolved regex pattern mismatch in status check. + ## [1.60.1.0] - 2026-07-09 ## **The /autoplan dual-voice eval is back on the board, catching real regressions.** diff --git a/VERSION b/VERSION index c4190e0048..82b3fa4c6d 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -1.60.1.0 +1.61.0.0 diff --git a/package.json b/package.json index c5168d6486..7488799fa5 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "gstack", - "version": "1.60.1.0", + "version": "1.61.0.0", "description": "Garry's Stack — Claude Code skills + fast headless browser. One repo, one install, entire AI engineering workflow.", "license": "MIT", "type": "module", From 6e4ed2b292f874bd52547cb1b00473aa76a40da0 Mon Sep 17 00:00:00 2001 From: Mattias Petersson Date: Mon, 13 Jul 2026 03:24:11 +0200 Subject: [PATCH 07/12] fix agy setup runtime generation --- setup | 13 +++++++++---- test/gen-skill-docs.test.ts | 13 +++++++++++++ 2 files changed, 22 insertions(+), 4 deletions(-) diff --git a/setup b/setup index c96a311819..dfd0e160be 100755 --- a/setup +++ b/setup @@ -992,7 +992,9 @@ link_opencode_skill_dirs() { create_agy_runtime_root() { local gstack_dir="$1" local agy_gstack="$2" - local agy_dir="$gstack_dir/.gemini/config/plugins/gstack/skills" + local agy_version + + agy_version="$(tr -d '[:space:]' < "$gstack_dir/VERSION")" if [ -L "$agy_gstack" ]; then rm -f "$agy_gstack" @@ -1003,7 +1005,8 @@ create_agy_runtime_root() { mkdir -p "$agy_gstack" "$agy_gstack/browse" "$agy_gstack/design" "$agy_gstack/gstack-upgrade" "$agy_gstack/review" "$agy_gstack/qa" "$agy_gstack/plan-devex-review" "$agy_gstack/skills" # Create plugin.json for agy plugin structure - echo '{"name": "gstack", "version": "1.58.5.0", "description": "gstack skills for Google Antigravity", "author": {"name": "gstack"}, "license": "MIT"}' > "$agy_gstack/plugin.json" + printf '{"name": "gstack", "version": "%s", "description": "gstack skills for Google Antigravity", "author": {"name": "gstack"}, "license": "MIT"}\n' \ + "$agy_version" > "$agy_gstack/plugin.json" if [ -d "$gstack_dir/bin" ]; then _link_or_copy "$gstack_dir/bin" "$agy_gstack/bin" @@ -1045,12 +1048,14 @@ link_agy_skill_dirs() { local agy_dir="$gstack_dir/.gemini/config/plugins/gstack/skills" local linked=() - if [ ! -d "$agy_dir" ]; then + # create_agy_runtime_root intentionally replaces the generated plugin tree. + # A directory alone is therefore not proof that skill generation succeeded. + if ! find "$agy_dir" -mindepth 2 -maxdepth 2 -name SKILL.md -print -quit 2>/dev/null | grep -q .; then echo " Generating .gemini/ skill docs..." ( cd "$gstack_dir" && bun run gen:skill-docs --host agy ) fi - if [ ! -d "$agy_dir" ]; then + if ! find "$agy_dir" -mindepth 2 -maxdepth 2 -name SKILL.md -print -quit 2>/dev/null | grep -q .; then echo " warning: .gemini/config/plugins/gstack/skills/ generation failed — run 'bun run gen:skill-docs --host agy' manually" >&2 return 1 fi diff --git a/test/gen-skill-docs.test.ts b/test/gen-skill-docs.test.ts index d4ec1f0351..5e580b7d27 100644 --- a/test/gen-skill-docs.test.ts +++ b/test/gen-skill-docs.test.ts @@ -2441,6 +2441,19 @@ describe('setup script validation', () => { expect(setupContent).toContain('agy plugin install'); }); + test('setup regenerates deleted agy skills and derives the plugin version from VERSION', () => { + const fnStart = setupContent.indexOf('create_agy_runtime_root()'); + const fnEnd = setupContent.indexOf('link_agy_skill_dirs()', fnStart); + const runtimeFn = setupContent.slice(fnStart, fnEnd); + const linkFn = setupContent.slice(fnEnd, setupContent.indexOf('# 4. Install for Claude', fnEnd)); + + expect(runtimeFn).toContain('agy_version="$(tr -d \'[:space:]\' < "$gstack_dir/VERSION")"'); + expect(runtimeFn).toContain('"$agy_version" > "$agy_gstack/plugin.json"'); + expect(runtimeFn).not.toContain('"version": "1.58.5.0"'); + expect(linkFn).toContain('-name SKILL.md'); + expect(linkFn).toContain('bun run gen:skill-docs --host agy'); + }); + test('create_agents_sidecar links runtime assets', () => { // Sidecar must link bin, browse, review, qa const fnStart = setupContent.indexOf('create_agents_sidecar()'); From ea7b46d5a5cb79e5003ec21f84a636d45f1ce26c Mon Sep 17 00:00:00 2001 From: Mattias Petersson Date: Mon, 13 Jul 2026 03:28:55 +0200 Subject: [PATCH 08/12] docs: document Agy host and benchmark support --- AGENTS.md | 2 +- CLAUDE.md | 2 +- CONTRIBUTING.md | 7 ++++--- README.md | 7 +++++-- 4 files changed, 11 insertions(+), 7 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index 69651022d7..1e3c885037 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -62,7 +62,7 @@ Invoke them by name (e.g., `/office-hours`). | `/retro` | Weekly retro with per-person breakdowns and shipping streaks. | | `/health` | Code quality dashboard (type checker, linter, tests, dead code). | | `/benchmark` | Performance regression detection (page load, Core Web Vitals). | -| `/benchmark-models` | Cross-model benchmark for skills (Claude, GPT, Gemini side-by-side). | +| `/benchmark-models` | Cross-model benchmark for skills (Claude, GPT, Gemini, and Agy side-by-side). | | `/cso` | OWASP Top 10 + STRIDE security audit. | | `/setup-gbrain` | Set up gbrain for cross-machine session memory sync. | | `/sync-gbrain` | Keep gbrain current with this repo's code; refresh agent search guidance in CLAUDE.md. | diff --git a/CLAUDE.md b/CLAUDE.md index 9848449020..8bca27893e 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -95,7 +95,7 @@ gstack/ │ ├── claude.ts # Primary host config │ ├── codex.ts, factory.ts, kiro.ts # Existing hosts │ ├── opencode.ts, slate.ts, cursor.ts, openclaw.ts # IDE hosts -│ ├── hermes.ts, gbrain.ts # Agent runtime hosts +│ ├── hermes.ts, gbrain.ts, agy.ts # Agent runtime hosts │ └── index.ts # Registry: exports all, derives Host type ├── scripts/ # Build + DX tooling │ ├── gen-skill-docs.ts # Template → SKILL.md generator (config-driven) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index b75d4a898f..65d6320ba9 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -308,11 +308,11 @@ worth the review overhead. ## Multi-host development -gstack generates SKILL.md files for 8 hosts from one set of `.tmpl` templates. +gstack generates SKILL.md files for 11 hosts from one set of `.tmpl` templates. Each host is a typed config in `hosts/*.ts`. The generator reads these configs to produce host-appropriate output (different frontmatter, paths, tool names). -**Supported hosts:** Claude (primary), Codex, Factory, Kiro, OpenCode, Slate, Cursor, OpenClaw. +**Supported hosts:** Claude (primary), Codex, Factory, Kiro, OpenCode, Slate, Cursor, OpenClaw, Hermes, GBrain, and Agy. ### Generating for all hosts @@ -321,7 +321,8 @@ to produce host-appropriate output (different frontmatter, paths, tool names). bun run gen:skill-docs # Claude (default) bun run gen:skill-docs --host codex # Codex bun run gen:skill-docs --host opencode # OpenCode -bun run gen:skill-docs --host all # All 8 hosts +bun run gen:skill-docs --host agy # Google Antigravity +bun run gen:skill-docs --host all # All 11 hosts # Or use build, which does all hosts + compiles binaries bun run build diff --git a/README.md b/README.md index 4bb177c3a7..574b048e0f 100644 --- a/README.md +++ b/README.md @@ -101,7 +101,7 @@ These are conversational skills. Your OpenClaw agent runs them directly via chat ### Other AI Agents -gstack works on 10 AI coding agents, not just Claude. Setup auto-detects which +gstack works on 11 AI coding agents, not just Claude. Setup auto-detects which agents you have installed: ```bash @@ -121,10 +121,13 @@ Or target a specific agent with `./setup --host `: | Kiro | `--host kiro` | `~/.kiro/skills/gstack-*/` | | Hermes | `--host hermes` | `~/.hermes/skills/gstack-*/` | | GBrain (mod) | `--host gbrain` | `~/.gbrain/skills/gstack-*/` | +| Google Antigravity (Agy) | `--host agy` | `~/.gemini/config/plugins/gstack/skills/` | **Want to add support for another agent?** See [docs/ADDING_A_HOST.md](docs/ADDING_A_HOST.md). It's one TypeScript config file, zero code changes. +For Agy plugin registration and model benchmarking, see the [Agy integration guide](docs/AGY.md). + ## See it work ``` @@ -241,7 +244,7 @@ Beyond the slash-command skills, gstack ships standalone CLIs for workflows that | Command | What it does | |---------|-------------| -| `gstack-model-benchmark` | **Cross-model benchmark** — run the same prompt through Claude, GPT (via Codex CLI), and Gemini; compare latency, tokens, cost, and (optionally) LLM-judge quality score. Auth detected per provider, unavailable providers skip cleanly. Output as table, JSON, or markdown. `--dry-run` validates flags + auth without spending API calls. | +| `gstack-model-benchmark` | **Cross-model benchmark** — run the same prompt through Claude, GPT (via Codex CLI), Gemini, and Agy; compare latency, tokens, cost, and (optionally) LLM-judge quality score. Auth detected per provider, unavailable providers skip cleanly. Output as table, JSON, or markdown. `--dry-run` validates flags + auth without spending API calls. See the [Agy integration guide](docs/AGY.md) for Agy setup. | | `gstack-taste-update` | **Design taste learning** — writes approvals and rejections from `/design-shotgun` into a persistent per-project taste profile. Decays 5%/week. Feeds back into future variant generation so the system learns what you actually pick. | | `gstack-ios-qa-daemon` | **iOS QA daemon** — Mac-side broker between an agent and a connected iPhone over USB CoreDevice. Loopback by default; `--tailnet` opens a Tailscale-facing listener with identity-gated capability tiers. Single-instance via flock on `~/.gstack/ios-qa-daemon.pid`. See [docs/howto-ios-testing-with-gstack.md](docs/howto-ios-testing-with-gstack.md). | | `gstack-ios-qa-mint` | **iOS allowlist manager** — owner-grant CLI for the tailnet allowlist. `grant`/`revoke`/`list` against `~/.gstack/ios-qa-allowlist.json` (mode 0600). Remote agents never auto-allowlist; this is the explicit-intent path. | From 49964ee712c0109fd226e170e7713ae4a11a9e06 Mon Sep 17 00:00:00 2001 From: Mattias Petersson Date: Mon, 13 Jul 2026 03:42:38 +0200 Subject: [PATCH 09/12] fix: harden local Agy integration --- benchmark-models/SKILL.md | 8 ++++---- benchmark-models/SKILL.md.tmpl | 8 ++++---- scripts/proactive-suggestions.json | 2 +- setup | 16 ++++++++++++---- test/gen-skill-docs.test.ts | 3 +++ test/helpers/providers/agy.ts | 10 +++++++++- test/skill-e2e-benchmark-providers.test.ts | 14 ++++++++++++++ 7 files changed, 47 insertions(+), 14 deletions(-) diff --git a/benchmark-models/SKILL.md b/benchmark-models/SKILL.md index 6a9d62616d..80bcbd416b 100644 --- a/benchmark-models/SKILL.md +++ b/benchmark-models/SKILL.md @@ -5,7 +5,7 @@ version: 1.0.0 description: Cross-model benchmark for gstack skills. (gstack) triggers: - cross model benchmark - - compare claude gpt gemini + - compare claude gpt gemini agy - benchmark skill across models - which model should I use allowed-tools: @@ -20,7 +20,7 @@ allowed-tools: ## When to invoke this skill Runs the same prompt through Claude, -GPT (via Codex CLI), and Gemini side-by-side — compares latency, tokens, cost, +GPT (via Codex CLI), Gemini, and Agy side-by-side — compares latency, tokens, cost, and optionally quality via LLM judge. Answers "which model is actually best for this skill?" with data instead of vibes. Separate from /benchmark, which measures web page performance. Use when: "benchmark models", "compare models", @@ -578,12 +578,12 @@ If C: ask for the path. Verify it exists. Use as positional argument. ## Step 2: Choose providers ```bash -"$BIN" --prompt "unused, dry-run" --models claude,gpt,gemini --dry-run +"$BIN" --prompt "unused, dry-run" --models claude,gpt,gemini,agy --dry-run ``` Show the dry-run output. The "Adapter availability" section tells the user which providers will actually run (OK) vs skip (NOT READY — remediation hint included). -If ALL three show NOT READY: stop with a clear message — benchmark can't run without at least one authed provider. Suggest `claude login`, `codex login`, or `gemini login` / `export GOOGLE_API_KEY`. +If ALL four show NOT READY: stop with a clear message — benchmark can't run without at least one authenticated provider. Suggest `claude login`, `codex login`, `gemini login` / `export GOOGLE_API_KEY`, or logging in through `agy`. If at least one is OK: AskUserQuestion: - **Simplify:** "Which models should we include? The dry-run above showed which are authed. Unauthed ones will be skipped cleanly — they won't abort the batch." diff --git a/benchmark-models/SKILL.md.tmpl b/benchmark-models/SKILL.md.tmpl index 034cda1824..dc40ddbf4e 100644 --- a/benchmark-models/SKILL.md.tmpl +++ b/benchmark-models/SKILL.md.tmpl @@ -4,7 +4,7 @@ preamble-tier: 1 version: 1.0.0 description: | Cross-model benchmark for gstack skills. Runs the same prompt through Claude, - GPT (via Codex CLI), and Gemini side-by-side — compares latency, tokens, cost, + GPT (via Codex CLI), Gemini, and Agy side-by-side — compares latency, tokens, cost, and optionally quality via LLM judge. Answers "which model is actually best for this skill?" with data instead of vibes. Separate from /benchmark, which measures web page performance. Use when: "benchmark models", "compare models", @@ -15,7 +15,7 @@ voice-triggers: - "which model is best" triggers: - cross model benchmark - - compare claude gpt gemini + - compare claude gpt gemini agy - benchmark skill across models - which model should I use allowed-tools: @@ -69,12 +69,12 @@ If C: ask for the path. Verify it exists. Use as positional argument. ## Step 2: Choose providers ```bash -"$BIN" --prompt "unused, dry-run" --models claude,gpt,gemini --dry-run +"$BIN" --prompt "unused, dry-run" --models claude,gpt,gemini,agy --dry-run ``` Show the dry-run output. The "Adapter availability" section tells the user which providers will actually run (OK) vs skip (NOT READY — remediation hint included). -If ALL three show NOT READY: stop with a clear message — benchmark can't run without at least one authed provider. Suggest `claude login`, `codex login`, or `gemini login` / `export GOOGLE_API_KEY`. +If ALL four show NOT READY: stop with a clear message — benchmark can't run without at least one authenticated provider. Suggest `claude login`, `codex login`, `gemini login` / `export GOOGLE_API_KEY`, or logging in through `agy`. If at least one is OK: AskUserQuestion: - **Simplify:** "Which models should we include? The dry-run above showed which are authed. Unauthed ones will be skipped cleanly — they won't abort the batch." diff --git a/scripts/proactive-suggestions.json b/scripts/proactive-suggestions.json index d08c608533..4661927387 100644 --- a/scripts/proactive-suggestions.json +++ b/scripts/proactive-suggestions.json @@ -15,7 +15,7 @@ }, "benchmark-models": { "lead": "Cross-model benchmark for gstack skills.", - "routing": "Runs the same prompt through Claude,\nGPT (via Codex CLI), and Gemini side-by-side — compares latency, tokens, cost,\nand optionally quality via LLM judge. Answers \"which model is actually best\nfor this skill?\" with data instead of vibes. Separate from /benchmark, which\nmeasures web page performance. Use when: \"benchmark models\", \"compare models\",\n\"which model is best for X\", \"cross-model comparison\", \"model shootout\".", + "routing": "Runs the same prompt through Claude,\nGPT (via Codex CLI), Gemini, and Agy side-by-side — compares latency, tokens, cost,\nand optionally quality via LLM judge. Answers \"which model is actually best\nfor this skill?\" with data instead of vibes. Separate from /benchmark, which\nmeasures web page performance. Use when: \"benchmark models\", \"compare models\",\n\"which model is best for X\", \"cross-model comparison\", \"model shootout\".", "voice_line": "Voice triggers (speech-to-text aliases): \"compare models\", \"model shootout\", \"which model is best\"." }, "browse": { diff --git a/setup b/setup index dfd0e160be..5e5e6bc00e 100755 --- a/setup +++ b/setup @@ -1304,10 +1304,18 @@ if [ "$INSTALL_AGY" -eq 1 ]; then mkdir -p "$AGY_SKILLS_SOURCE" create_agy_runtime_root "$SOURCE_GSTACK_DIR" "$AGY_GSTACK_SOURCE" link_agy_skill_dirs "$SOURCE_GSTACK_DIR" "$AGY_SKILLS_SOURCE" - # Register/import the plugin into Agy - if command -v agy >/dev/null 2>&1; then - echo "Registering gstack plugin with Agy..." - agy plugin install "$AGY_GSTACK_SOURCE" >/dev/null 2>&1 || agy plugin import "$AGY_GSTACK_SOURCE" --force >/dev/null 2>&1 || true + # Explicit Agy setup is incomplete unless the CLI can register the plugin. + if ! command -v agy >/dev/null 2>&1; then + echo "gstack setup failed: agy CLI not found on PATH; install Agy, then rerun './setup --host agy'." >&2 + exit 1 + fi + echo "Registering gstack plugin with Agy..." + if ! agy plugin install "$AGY_GSTACK_SOURCE"; then + echo "Agy plugin install failed; trying import --force..." >&2 + if ! agy plugin import "$AGY_GSTACK_SOURCE" --force; then + echo "gstack setup failed: Agy could not register the plugin at $AGY_GSTACK_SOURCE." >&2 + exit 1 + fi fi echo "gstack ready (agy)." echo " browse: $BROWSE_BIN" diff --git a/test/gen-skill-docs.test.ts b/test/gen-skill-docs.test.ts index 5e580b7d27..c06b69f8e7 100644 --- a/test/gen-skill-docs.test.ts +++ b/test/gen-skill-docs.test.ts @@ -2439,6 +2439,9 @@ describe('setup script validation', () => { expect(setupContent).toContain('qa/references'); expect(setupContent).toContain('dx-hall-of-fame.md'); expect(setupContent).toContain('agy plugin install'); + expect(setupContent).toContain('agy CLI not found on PATH'); + expect(setupContent).toContain('Agy could not register the plugin'); + expect(setupContent).not.toContain('agy plugin import "$AGY_GSTACK_SOURCE" --force >/dev/null 2>&1 || true'); }); test('setup regenerates deleted agy skills and derives the plugin version from VERSION', () => { diff --git a/test/helpers/providers/agy.ts b/test/helpers/providers/agy.ts index 3a50efe6d6..c37129f082 100644 --- a/test/helpers/providers/agy.ts +++ b/test/helpers/providers/agy.ts @@ -30,8 +30,16 @@ export class AgyAdapter implements ProviderAdapter { async run(opts: RunOpts): Promise { const start = Date.now(); - const args = ['--print', opts.prompt, '--dangerously-skip-permissions']; + // Benchmarks must never grant an agent write/tool approval in the caller's + // workdir. Plan mode plus the terminal sandbox is Agy's read-only boundary. + const args = ['--print', opts.prompt, '--mode', 'plan', '--sandbox']; if (opts.model) args.push('--model', opts.model); + if (opts.extraArgs?.includes('--dangerously-skip-permissions')) { + return this.emptyResult(0, { + code: 'unknown', + reason: '--dangerously-skip-permissions is not allowed in benchmarks', + }, opts.model); + } if (opts.extraArgs) args.push(...opts.extraArgs); try { diff --git a/test/skill-e2e-benchmark-providers.test.ts b/test/skill-e2e-benchmark-providers.test.ts index 446762d11b..bd60c71cfe 100644 --- a/test/skill-e2e-benchmark-providers.test.ts +++ b/test/skill-e2e-benchmark-providers.test.ts @@ -42,6 +42,20 @@ const gpt = new GptAdapter(); const gemini = new GeminiAdapter(); const agy = new AgyAdapter(); +test('agy adapter enforces sandboxed plan mode and rejects permission bypass', async () => { + const source = fs.readFileSync(path.join(import.meta.dir, 'helpers/providers/agy.ts'), 'utf8'); + expect(source).toContain("'--mode', 'plan', '--sandbox'"); + expect(source).not.toContain("opts.prompt, '--dangerously-skip-permissions'"); + + const result = await agy.run({ + prompt: PROMPT, + workdir: os.tmpdir(), + timeoutMs: 1000, + extraArgs: ['--dangerously-skip-permissions'], + }); + expect(result.error?.reason).toContain('not allowed'); +}); + // Use a temp working directory so provider CLIs can't accidentally touch the repo. // Created in beforeAll / cleaned in afterAll so concurrent CI runs don't leak. let workdir: string; From 9c2414016c3cea5299ab43c951acc0d5b8e2f7a3 Mon Sep 17 00:00:00 2001 From: Mattias Petersson Date: Thu, 16 Jul 2026 09:06:58 +0200 Subject: [PATCH 10/12] fix benchmark adapters and stdin handling --- test/benchmark-cli.test.ts | 48 ++++++++++++++++++++++ test/benchmark-runner.test.ts | 11 ++++- test/helpers/benchmark-runner.ts | 20 +++++---- test/helpers/providers/gemini.ts | 14 +++++-- test/helpers/providers/gpt.ts | 18 ++++++-- test/helpers/secret-sink-harness.ts | 10 ++--- test/skill-e2e-benchmark-providers.test.ts | 16 ++++++++ 7 files changed, 117 insertions(+), 20 deletions(-) diff --git a/test/benchmark-cli.test.ts b/test/benchmark-cli.test.ts index 1912f3d8ff..aa86a9c425 100644 --- a/test/benchmark-cli.test.ts +++ b/test/benchmark-cli.test.ts @@ -142,6 +142,54 @@ describe('gstack-model-benchmark --dry-run', () => { } }); + test('Gemini OAuth-only auth is reported as NOT READY with Antigravity remediation', () => { + const oauthHome = fs.mkdtempSync(path.join(os.tmpdir(), 'bench-gemini-oauth-home-')); + try { + const oauthDir = path.join(oauthHome, '.gemini'); + fs.mkdirSync(oauthDir, { recursive: true }); + fs.writeFileSync(path.join(oauthDir, 'oauth_creds.json'), '{}'); + const result = spawnSync('bun', ['run', BIN, '--prompt', 'hi', '--models', 'gemini', '--dry-run'], { + cwd: ROOT, + env: { + PATH: process.env.PATH ?? '', + TERM: process.env.TERM ?? 'xterm', + HOME: oauthHome, + }, + encoding: 'utf-8', + timeout: 15000, + }); + expect(result.status).toBe(0); + const out = result.stdout?.toString() ?? ''; + expect(out).toMatch(/gemini:\s+NOT READY/); + expect(out).toContain('UNSUPPORTED_CLIENT'); + expect(out).toContain('`agy` adapter'); + expect(out).toContain('GOOGLE_API_KEY'); + } finally { + fs.rmSync(oauthHome, { recursive: true, force: true }); + } + }); + + test('Gemini API-key auth remains locally ready', () => { + const emptyHome = fs.mkdtempSync(path.join(os.tmpdir(), 'bench-gemini-key-home-')); + try { + const result = spawnSync('bun', ['run', BIN, '--prompt', 'hi', '--models', 'gemini', '--dry-run'], { + cwd: ROOT, + env: { + PATH: process.env.PATH ?? '', + TERM: process.env.TERM ?? 'xterm', + HOME: emptyHome, + GOOGLE_API_KEY: 'offline-readiness-test-key', + }, + encoding: 'utf-8', + timeout: 15000, + }); + expect(result.status).toBe(0); + expect(result.stdout?.toString() ?? '').toMatch(/gemini:\s+OK/); + } finally { + fs.rmSync(emptyHome, { recursive: true, force: true }); + } + }); + test('long prompt is truncated in dry-run display', () => { const longPrompt = 'x'.repeat(200); const r = run(['--prompt', longPrompt, '--dry-run']); diff --git a/test/benchmark-runner.test.ts b/test/benchmark-runner.test.ts index ecd503ea8f..cb6c0c1976 100644 --- a/test/benchmark-runner.test.ts +++ b/test/benchmark-runner.test.ts @@ -37,6 +37,13 @@ test('estimateCostUsd applies cached input discount alongside uncached input', ( expect(cost2).toBeCloseTo(8.25, 2); }); +test('estimateCostUsd prices normalized Codex cache reads at the discount', () => { + // Raw Codex usage was 21,155 total input with 9,984 cached. The adapter + // normalizes that to 11,171 uncached + 9,984 cached before pricing. + const cost = estimateCostUsd({ input: 11_171, cached: 9_984, output: 5 }, 'gpt-5.4'); + expect(cost).toBeCloseTo(0.030474, 6); +}); + test('PRICING table covers the key model families', () => { expect(PRICING['claude-opus-4-7']).toBeDefined(); expect(PRICING['claude-sonnet-4-6']).toBeDefined(); @@ -72,7 +79,7 @@ test('formatTable handles a report with mixed success/error/unavailable entries' available: true, result: { output: 'ok', - tokens: { input: 100, output: 200 }, + tokens: { input: 100, cached: 50, output: 200 }, durationMs: 800, toolCalls: 3, modelUsed: 'claude-opus-4-7', @@ -107,6 +114,8 @@ test('formatTable handles a report with mixed success/error/unavailable entries' expect(table).toContain('ERROR auth'); expect(table).toContain('unavailable'); expect(table).toContain('9.2/10'); + expect(table).toContain('100+50→200'); + expect(formatMarkdown(report)).toContain('100+50→200'); }); test('formatJson produces parseable JSON', () => { diff --git a/test/helpers/benchmark-runner.ts b/test/helpers/benchmark-runner.ts index 01ca00b737..d00ac7ab09 100644 --- a/test/helpers/benchmark-runner.ts +++ b/test/helpers/benchmark-runner.ts @@ -101,21 +101,21 @@ export async function runBenchmark(input: BenchmarkInput): Promise { diff --git a/test/helpers/providers/gpt.ts b/test/helpers/providers/gpt.ts index 07757dc2f4..8a3a66e9f3 100644 --- a/test/helpers/providers/gpt.ts +++ b/test/helpers/providers/gpt.ts @@ -46,6 +46,11 @@ export class GptAdapter implements ProviderAdapter { timeout: opts.timeoutMs, encoding: 'utf-8', maxBuffer: 32 * 1024 * 1024, + // Benchmarks can themselves run under an interactive agent process. + // Never let nested Codex inherit that process's stdin: it may wait for + // additional input instead of executing the prompt passed in `args`. + // Keep stdout/stderr piped so JSONL parsing and error routing still work. + stdio: ['ignore', 'pipe', 'pipe'], }); const parsed = this.parseJsonl(out); return { @@ -84,9 +89,10 @@ export class GptAdapter implements ProviderAdapter { * - turn.completed → usage.input_tokens, usage.output_tokens * - thread.started → session id (not used here) */ - private parseJsonl(raw: string): { output: string; tokens: { input: number; output: number }; toolCalls: number; modelUsed?: string } { + private parseJsonl(raw: string): { output: string; tokens: { input: number; output: number; cached?: number }; toolCalls: number; modelUsed?: string } { let output = ''; let input = 0; + let cached = 0; let out = 0; let toolCalls = 0; let modelUsed: string | undefined; @@ -103,7 +109,13 @@ export class GptAdapter implements ProviderAdapter { } } else if (obj.type === 'turn.completed') { const u = obj.usage ?? {}; - input += u.input_tokens ?? 0; + // Codex reports cached_input_tokens as a subset of input_tokens. + // Normalize to the benchmark contract, where input is uncached-only + // and cached is a separate bucket billed at the cache-read rate. + const turnInput = u.input_tokens ?? 0; + const turnCached = u.cached_input_tokens ?? 0; + input += Math.max(0, turnInput - turnCached); + cached += turnCached; out += u.output_tokens ?? 0; if (obj.model) modelUsed = obj.model; } @@ -111,7 +123,7 @@ export class GptAdapter implements ProviderAdapter { // skip malformed lines — codex stderr can leak in } } - return { output, tokens: { input, output: out }, toolCalls, modelUsed }; + return { output, tokens: { input, output: out, cached }, toolCalls, modelUsed }; } private emptyResult(durationMs: number, error: RunResult['error'], model?: string): RunResult { diff --git a/test/helpers/secret-sink-harness.ts b/test/helpers/secret-sink-harness.ts index d97ffd91c6..7c69a48b9b 100644 --- a/test/helpers/secret-sink-harness.ts +++ b/test/helpers/secret-sink-harness.ts @@ -88,12 +88,12 @@ export async function runWithSecretSink(opts: SecretSinkOptions): Promise { diff --git a/test/skill-e2e-benchmark-providers.test.ts b/test/skill-e2e-benchmark-providers.test.ts index bd60c71cfe..cef28e3632 100644 --- a/test/skill-e2e-benchmark-providers.test.ts +++ b/test/skill-e2e-benchmark-providers.test.ts @@ -56,6 +56,22 @@ test('agy adapter enforces sandboxed plan mode and rejects permission bypass', a expect(result.error?.reason).toContain('not allowed'); }); +test('gpt adapter disables inherited stdin for nested codex', () => { + const source = fs.readFileSync(path.join(import.meta.dir, 'helpers/providers/gpt.ts'), 'utf8'); + expect(source).toContain("stdio: ['ignore', 'pipe', 'pipe']"); +}); + +test('gpt adapter normalizes cached input out of Codex total input', () => { + const parseJsonl = (gpt as unknown as { + parseJsonl(raw: string): { tokens: { input: number; output: number; cached?: number } }; + }).parseJsonl.bind(gpt); + const parsed = parseJsonl([ + JSON.stringify({ type: 'turn.completed', usage: { input_tokens: 21_155, cached_input_tokens: 9_984, output_tokens: 5 } }), + JSON.stringify({ type: 'turn.completed', usage: { input_tokens: 200, cached_input_tokens: 150, output_tokens: 10 } }), + ].join('\n')); + expect(parsed.tokens).toEqual({ input: 11_221, cached: 10_134, output: 15 }); +}); + // Use a temp working directory so provider CLIs can't accidentally touch the repo. // Created in beforeAll / cleaned in afterAll so concurrent CI runs don't leak. let workdir: string; From e9fb8f311bcbe69a831fa022cd1b85548d4c7678 Mon Sep 17 00:00:00 2001 From: Mattias Petersson Date: Thu, 16 Jul 2026 09:18:36 +0200 Subject: [PATCH 11/12] classify provider capacity errors --- test/benchmark-provider-errors.test.ts | 27 ++++++++++++++++++++++++++ test/gen-skill-docs.test.ts | 2 +- test/helpers/providers/agy.ts | 16 +++++++++------ test/helpers/providers/claude.ts | 16 +++++++++------ test/helpers/providers/errors.ts | 20 +++++++++++++++++++ test/helpers/providers/gemini.ts | 16 +++++++++------ test/helpers/providers/gpt.ts | 16 +++++++++------ test/helpers/providers/types.ts | 1 + 8 files changed, 89 insertions(+), 25 deletions(-) create mode 100644 test/benchmark-provider-errors.test.ts create mode 100644 test/helpers/providers/errors.ts diff --git a/test/benchmark-provider-errors.test.ts b/test/benchmark-provider-errors.test.ts new file mode 100644 index 0000000000..ff2b02f04f --- /dev/null +++ b/test/benchmark-provider-errors.test.ts @@ -0,0 +1,27 @@ +import { describe, expect, test } from 'bun:test'; +import { isCapacityError, providerErrorDetail } from './helpers/providers/errors'; + +describe('benchmark provider error classification', () => { + test('recognizes the Antigravity model-capacity message', () => { + expect(isCapacityError('Selected model is at capacity. Please try a different model')).toBe(true); + }); + + test('recognizes overloaded and high-load variants', () => { + expect(isCapacityError('The provider is overloaded right now')).toBe(true); + expect(isCapacityError('Model temporarily unavailable due to high demand')).toBe(true); + }); + + test('does not misclassify auth, quota, or generic failures as capacity', () => { + expect(isCapacityError('Error 401: login required')).toBe(false); + expect(isCapacityError('429 quota exceeded')).toBe(false); + expect(isCapacityError('connection reset by peer')).toBe(false); + }); + + test('combines stderr and message without duplicating subprocess output', () => { + const stderr = 'Selected model is at capacity.'; + expect(providerErrorDetail({ stderr: Buffer.from(stderr), message: `Command failed\n${stderr}` })) + .toBe(`Command failed\n${stderr}`); + expect(providerErrorDetail({ stderr: Buffer.from('stderr only'), message: 'message only' })) + .toBe('stderr only\nmessage only'); + }); +}); diff --git a/test/gen-skill-docs.test.ts b/test/gen-skill-docs.test.ts index c06b69f8e7..0548b686ac 100644 --- a/test/gen-skill-docs.test.ts +++ b/test/gen-skill-docs.test.ts @@ -2452,7 +2452,7 @@ describe('setup script validation', () => { expect(runtimeFn).toContain('agy_version="$(tr -d \'[:space:]\' < "$gstack_dir/VERSION")"'); expect(runtimeFn).toContain('"$agy_version" > "$agy_gstack/plugin.json"'); - expect(runtimeFn).not.toContain('"version": "1.58.5.0"'); + expect(runtimeFn).not.toMatch(/"version": "\d+\.\d+\.\d+\.\d+"/); expect(linkFn).toContain('-name SKILL.md'); expect(linkFn).toContain('bun run gen:skill-docs --host agy'); }); diff --git a/test/helpers/providers/agy.ts b/test/helpers/providers/agy.ts index c37129f082..e30a34bfac 100644 --- a/test/helpers/providers/agy.ts +++ b/test/helpers/providers/agy.ts @@ -4,6 +4,7 @@ import { execFileSync, spawnSync } from 'child_process'; import * as fs from 'fs'; import * as path from 'path'; import * as os from 'os'; +import { isCapacityError, providerErrorDetail } from './errors'; /** * Agy adapter — wraps the `agy` CLI using `agy --print`. @@ -64,17 +65,20 @@ export class AgyAdapter implements ProviderAdapter { } catch (err: unknown) { const durationMs = Date.now() - start; const e = err as { code?: string; stderr?: Buffer; signal?: string; message?: string }; - const stderr = e.stderr?.toString() ?? ''; + const detail = providerErrorDetail(e); if (e.signal === 'SIGTERM' || e.code === 'ETIMEDOUT') { return this.emptyResult(durationMs, { code: 'timeout', reason: `exceeded ${opts.timeoutMs}ms` }, opts.model); } - if (/unauthorized|auth|login|api key/i.test(stderr)) { - return this.emptyResult(durationMs, { code: 'auth', reason: stderr.slice(0, 400) }, opts.model); + if (/unauthorized|auth|login|api key/i.test(detail)) { + return this.emptyResult(durationMs, { code: 'auth', reason: detail.slice(0, 400) }, opts.model); } - if (/rate[- ]?limit|429|quota/i.test(stderr)) { - return this.emptyResult(durationMs, { code: 'rate_limit', reason: stderr.slice(0, 400) }, opts.model); + if (/rate[- ]?limit|429|quota/i.test(detail)) { + return this.emptyResult(durationMs, { code: 'rate_limit', reason: detail.slice(0, 400) }, opts.model); } - return this.emptyResult(durationMs, { code: 'unknown', reason: (e.message ?? stderr ?? 'unknown').slice(0, 400) }, opts.model); + if (isCapacityError(detail)) { + return this.emptyResult(durationMs, { code: 'capacity', reason: detail.slice(0, 400) }, opts.model); + } + return this.emptyResult(durationMs, { code: 'unknown', reason: (detail || 'unknown').slice(0, 400) }, opts.model); } } diff --git a/test/helpers/providers/claude.ts b/test/helpers/providers/claude.ts index ce77767c2d..07eb7e6ca3 100644 --- a/test/helpers/providers/claude.ts +++ b/test/helpers/providers/claude.ts @@ -5,6 +5,7 @@ import * as fs from 'fs'; import * as path from 'path'; import * as os from 'os'; import { resolveClaudeCommand } from '../../../browse/src/claude-bin'; +import { isCapacityError, providerErrorDetail } from './errors'; /** * Claude adapter — wraps the `claude` CLI via claude -p. @@ -67,17 +68,20 @@ export class ClaudeAdapter implements ProviderAdapter { } catch (err: unknown) { const durationMs = Date.now() - start; const e = err as { code?: string; stderr?: Buffer; signal?: string; message?: string }; - const stderr = e.stderr?.toString() ?? ''; + const detail = providerErrorDetail(e); if (e.signal === 'SIGTERM' || e.code === 'ETIMEDOUT') { return this.emptyResult(durationMs, { code: 'timeout', reason: `exceeded ${opts.timeoutMs}ms` }, opts.model); } - if (/unauthorized|auth|login/i.test(stderr)) { - return this.emptyResult(durationMs, { code: 'auth', reason: stderr.slice(0, 400) }, opts.model); + if (/unauthorized|auth|login/i.test(detail)) { + return this.emptyResult(durationMs, { code: 'auth', reason: detail.slice(0, 400) }, opts.model); } - if (/rate[- ]?limit|429/i.test(stderr)) { - return this.emptyResult(durationMs, { code: 'rate_limit', reason: stderr.slice(0, 400) }, opts.model); + if (/rate[- ]?limit|429/i.test(detail)) { + return this.emptyResult(durationMs, { code: 'rate_limit', reason: detail.slice(0, 400) }, opts.model); } - return this.emptyResult(durationMs, { code: 'unknown', reason: (e.message ?? stderr ?? 'unknown').slice(0, 400) }, opts.model); + if (isCapacityError(detail)) { + return this.emptyResult(durationMs, { code: 'capacity', reason: detail.slice(0, 400) }, opts.model); + } + return this.emptyResult(durationMs, { code: 'unknown', reason: (detail || 'unknown').slice(0, 400) }, opts.model); } } diff --git a/test/helpers/providers/errors.ts b/test/helpers/providers/errors.ts new file mode 100644 index 0000000000..29b6a9d23c --- /dev/null +++ b/test/helpers/providers/errors.ts @@ -0,0 +1,20 @@ +export interface ProviderProcessError { + stderr?: Buffer | string; + message?: string; +} + +/** Combine subprocess diagnostics without duplicating identical text. */ +export function providerErrorDetail(error: ProviderProcessError): string { + const stderr = typeof error.stderr === 'string' + ? error.stderr + : error.stderr?.toString() ?? ''; + const message = error.message ?? ''; + return stderr && message.includes(stderr) + ? message + : [stderr, message].filter(Boolean).join('\n'); +} + +/** Provider/model saturation is transient and distinct from auth or quota. */ +export function isCapacityError(detail: string): boolean { + return /(?:selected\s+)?model\s+is\s+at\s+capacity|\bat\s+capacity\b|\bmodel\s+capacity\b|\boverloaded\b|temporarily unavailable due to (?:high )?(?:load|demand)/i.test(detail); +} diff --git a/test/helpers/providers/gemini.ts b/test/helpers/providers/gemini.ts index 46f8dc9a09..93d866c7d6 100644 --- a/test/helpers/providers/gemini.ts +++ b/test/helpers/providers/gemini.ts @@ -4,6 +4,7 @@ import { execFileSync, spawnSync } from 'child_process'; import * as fs from 'fs'; import * as path from 'path'; import * as os from 'os'; +import { isCapacityError, providerErrorDetail } from './errors'; /** * Gemini adapter — wraps the `gemini` CLI. @@ -65,17 +66,20 @@ export class GeminiAdapter implements ProviderAdapter { } catch (err: unknown) { const durationMs = Date.now() - start; const e = err as { code?: string; stderr?: Buffer; signal?: string; message?: string }; - const stderr = e.stderr?.toString() ?? ''; + const detail = providerErrorDetail(e); if (e.signal === 'SIGTERM' || e.code === 'ETIMEDOUT') { return this.emptyResult(durationMs, { code: 'timeout', reason: `exceeded ${opts.timeoutMs}ms` }, opts.model); } - if (/unauthorized|auth|login|api key/i.test(stderr)) { - return this.emptyResult(durationMs, { code: 'auth', reason: stderr.slice(0, 400) }, opts.model); + if (/unauthorized|auth|login|api key/i.test(detail)) { + return this.emptyResult(durationMs, { code: 'auth', reason: detail.slice(0, 400) }, opts.model); } - if (/rate[- ]?limit|429|quota/i.test(stderr)) { - return this.emptyResult(durationMs, { code: 'rate_limit', reason: stderr.slice(0, 400) }, opts.model); + if (/rate[- ]?limit|429|quota/i.test(detail)) { + return this.emptyResult(durationMs, { code: 'rate_limit', reason: detail.slice(0, 400) }, opts.model); } - return this.emptyResult(durationMs, { code: 'unknown', reason: (e.message ?? stderr ?? 'unknown').slice(0, 400) }, opts.model); + if (isCapacityError(detail)) { + return this.emptyResult(durationMs, { code: 'capacity', reason: detail.slice(0, 400) }, opts.model); + } + return this.emptyResult(durationMs, { code: 'unknown', reason: (detail || 'unknown').slice(0, 400) }, opts.model); } } diff --git a/test/helpers/providers/gpt.ts b/test/helpers/providers/gpt.ts index 8a3a66e9f3..7dc19faa8f 100644 --- a/test/helpers/providers/gpt.ts +++ b/test/helpers/providers/gpt.ts @@ -4,6 +4,7 @@ import { execFileSync, spawnSync } from 'child_process'; import * as fs from 'fs'; import * as path from 'path'; import * as os from 'os'; +import { isCapacityError, providerErrorDetail } from './errors'; /** * GPT adapter — wraps the OpenAI `codex` CLI (codex exec with --json output). @@ -63,17 +64,20 @@ export class GptAdapter implements ProviderAdapter { } catch (err: unknown) { const durationMs = Date.now() - start; const e = err as { code?: string; stderr?: Buffer; signal?: string; message?: string }; - const stderr = e.stderr?.toString() ?? ''; + const detail = providerErrorDetail(e); if (e.signal === 'SIGTERM' || e.code === 'ETIMEDOUT') { return this.emptyResult(durationMs, { code: 'timeout', reason: `exceeded ${opts.timeoutMs}ms` }, opts.model); } - if (/unauthorized|auth|login/i.test(stderr)) { - return this.emptyResult(durationMs, { code: 'auth', reason: stderr.slice(0, 400) }, opts.model); + if (/unauthorized|auth|login/i.test(detail)) { + return this.emptyResult(durationMs, { code: 'auth', reason: detail.slice(0, 400) }, opts.model); } - if (/rate[- ]?limit|429/i.test(stderr)) { - return this.emptyResult(durationMs, { code: 'rate_limit', reason: stderr.slice(0, 400) }, opts.model); + if (/rate[- ]?limit|429/i.test(detail)) { + return this.emptyResult(durationMs, { code: 'rate_limit', reason: detail.slice(0, 400) }, opts.model); } - return this.emptyResult(durationMs, { code: 'unknown', reason: (e.message ?? stderr ?? 'unknown').slice(0, 400) }, opts.model); + if (isCapacityError(detail)) { + return this.emptyResult(durationMs, { code: 'capacity', reason: detail.slice(0, 400) }, opts.model); + } + return this.emptyResult(durationMs, { code: 'unknown', reason: (detail || 'unknown').slice(0, 400) }, opts.model); } } diff --git a/test/helpers/providers/types.ts b/test/helpers/providers/types.ts index 1680d0ceb1..ddaa01f09c 100644 --- a/test/helpers/providers/types.ts +++ b/test/helpers/providers/types.ts @@ -31,6 +31,7 @@ export type RunError = | 'auth' // Credentials missing or invalid. | 'timeout' // Exceeded timeoutMs. | 'rate_limit' // Provider rate-limited us; backoff exceeded. + | 'capacity' // Selected model/provider is temporarily saturated. | 'binary_missing' // CLI not found on PATH. | 'unknown'; // Catch-all with reason populated. From eccb4aef495019dc758ddecf7b1492cea371f339 Mon Sep 17 00:00:00 2001 From: Mattias Petersson Date: Thu, 16 Jul 2026 09:39:02 +0200 Subject: [PATCH 12/12] fix Claude skill template validation --- scripts/skill-check.ts | 15 +++++++++++++-- test/skill-check.test.ts | 19 +++++++++++++++++++ 2 files changed, 32 insertions(+), 2 deletions(-) create mode 100644 test/skill-check.test.ts diff --git a/scripts/skill-check.ts b/scripts/skill-check.ts index 9182737ee1..850d58845e 100644 --- a/scripts/skill-check.ts +++ b/scripts/skill-check.ts @@ -13,6 +13,7 @@ import { discoverTemplates, discoverSkillFiles } from './discover-skills'; import * as fs from 'fs'; import * as path from 'path'; import { execSync } from 'child_process'; +import { claude, getExternalHosts } from '../hosts/index'; const ROOT = path.resolve(import.meta.dir, '..'); const ROOT_REALPATH = fs.realpathSync(ROOT); @@ -64,14 +65,26 @@ for (const file of SKILL_FILES) { console.log('\n Templates:'); const TEMPLATES = discoverTemplates(ROOT); +const CLAUDE_SKIPPED_SKILLS = new Set(claude.generation.skipSkills ?? []); for (const { tmpl, output } of TEMPLATES) { const tmplPath = path.join(ROOT, tmpl); const outPath = path.join(ROOT, output); + const skillDir = path.dirname(tmpl); + const skillName = skillDir === '.' ? null : skillDir.split(path.sep)[0]; if (!fs.existsSync(tmplPath)) { console.log(` \u26a0\ufe0f ${output.padEnd(30)} — no template`); continue; } + if (skillName && CLAUDE_SKIPPED_SKILLS.has(skillName)) { + if (fs.existsSync(outPath)) { + hasErrors = true; + console.log(` \u274c ${output.padEnd(30)} — generated file exists but Claude Code skips this skill`); + } else { + console.log(` \u23ed\ufe0f ${output.padEnd(30)} — intentionally skipped for Claude Code`); + } + continue; + } if (!fs.existsSync(outPath)) { hasErrors = true; console.log(` \u274c ${output.padEnd(30)} — generated file missing! Run: bun run gen:skill-docs`); @@ -90,8 +103,6 @@ for (const file of SKILL_FILES) { // ─── External Host Skills (config-driven) ─────────────────── -import { getExternalHosts } from '../hosts/index'; - for (const hostConfig of getExternalHosts()) { const hostDir = path.join(ROOT, hostConfig.hostSubdir, 'skills'); if (fs.existsSync(hostDir)) { diff --git a/test/skill-check.test.ts b/test/skill-check.test.ts new file mode 100644 index 0000000000..f895efe5bf --- /dev/null +++ b/test/skill-check.test.ts @@ -0,0 +1,19 @@ +import { describe, expect, test } from 'bun:test'; +import { spawnSync } from 'child_process'; +import * as path from 'path'; + +const ROOT = path.resolve(import.meta.dir, '..'); + +describe('skill-check template coverage', () => { + test('accepts skills intentionally skipped by the Claude host', () => { + const result = spawnSync('bun', ['run', 'scripts/skill-check.ts'], { + cwd: ROOT, + encoding: 'utf8', + }); + + expect(result.status).toBe(0); + expect(result.stdout).toContain('claude/SKILL.md'); + expect(result.stdout).toContain('intentionally skipped for Claude Code'); + expect(result.stdout).not.toContain('generated file missing'); + }); +});