diff --git a/.github/workflows/benchmark-external.yml b/.github/workflows/benchmark-external.yml
new file mode 100644
index 0000000..9fec890
--- /dev/null
+++ b/.github/workflows/benchmark-external.yml
@@ -0,0 +1,111 @@
+name: NAD Benchmark (label-triggered)
+
+# The active NAD benchmark gate. Policy: EVERY PR (internal or external fork) is
+# gated on demand — a maintainer reviews the PR and adds the `run-benchmark`
+# label to run it. This is the only NAD benchmark workflow.
+#
+# Maintainers can also run it MANUALLY from the Actions tab (Run workflow /
+# workflow_dispatch) against any branch, with optional subset/count/gate inputs.
+# Manual runs are trusted (only maintainers can trigger them), so they check out
+# the selected branch directly instead of an untrusted PR head.
+#
+# It uses pull_request_target so the run has access to KIRO_API_KEY even for fork
+# PRs (a plain pull_request gate can't — fork PRs receive no secrets), and is
+# guarded so it runs ONLY when a maintainer explicitly opts a PR in.
+#
+# How it's controlled:
+# - Triggers ONLY on the `labeled` event, and the job runs ONLY if the label is
+# `run-benchmark`. Applying a label requires the repo's Triage role or above;
+# a fork contributor has no role on this repo, so nothing runs until a
+# maintainer (or a member explicitly granted Triage+) reviews the PR and
+# applies the label.
+# - A later push does NOT re-run it — a maintainer must re-apply the label, i.e.
+# re-review, before new commits are scored.
+# - permissions: contents: read (drops the default write-capable GITHUB_TOKEN).
+#
+# The person who applies the label is vouching for the PR: this job checks out and
+# runs the PR's contributed skill in a job that holds KIRO_API_KEY, so only label
+# PRs whose changes you've looked at. NOTE: `contents: read` restricts only the
+# GITHUB_TOKEN — it does NOT shield KIRO_API_KEY, which is fully exposed to the
+# untrusted PR code this job runs. Review the diff before labeling. Use a
+# service-account key (not a personal one) so it can be rotated independently.
+
+on:
+ pull_request_target:
+ # ONLY the label event — never `synchronize`/`opened`, so untrusted code
+ # never auto-runs on push. A maintainer must (re-)apply the label each time.
+ types: [labeled]
+
+ # Manual runs from the Actions tab. Only users with write access can dispatch
+ # this, so the run is trusted — it scores the selected branch's own code.
+ workflow_dispatch:
+ inputs:
+ subset:
+ description: "Subset(s): 'all', a single name, or a comma-separated list"
+ default: 'all'
+ count:
+ description: 'Pass@k attempts per scenario'
+ default: '1'
+ gate:
+ description: 'Required pass rate (0-100); job fails below this'
+ default: '90'
+
+# Least privilege: the base-context GITHUB_TOKEN is write-capable by default under
+# pull_request_target. Restrict it to read so untrusted code can't use it to push,
+# open releases, or modify the repo.
+permissions:
+ contents: read
+
+concurrency:
+ group: nad-benchmark-external-${{ github.event.pull_request.number || github.ref }}
+ cancel-in-progress: true
+
+jobs:
+ gate:
+ # Run on a manual dispatch, OR when a maintainer added the `run-benchmark`
+ # label. For the label path this `if` is the trust gate — without it,
+ # pull_request_target would run untrusted code with secrets on every PR event.
+ if: github.event_name == 'workflow_dispatch' || github.event.label.name == 'run-benchmark'
+ name: NAD external benchmark (labeled)
+ runs-on: ubuntu-latest
+ timeout-minutes: 20
+
+ steps:
+ # Label runs: check out the PR's HEAD — i.e. the contributor's UNTRUSTED
+ # code. This is required (we must score their skill) and is the crux of the
+ # risk above. The workflow FILE itself still comes from base
+ # (pull_request_target), so the contributor cannot alter these steps — only
+ # the skill/agent/test/grader content that the steps then run.
+ # Manual runs: head.sha is empty, so checkout falls back to the selected
+ # branch (trusted — only maintainers can dispatch).
+ - name: Checkout PR head (untrusted on label runs)
+ uses: actions/checkout@v4
+ with:
+ ref: ${{ github.event.pull_request.head.sha }}
+
+ - name: Setup Python
+ uses: actions/setup-python@v5
+ with:
+ python-version: '3.11'
+
+ - name: Run NAD benchmark
+ env:
+ KIRO_API_KEY: ${{ secrets.KIRO_API_KEY }}
+ run: |
+ set -euo pipefail
+ # Inputs come from the manual dispatch form; label runs use the defaults.
+ python benchmark/run_skill_tests.py \
+ --nad-root=. \
+ --subset="${{ github.event.inputs.subset || 'all' }}" \
+ --count="${{ github.event.inputs.count || '1' }}" \
+ --workers=4 \
+ --judge \
+ --gate="${{ github.event.inputs.gate || '90' }}"
+
+ - name: Upload results
+ if: always()
+ uses: actions/upload-artifact@v4
+ with:
+ name: nad-benchmark-results
+ path: benchmark/build/nad_benchmark_results.json
+ if-no-files-found: warn
diff --git a/.gitignore b/.gitignore
new file mode 100644
index 0000000..8801dd2
--- /dev/null
+++ b/.gitignore
@@ -0,0 +1,5 @@
+
+# NAD benchmark artifacts
+benchmark/build/
+__pycache__/
+*.pyc
diff --git a/agents/neuron-dev-contribution-validation-agent.agent-spec.json b/agents/neuron-dev-contribution-validation-agent.agent-spec.json
new file mode 100644
index 0000000..914eb0a
--- /dev/null
+++ b/agents/neuron-dev-contribution-validation-agent.agent-spec.json
@@ -0,0 +1,20 @@
+{
+ "schemaVersion": "1",
+ "name": "neuron-dev-contribution-validation-agent",
+ "config": {
+ "description": "Validates agentic artifact contributions to the NeuronAgenticDevelopment (NAD) package against the contribution model requirements. Use when reviewing CRs or checking contribution readiness.",
+ "systemPrompt": "{{aim:include:agents/neuron-dev-contribution-validation-agent.md}}",
+ "model": "claude-sonnet-4.6"
+ },
+ "dependencies": {
+ "skills": {
+ "skillNames": ["neuron-dev-contribution-validation"]
+ }
+ },
+ "clientConfig": {
+ "kiroCli": {
+ "tools": ["@builtin"],
+ "allowedTools": ["fs_read", "fs_write", "grep", "glob", "execute_bash", "todo_list"]
+ }
+ }
+}
diff --git a/agents/neuron-dev-contribution-validation-agent.md b/agents/neuron-dev-contribution-validation-agent.md
new file mode 100644
index 0000000..34ebf6b
--- /dev/null
+++ b/agents/neuron-dev-contribution-validation-agent.md
@@ -0,0 +1,42 @@
+---
+name: neuron-dev-contribution-validation-agent
+description: |
+ Validates agentic artifact contributions to the NeuronAgenticDevelopment (NAD) package
+ against the contribution model requirements. Use when reviewing CRs, checking contribution
+ readiness, or validating that skills/agents meet NAD standards.
+
+
+ Context: User wants to validate their NAD contribution before submitting a CR
+ user: "Validate my contribution to NAD"
+ assistant: "I'll use neuron-dev-contribution-validation-agent to check your artifacts against the contribution model."
+
+
+
+ Context: User is reviewing a CR for NAD
+ user: "Review this CR for NAD contribution compliance"
+ assistant: "Let me use neuron-dev-contribution-validation-agent to validate the contribution."
+
+
+model: sonnet
+tools: ["Read", "Grep", "Glob", "Bash", "Task", "TodoWrite", "Skill"]
+skills:
+ - neuron-dev-contribution-validation
+---
+
+You are a NAD contribution validation agent. Your job is to validate agentic artifact
+contributions (skills, agents, hooks, tools) to the NeuronAgenticDevelopment package
+against the contribution model requirements.
+
+When asked to validate a contribution:
+
+1. Use Glob and Read to discover the artifacts being contributed
+2. Check each artifact against the validation checklist from the neuron-dev-contribution-validation skill
+3. Report results in the structured format defined by the skill
+
+Always check:
+- Namespace compliance (valid prefix from the approved list)
+- Required files exist (SKILL.md, NEURON_METADATA.md)
+- NEURON_METADATA.md has all required fields
+- Agent specs exist in both Kiro and Claude Code formats (if contributing an agent)
+- AIM benchmark tests exist and are registered
+- Minimum test counts are met for the release category
diff --git a/benchmark/README.md b/benchmark/README.md
new file mode 100644
index 0000000..dc2eabd
--- /dev/null
+++ b/benchmark/README.md
@@ -0,0 +1,94 @@
+# NAD Skill Benchmark Gate
+
+A CI gate that evaluates this repo's agent skills by running them through the
+`kiro-cli` agent in Docker and scoring the output against per-scenario rubrics.
+A PR that changes `skills/`, `agents/`, `tests/`, or this `benchmark/` grader is
+scored against the repo's own assets.
+
+## Layout (single-repo)
+
+Everything the gate needs lives in this repo:
+
+- **Inputs** (at the repo root, scored by the gate):
+ - `skills/` — the skills under test
+ - `agents/` — agent specs (`*.agent-spec.json`) listing each agent's `skillNames`
+ - `tests/` — scenario definitions per subset + `tests/registry.json` (the source
+ of truth for which scenarios are *active/gated*)
+- **Grader** (this `benchmark/` directory):
+ - `run_skill_tests.py` — loads scenarios, runs the agent in Docker, scores, gates
+ - `shell/{docker,common}.sh` — build the image + run `kiro-cli` in Docker
+ - `environment/Dockerfile` — the run image (`python:3.11-slim` + kiro-cli)
+
+## How it runs
+
+The gate scores the repo's own `skills/` + `agents/` + `tests/` by running
+`benchmark/run_skill_tests.py --nad-root=.`. No cross-repo references or tokens.
+
+**Every PR is gated on demand via a label** (`.github/workflows/benchmark-external.yml`):
+
+- A maintainer reviews a PR and adds the **`run-benchmark`** label; that triggers
+ the gate on the PR. Applying a label requires the repo's Triage role or above, so
+ a fork contributor (no role on this repo) can't opt themselves in — nothing runs
+ until a maintainer does. Pushing new commits does not re-run it — the maintainer
+ must re-apply the label (re-review).
+- It uses `pull_request_target` so the run can access the `KIRO_API_KEY` secret
+ even for PRs from external forks (a plain `pull_request` run gets no secrets on
+ forks). The person who labels a PR is vouching for it: the job runs the PR's
+ contributed skill code with the key present, so only label PRs you've reviewed.
+ Note that `contents: read` restricts only the `GITHUB_TOKEN` — it does **not**
+ shield `KIRO_API_KEY`, which is exposed to the untrusted PR code, so review the
+ diff before labeling.
+- **Requires** on the repo: a `KIRO_API_KEY` Actions secret (authenticates both the
+ agent and the LLM judge; use a service-account key) and a `run-benchmark` label.
+
+The same workflow also supports **manual** `workflow_dispatch` runs from the Actions
+tab (inputs: `subset`, `count`, `gate`). Manual dispatch is restricted to users with
+write access, so it checks out the selected branch directly (trusted) rather than an
+untrusted PR head.
+
+## Scoring
+
+Per scenario:
+
+- `pattern_score` (75%) — proportional match of `expectedStrings` (present) and
+ `forbiddenStrings` (absent), matched against the output file + the agent's response.
+- `judge_score` (25%) — an LLM judge (`kiro-cli`) verdict against the scenario's
+ rubric: `1.0` pass / `0.0` fail (or `null` if no rubric).
+- `combined_score = 0.75*pattern + 0.25*judge` (pattern-only if the judge is off).
+- `passed = combined_score >= 0.90` (per-scenario override via `metadata.baseline_k`).
+
+The gate fails if the overall pass rate is below `--gate` (default 90%).
+
+> **Note on the judge:** the LLM judge is non-deterministic, so a scenario with a
+> perfect `pattern_score=1.00` can still score `judge=0` on an unlucky verdict,
+> pulling `combined` to 0.75. The overall pass rate can therefore vary run-to-run
+> near the threshold independent of any code change.
+
+## Reliability handling
+
+- **No infra-error retry.** A failure is reported as-is — including a timeout that
+ produced no output, which typically reflects the *agent* going down a bad path
+ (e.g. fetching a dead URL and stalling). This is skill behavior the benchmark
+ should measure, not hide by retrying until it happens to pass. For deliberate
+ multi-attempt runs use `--count` (transparent Pass@k).
+- **Salvage-on-timeout:** if the agent times out but has already written a non-empty
+ `output.py`, that output is scored instead of being discarded (the agent often
+ finishes the task and then hangs on a trivial final step). This scores what the
+ agent actually produced — it does not grant a fresh attempt. A timeout that
+ produced no output is reported as an infrastructure error (`run_status:
+ infra_error`), logged with its exception type + container-output tail.
+
+## Running it
+
+```bash
+# On a PR (primary path): a maintainer adds the `run-benchmark` label to the PR.
+
+# Manually via CLI (workflow_dispatch on benchmark-external.yml):
+gh workflow run benchmark-external.yml -f subset=all -f count=1 -f gate=90
+
+# Locally (needs Docker + KIRO_API_KEY):
+python benchmark/run_skill_tests.py --nad-root=. --subset=all --count=1 --judge --gate=90
+```
+
+`--subset` accepts `all`, a single subset name (e.g. `neuron-nki-agent`), or a
+comma-separated list. Results are written to `benchmark/build/nad_benchmark_results.json`.
diff --git a/benchmark/environment/Dockerfile b/benchmark/environment/Dockerfile
new file mode 100644
index 0000000..653fa8b
--- /dev/null
+++ b/benchmark/environment/Dockerfile
@@ -0,0 +1,13 @@
+FROM python:3.11-slim
+
+WORKDIR /app
+
+# Install system deps for kiro-cli
+RUN apt-get update && apt-get install -y curl unzip && rm -rf /var/lib/apt/lists/*
+
+# Install kiro-cli
+RUN curl -fsSL https://cli.kiro.dev/install | bash
+
+ENV PATH="/root/.local/bin:$PATH"
+
+COPY . .
diff --git a/benchmark/run_skill_tests.py b/benchmark/run_skill_tests.py
new file mode 100644
index 0000000..b96d3ac
--- /dev/null
+++ b/benchmark/run_skill_tests.py
@@ -0,0 +1,656 @@
+"""Run all NeuronAgenticDevelopment skill tests using skills-benchmarks framework.
+
+Each scenario runs kiro-cli inside Docker (same flow as Claude in skills-benchmarks).
+Parallelized with ProcessPoolExecutor — one Docker container per scenario.
+
+Usage:
+ python run_skill_tests.py --subset=neuron-nki-agent --count=2 --workers=4
+ python run_skill_tests.py --subset=all --count=1 --judge # all subsets
+ python run_skill_tests.py --task-id=beta3-no-platform-target --count=1
+ python run_skill_tests.py --subset=neuron-nki-agent --count=1 --judge
+"""
+
+import json
+import os
+import shutil
+import subprocess
+import sys
+import tempfile
+from concurrent.futures import ProcessPoolExecutor, as_completed
+from pathlib import Path
+
+# NAD benchmark inputs (tests/, skills/, agents/) live at the repo root and are
+# resolved via --nad-root (default ".", handled in main via set_nad_root). In this
+# single-repo layout the gate scores the repo's own assets, so --nad-root=. points
+# at the checkout root. --nad-root can also point at any other NAD checkout to
+# score that repo's native assets instead.
+#
+# FRAMEWORK_DIR (shell/, environment/Dockerfile, build/) is SEPARATE and always anchored to
+# this file — only the three input dirs move behind --nad-root.
+PROJECT_DIR = Path(__file__).parent.parent
+TESTS_DIR = PROJECT_DIR / "tests"
+SKILLS_DIR = PROJECT_DIR / "skills"
+AGENTS_DIR = PROJECT_DIR / "agents"
+FRAMEWORK_DIR = Path(__file__).parent
+DOCKER_SH = FRAMEWORK_DIR / "shell" / "docker.sh"
+
+
+def set_nad_root(nad_root: str) -> None:
+ """Point the input paths (tests/, skills/, agents/) at a NAD checkout.
+
+ Reassigns the module-level input dirs so both the main process and forked
+ worker processes (ProcessPoolExecutor uses fork on Linux) see the same root.
+ Call this BEFORE run_all() creates the executor. FRAMEWORK_DIR is unaffected.
+ """
+ global PROJECT_DIR, TESTS_DIR, SKILLS_DIR, AGENTS_DIR
+ PROJECT_DIR = Path(nad_root).expanduser().resolve()
+ TESTS_DIR = PROJECT_DIR / "tests"
+ SKILLS_DIR = PROJECT_DIR / "skills"
+ AGENTS_DIR = PROJECT_DIR / "agents"
+
+PATTERN_WEIGHT = 0.75
+JUDGE_WEIGHT = 0.25
+DEFAULT_BASELINE_K = 0.90
+
+# NOTE: There is deliberately NO infra-error retry. A timeout that produced no
+# output.py reflects the AGENT going down a bad path (e.g. fetching a 404 URL,
+# wandering, hanging) — that is skill behavior the benchmark should MEASURE, not
+# mask by retrying until it happens to succeed. Retrying would turn "this skill
+# fails ~1 in N runs" into a false 100% pass. Completed-but-hung runs are instead
+# recovered honestly by the salvage path in _run_scenario_once (score the output
+# the agent actually wrote). For deliberate multi-attempt runs, use --count
+# (Pass@k), which is transparent in the reported metric.
+
+# Max chars of the submitted file handed to the LLM judge. The old 4000 cap
+# silently truncated large outputs (e.g. a ~25K-char autoport modeling file),
+# so the judge only saw the first ~16% and reported the file "cut off" — a
+# harness artifact, not a real failure. Set well above the largest expected
+# submission so the judge sees the whole file.
+JUDGE_SOURCE_LIMIT = 40000
+
+
+def load_registry_active() -> dict:
+ """Map subset name -> set of active (gated) taskIds, from registry.json.
+
+ registry.json is the source of truth for which scenarios are ACTIVE. Some
+ subsets have extra fixtures on disk that are not gated (e.g. autoport ships 4
+ *.json but only 1 is active), so globbing files would over-count. Returns {}
+ if the registry is missing (callers then fall back to all on-disk files).
+ """
+ reg_file = TESTS_DIR / "registry.json"
+ if not reg_file.exists():
+ return {}
+ reg = json.loads(reg_file.read_text())
+ return {s["name"]: set(s.get("taskIdSubset", [])) for s in reg.get("subsets", [])}
+
+
+def load_scenarios(subset: str = None, task_id: str = None) -> list[dict]:
+ """Load scenarios, optionally filtered.
+
+ subset may be None/"all" (every subset), a single subset name, or a
+ comma-separated list. Scenarios are restricted to the registry's active
+ taskIds unless a specific --task-id is requested (which allows loading
+ inactive fixtures for debugging).
+ """
+ # None / "" / "all" means every subset; otherwise accept a comma-sep list.
+ wanted = None
+ if subset and subset.strip().lower() != "all":
+ wanted = {s.strip() for s in subset.split(",")}
+
+ # Only gate to registry-active scenarios for normal runs; an explicit
+ # --task-id may target an inactive fixture, so skip the filter in that case.
+ active = load_registry_active() if not task_id else {}
+
+ scenarios = []
+ for subset_dir in sorted(TESTS_DIR.iterdir()):
+ if not subset_dir.is_dir() or subset_dir.name.startswith("."):
+ continue
+ if wanted is not None and subset_dir.name not in wanted:
+ continue
+ active_ids = active.get(subset_dir.name)
+ for json_file in sorted(subset_dir.glob("*.json")):
+ data = json.loads(json_file.read_text())
+ for s in data.get("scenarios", []):
+ # Registry is the source of truth for active/gated scenarios.
+ if active_ids is not None and s["taskId"] not in active_ids:
+ continue
+ s["_subset"] = subset_dir.name
+ scenarios.append(s)
+
+ if task_id:
+ task_ids = [t.strip() for t in task_id.split(",")]
+ scenarios = [s for s in scenarios if s["taskId"] in task_ids]
+
+ return scenarios
+
+
+def get_agent_skills(agent_name: str) -> list[str]:
+ """Get skill names from agent spec."""
+ spec_file = AGENTS_DIR / f"{agent_name}.agent-spec.json"
+ if not spec_file.exists():
+ return []
+ spec = json.loads(spec_file.read_text())
+ return spec.get("dependencies", {}).get("skills", {}).get("skillNames", [])
+
+
+def setup_workspace(skills: list[str], dockerfile_dir: Path) -> Path:
+ """Create temp workspace with skills and Dockerfile for Docker execution."""
+ test_dir = Path(tempfile.mkdtemp(prefix="skill_eval_"))
+
+ # Install skills into .kiro/skills/ — copy full directory (references, examples, etc.)
+ for skill_name in skills:
+ skill_src = SKILLS_DIR / skill_name
+ if not skill_src.exists():
+ continue
+ skill_dst = test_dir / ".kiro" / "skills" / skill_name
+ shutil.copytree(skill_src, skill_dst)
+
+ # Copy Dockerfile into test_dir (required by docker.sh build)
+ shutil.copy(dockerfile_dir / "Dockerfile", test_dir / "Dockerfile")
+
+ return test_dir
+
+
+def run_kiro_in_docker(test_dir: Path, prompt: str, timeout: int = 120) -> str:
+ """Run kiro-cli inside Docker container. Returns stdout+stderr."""
+ cmd = ["bash", str(DOCKER_SH), "run-kiro", str(test_dir), prompt, "--timeout", str(timeout)]
+ result = subprocess.run(
+ cmd, capture_output=True, text=True,
+ timeout=timeout + 30, env=os.environ.copy(),
+ )
+ return result.stdout + result.stderr
+
+
+def _strip_ansi(text: str) -> str:
+ """Remove ANSI escape sequences from text.
+
+ Covers SGR color codes (…m) AND cursor/mode codes like \\x1b[?25l (hide cursor,
+ emitted per spinner frame) — the latter otherwise flood the log as '25l25l25l…'.
+ """
+ import re
+ return re.sub(r'\x1b\[[?0-9;]*[a-zA-Z]', '', text)
+
+
+def extract_agent_response(raw_transcript: str) -> str:
+ """Extract the agent's own response prose from a kiro-cli transcript.
+
+ kiro-cli prefixes the assistant's narration lines with "> "; everything else
+ is tool output (files it read, shell results, skill reference docs). We match
+ expectedStrings/forbiddenStrings against the agent's response ONLY — never the
+ tool output — mirroring the internal harness, which matches its `actualOutput`
+ (the agent's final response), not what the agent read.
+
+ This distinction is load-bearing: 35 of 42 NKI forbiddenStrings (deprecated
+ APIs like `nl.load`, `neuronxcc`, `mask=`) ALSO appear in the skill reference
+ docs the agent reads. Matching the whole transcript would false-flag those as
+ violations. Matching the "> " response lines does not.
+ """
+ text = _strip_ansi(raw_transcript)
+ lines = [ln.strip()[2:] for ln in text.splitlines() if ln.strip().startswith("> ")]
+ return "\n".join(lines)
+
+
+def _extract_json_object(text: str) -> dict | None:
+ """Extract the first balanced {...} JSON object from arbitrary text.
+
+ kiro-cli may pretty-print the verdict across multiple lines or wrap it with
+ surrounding prose/footer, so neither json.loads(whole) nor a single-line
+ scan is reliable. Walk the text tracking brace depth (ignoring braces inside
+ strings) and try to parse each complete top-level object.
+ """
+ start = None
+ depth = 0
+ in_str = False
+ escape = False
+ for i, ch in enumerate(text):
+ if start is None:
+ if ch == "{":
+ start = i
+ depth = 1
+ continue
+ if escape:
+ escape = False
+ elif ch == "\\":
+ escape = True
+ elif ch == '"':
+ in_str = not in_str
+ elif not in_str:
+ if ch == "{":
+ depth += 1
+ elif ch == "}":
+ depth -= 1
+ if depth == 0:
+ candidate = text[start:i + 1]
+ try:
+ return json.loads(candidate)
+ except json.JSONDecodeError:
+ start = None # not valid; look for the next object
+ return None
+
+
+def score_judge(source: str, rubric: str, dockerfile_dir: Path) -> tuple[float, str, str]:
+ """Score via LLM judge using kiro-cli in Docker. Returns (score, reason, status).
+
+ The judge runs through the same Docker path as the agent (docker.sh run-kiro)
+ so it shares one kiro-cli install and the KIRO_API_KEY auth passthrough — the
+ host runner does NOT need kiro-cli installed. The judge workspace has NO skills
+ loaded, so it is a raw LLM call (no agent context).
+
+ status is one of:
+ "ok" — a real verdict was parsed (score is 0.0 or 1.0)
+ "skipped" — no rubric to judge against
+ "error" — the judge could not run or returned no parseable verdict
+ """
+ if not rubric:
+ return 0.5, "no rubric", "skipped"
+
+ prompt = (
+ "You are an expert code evaluator. Judge whether the following code meets the rubric.\n\n"
+ f"## Rubric\n{rubric}\n\n"
+ f"## Code to evaluate\n```python\n{source[:JUDGE_SOURCE_LIMIT]}\n```\n\n"
+ 'Respond ONLY with JSON, no other text: {"passed": true/false, "reason": "brief explanation"}'
+ )
+
+ judge_dir = None
+ try:
+ # Empty workspace (no skills) so the judge is a plain LLM call, not an agent.
+ judge_dir = setup_workspace([], dockerfile_dir)
+ raw = run_kiro_in_docker(judge_dir, prompt, timeout=60)
+
+ # Strip ANSI color codes and kiro-cli prompt markers before parsing
+ output = _strip_ansi(raw).strip()
+ cleaned_lines = []
+ for line in output.splitlines():
+ line = line.strip()
+ if line.startswith("> "):
+ line = line[2:]
+ cleaned_lines.append(line)
+ output = "\n".join(cleaned_lines)
+
+ # kiro-cli may pretty-print the JSON across lines or wrap it in prose,
+ # so extract the first balanced {...} object rather than assuming it is
+ # the whole output or on a single line.
+ verdict = _extract_json_object(output)
+
+ if verdict is None:
+ return 0.5, f"no JSON in output: {output[:80]}", "error"
+
+ passed = verdict.get("passed", False)
+ reason = verdict.get("reason", "")
+ return (1.0 if passed else 0.0), reason, "ok"
+ except subprocess.TimeoutExpired:
+ return 0.5, "judge timeout", "error"
+ except Exception as e:
+ return 0.5, f"judge error: {str(e)[:50]}", "error"
+ finally:
+ if judge_dir is not None:
+ shutil.rmtree(judge_dir, ignore_errors=True)
+
+
+def score_patterns(haystack: str, expected: list[str], forbidden: list[str]) -> tuple[float, list, list, list]:
+ """Score expected/forbidden string matches, case-insensitively.
+
+ `haystack` is what we match against — see evaluate(): the submitted file
+ PLUS the agent's response prose (extract_agent_response, not raw tool output).
+ Matching is against the agent's final response text (case-insensitively), not
+ raw tool output.
+ Some expectedStrings are not source tokens at all — e.g. the autoport
+ scenario expects `neuron_port/modeling_`, an output *path* the agent names
+ in its narration but that never appears inside the .py file. Matching the
+ file alone caps such scenarios below pass; matching file+transcript does not.
+ """
+ if not haystack:
+ return 0.0, [], expected, []
+ total = len(expected) + len(forbidden)
+ if total == 0:
+ return 1.0, [], [], []
+ hay = haystack.lower()
+ hits = [p for p in expected if p.lower() in hay]
+ misses = [p for p in expected if p.lower() not in hay]
+ violations = [p for p in forbidden if p.lower() in hay]
+ score = (len(hits) + (len(forbidden) - len(violations))) / total
+ return score, hits, misses, violations
+
+
+def evaluate(source: str, scenario: dict, use_judge: bool = False, dockerfile_dir: Path = None,
+ agent_response: str = "") -> dict:
+ """Evaluate output against scenario spec.
+
+ `source` is the submitted file (output.py); `agent_response` is the agent's
+ own response prose (see extract_agent_response — NOT the raw transcript).
+ Pattern matching (approach "a") runs over BOTH so path-style expectedStrings
+ that appear only in the agent's narration still match, while the LLM judge
+ scores the SUBMITTED FILE only (the rubric explicitly judges "the final
+ Python file the agent pasted").
+ """
+ expected = scenario.get("expectedStrings", [])
+ forbidden = scenario.get("forbiddenStrings", [])
+ baseline_k = scenario.get("metadata", {}).get("baseline_k", DEFAULT_BASELINE_K)
+
+ # Match against file + agent response so narration-only strings (e.g. output
+ # paths) are found; the file is included so genuine code tokens still match.
+ # We deliberately exclude tool output (see extract_agent_response).
+ haystack = source + "\n" + agent_response
+ p_score, hits, misses, violations = score_patterns(haystack, expected, forbidden)
+
+ j_score = 0.5
+ j_reason = "skipped"
+ j_status = "off" # judge not requested
+ if use_judge:
+ rubric = scenario.get("expectedOutput", "")
+ j_score, j_reason, j_status = score_judge(source, rubric, dockerfile_dir)
+
+ # Only fold the judge into the combined score when it produced a real verdict.
+ # A "skipped" (no rubric) or "error" (couldn't run) judge must NOT silently
+ # degrade to pattern-only without surfacing why — see judge_status below.
+ judge_scored = j_status == "ok"
+ if judge_scored:
+ combined = (PATTERN_WEIGHT * p_score) + (JUDGE_WEIGHT * j_score)
+ else:
+ combined = p_score
+
+ # Always emit all three scores so a reader never has to guess what a single
+ # "score" means. combined_score is the headline (75% pattern + 25% judge when
+ # the judge ran; pattern-only otherwise — judge_status says which).
+ return {
+ "combined_score": combined,
+ "pattern_score": p_score,
+ "judge_score": j_score if judge_scored else None,
+ "baseline_k": baseline_k,
+ "passed": combined >= baseline_k,
+ "judge_reason": j_reason if j_status in ("ok", "error") else None,
+ "judge_status": j_status,
+ "expected_hits": hits,
+ "expected_misses": misses,
+ "forbidden_violations": violations,
+ }
+
+
+def _run_scenario_once(scenario: dict, skills: list, dockerfile_dir: Path, use_judge: bool) -> dict:
+ """One agent-run + scoring attempt for a scenario.
+
+ Returns a result dict. On an infrastructure failure (Docker/agent timeout,
+ etc.) returns run_status='infra_error' rather than raising, so the caller can
+ decide whether to retry. A returned dict WITHOUT run_status is a real scored
+ result (pass or genuine skill failure).
+ """
+ prompt = "\n".join(scenario["input"]) + "\n\nSave your output to output.py"
+ test_dir = setup_workspace(skills, dockerfile_dir)
+ try:
+ # 300s per-scenario agent budget (matches docker.sh's default). Heavy
+ # scenarios (e.g. the autoport model port) can still exceed this under
+ # parallel load and time out; a timeout that already produced output.py is
+ # salvaged below, and one that produced nothing is reported as infra_error.
+ transcript = run_kiro_in_docker(test_dir, prompt, timeout=300)
+ output_file = test_dir / "output.py"
+ source = output_file.read_text() if output_file.exists() else ""
+ # Match patterns against file + agent's response prose only (not tool
+ # output) — see extract_agent_response for why that distinction matters.
+ agent_response = extract_agent_response(transcript)
+ return evaluate(source, scenario, use_judge=use_judge,
+ dockerfile_dir=dockerfile_dir, agent_response=agent_response)
+ except subprocess.TimeoutExpired as e:
+ # The agent/Docker call exceeded the wall-clock budget. Capture the type,
+ # the actual timeout, and whatever the container printed before it was
+ # killed (TimeoutExpired carries partial stdout/stderr).
+ partial = ""
+ for stream in (e.stdout, e.stderr):
+ if stream:
+ partial += stream.decode() if isinstance(stream, bytes) else stream
+ partial = _strip_ansi(partial)
+
+ # SALVAGE completed work: the agent often finishes the real task (writes
+ # output.py) and then hangs on a trivial final step (e.g. a spinner after
+ # "save the port summary"), timing out with a VALID output already on disk.
+ # Killing that as infra_error throws away a real result and triggers a
+ # pointless retry of already-done work. If output.py exists, score it —
+ # the timeout was a tail-end hang, not a failure to produce output.
+ output_file = test_dir / "output.py"
+ if output_file.exists() and output_file.read_text().strip():
+ source = output_file.read_text()
+ agent_response = extract_agent_response(partial) # partial transcript
+ result = evaluate(source, scenario, use_judge=use_judge,
+ dockerfile_dir=dockerfile_dir, agent_response=agent_response)
+ # Note the salvage so it's visible we scored a timed-out-but-complete run.
+ result["salvaged_after_timeout"] = f"timed out after {e.timeout}s; scored existing output.py"
+ return result
+
+ # No usable output — a genuine infra failure; keep it as infra_error so the
+ # retry loop can try again.
+ return {
+ "combined_score": 0,
+ "passed": False,
+ "run_status": "infra_error",
+ "run_error_type": "TimeoutExpired",
+ "run_error": f"timed out after {e.timeout}s (no output.py produced)",
+ "run_error_detail": partial[-2000:], # tail of container output
+ "expected_misses": [],
+ "forbidden_violations": [],
+ }
+ except Exception as e:
+ # Any other infrastructure failure (Docker build/run error, kiro-cli
+ # crash, etc.) — NOT a skill failure. Record the exception TYPE + a wide
+ # message so the real cause is visible, not truncated to 120 chars.
+ return {
+ "combined_score": 0,
+ "passed": False,
+ "run_status": "infra_error",
+ "run_error_type": type(e).__name__,
+ "run_error": str(e)[:2000],
+ "expected_misses": [],
+ "forbidden_violations": [],
+ }
+ finally:
+ shutil.rmtree(test_dir, ignore_errors=True)
+
+
+def run_single_scenario(args: tuple) -> dict:
+ """Run a single scenario in Docker. Called by ProcessPoolExecutor.
+
+ Pass@k loop: run up to `count` times and keep the first PASS (transparent
+ Pass@k semantics). There is NO infra-error retry — a failure (including a
+ timeout that produced no output) is a real result and is reported as-is; the
+ agent going down a bad path is skill behavior the benchmark should measure,
+ not hide. A completed-but-hung run is recovered by salvage in _run_scenario_once.
+ """
+ scenario, skills, dockerfile_dir, count, use_judge = args
+ task_id_str = scenario["taskId"]
+ subset_str = scenario.get("_subset")
+ result = {"combined_score": 0, "passed": False, "expected_misses": [], "forbidden_violations": []}
+
+ for _attempt in range(count):
+ result = _run_scenario_once(scenario, skills, dockerfile_dir, use_judge)
+ # Log infra errors so a failed run is legible to a human (Docker crash vs.
+ # agent-wander vs. genuine skill fail) — visibility, not auto-retry.
+ if result.get("run_status") == "infra_error":
+ print(f" [infra_error] {task_id_str} attempt {_attempt + 1}: "
+ f"[{result.get('run_error_type')}] {result.get('run_error')}", flush=True)
+ if result.get("run_error_detail"):
+ print(f" container output (tail):\n{result['run_error_detail']}", flush=True)
+ if result["passed"]:
+ break # keep the first pass (Pass@k)
+
+ result.setdefault("run_status", "ok")
+ return {"taskId": task_id_str, "subset": subset_str, **result}
+
+
+def run_all(subset: str = None, task_id: str = None, count: int = 2, workers: int = 4, use_judge: bool = False):
+ """Run all scenarios in parallel Docker containers."""
+ scenarios = load_scenarios(subset, task_id)
+ if not scenarios:
+ print("No scenarios found.")
+ return
+
+ dockerfile_dir = FRAMEWORK_DIR / "environment"
+
+ # Each subset maps to its own agent spec with its own skills. Resolve skills
+ # PER SUBSET so a multi-subset run gives each scenario the right skills
+ # (previously one agent's skills were used for every scenario).
+ subsets_present = sorted({s["_subset"] for s in scenarios})
+ skills_by_subset = {name: get_agent_skills(name) for name in subsets_present}
+
+ judge_str = "pattern(75%) + judge(25%)" if use_judge else "pattern-only"
+ for name in subsets_present:
+ n = sum(1 for s in scenarios if s["_subset"] == name)
+ skills = skills_by_subset[name]
+ print(f"Agent: {name} ({n} scenario(s)) skills: {', '.join(skills) if skills else 'none'}")
+ print(f"Scenarios: {len(scenarios)}")
+ print(f"Pass@{count} | Workers: {workers} | Docker: yes | Scoring: {judge_str}")
+ print("=" * 80)
+
+ # Pre-build Docker image once
+ subprocess.run(
+ ["bash", str(DOCKER_SH), "build", str(dockerfile_dir)],
+ capture_output=True, timeout=300,
+ )
+
+ # Run in parallel processes — each scenario carries its own subset's skills.
+ work = [(s, skills_by_subset[s["_subset"]], dockerfile_dir, count, use_judge) for s in scenarios]
+ results = []
+
+ with ProcessPoolExecutor(max_workers=workers) as executor:
+ futures = {executor.submit(run_single_scenario, w): w[0]["taskId"] for w in work}
+ for future in as_completed(futures):
+ r = future.result()
+ results.append(r)
+ status = "✓" if r["passed"] else "✗"
+ # Infra failure (Docker/agent error) is neither pass nor a real fail —
+ # mark it distinctly so it's not read as "the skill produced bad code".
+ if r.get("run_status") == "infra_error":
+ status = "⚠"
+ # combined_score is the headline; also show its pattern/judge parts.
+ line = f" {status} {r['taskId']:<45} combined={r['combined_score']:.2f}"
+ if r.get("pattern_score") is not None:
+ line += f" pattern={r['pattern_score']:.2f}"
+ if r.get("judge_score") is not None:
+ line += f" judge={r['judge_score']:.0f}"
+ elif r.get("judge_status") == "error":
+ # Judge was requested but could not run — make it loud, don't hide it.
+ line += " judge=ERR"
+ if r.get("run_status") == "infra_error":
+ # Show the exception TYPE + message so the cause is diagnosable
+ # (e.g. TimeoutExpired vs. a Docker/kiro-cli crash), not opaque.
+ etype = r.get("run_error_type", "")
+ line += f" INFRA_ERROR[{etype}]: {r.get('run_error', '')[:120]}"
+ elif not r["passed"]:
+ if r.get("expected_misses"):
+ line += f" missing={r['expected_misses']}"
+ if r.get("forbidden_violations"):
+ line += f" forbidden={r['forbidden_violations']}"
+ # Note when a timed-out-but-complete run was salvaged (scored its output.py).
+ if r.get("salvaged_after_timeout"):
+ line += " (salvaged)"
+ if r.get("judge_reason") and r.get("judge_status") in ("ok", "error"):
+ print(line, flush=True)
+ print(f" judge: {r['judge_reason'][:100]}", flush=True)
+ else:
+ print(line, flush=True)
+
+ # Summary
+ total_passed = sum(1 for r in results if r["passed"])
+ pass_rate = total_passed / len(scenarios) * 100
+ print("=" * 80)
+ print(f"RESULT: {total_passed}/{len(scenarios)} passed ({pass_rate:.0f}%)")
+ print(f"TARGET: 90%+ | STATUS: {'✓ MEETS TARGET' if pass_rate >= 90 else '✗ BELOW TARGET'}")
+
+ # Per-subset breakdown — meaningful when running more than one subset so a
+ # single weak subset isn't hidden inside an aggregate pass rate.
+ by_subset = sorted({r.get("subset") for r in results})
+ if len(by_subset) > 1:
+ for name in by_subset:
+ rs = [r for r in results if r.get("subset") == name]
+ p = sum(1 for r in rs if r["passed"])
+ print(f" {name}: {p}/{len(rs)} passed ({p / len(rs) * 100:.0f}%)")
+
+ # Infra errors (Docker/agent failures) are counted here so they're not read
+ # as skill failures. They still count against pass rate, but this makes the
+ # cause visible — a low pass rate driven by flakes is not a skill regression.
+ infra_errors = [r for r in results if r.get("run_status") == "infra_error"]
+ if infra_errors:
+ print(f"INFRA: {len(infra_errors)} scenario(s) hit an infrastructure error "
+ f"(not skill failures): {[r['taskId'] for r in infra_errors]}")
+ # Dump each infra error's type + captured detail so the actual cause is
+ # visible in the log (e.g. what the container printed before a timeout).
+ for r in infra_errors:
+ print(f" --- {r['taskId']} [{r.get('run_error_type', '?')}]: {r.get('run_error', '')}")
+ if r.get("run_error_detail"):
+ print(f" container output (tail):\n{r['run_error_detail']}")
+
+ # Salvaged runs: the agent timed out but had already written a valid output.py,
+ # so it was scored rather than discarded. Surfaced for visibility.
+ salvaged = [r for r in results if r.get("salvaged_after_timeout")]
+ if salvaged:
+ print(f"SALVAGED: {len(salvaged)} scenario(s) timed out but had output.py "
+ f"scored: {[r['taskId'] for r in salvaged]}")
+
+ # Judge health: if the judge was requested, surface how many scenarios it
+ # actually scored vs. errored. A wholesale failure (0 scored) means the
+ # combined scores silently fell back to pattern-only — flag it loudly.
+ if use_judge:
+ judged_ok = sum(1 for r in results if r.get("judge_status") == "ok")
+ judged_err = sum(1 for r in results if r.get("judge_status") == "error")
+ print(f"JUDGE: {judged_ok}/{len(results)} scored | {judged_err} errored")
+ if judged_ok == 0:
+ print(" ⚠ WARNING: judge ran on NO scenarios — scores are pattern-only. "
+ "Check kiro-cli/Docker/KIRO_API_KEY.")
+
+ # Write machine-readable results for CI artifact upload / regression detection
+ results_file = FRAMEWORK_DIR / "build" / "nad_benchmark_results.json"
+ results_file.parent.mkdir(parents=True, exist_ok=True)
+ results_file.write_text(json.dumps(results, indent=2))
+ print(f"Results written to: {results_file}")
+
+ return results
+
+
+if __name__ == "__main__":
+ import argparse
+ parser = argparse.ArgumentParser()
+ parser.add_argument(
+ "--subset",
+ default="neuron-nki-agent",
+ help="Subset(s) to run: a single name, a comma-separated list, or 'all' "
+ "for every subset (nki + autoport + validation). Restricted to "
+ "registry-active scenarios unless --task-id is given.",
+ )
+ parser.add_argument(
+ "--nad-root",
+ default=str(Path(__file__).parent.parent),
+ help="Path to the NAD inputs root containing tests/, skills/, agents/. "
+ "Defaults to the repo root (this is a single-repo layout, so the gate scores "
+ "the repo's own assets). Point it at another NAD checkout to score that "
+ "repo's native assets instead.",
+ )
+ parser.add_argument("--task-id", default=None)
+ parser.add_argument("--count", type=int, default=1)
+ parser.add_argument("--workers", type=int, default=4)
+ parser.add_argument("--judge", action="store_true", help="Enable LLM judge (25%% weight) via kiro-cli")
+ parser.add_argument(
+ "--gate",
+ type=float,
+ default=None,
+ help="Exit non-zero if pass rate (0-100) is below this threshold. "
+ "Use --gate=100 for a strict PR gate. Omit for informational runs.",
+ )
+ args = parser.parse_args()
+
+ # Repoint input dirs before run_all() forks workers, so every process agrees.
+ set_nad_root(args.nad_root)
+ if not TESTS_DIR.exists():
+ print(f"ERROR: no tests/ under --nad-root={PROJECT_DIR}", file=sys.stderr)
+ sys.exit(2)
+
+ results = run_all(
+ subset=args.subset, task_id=args.task_id, count=args.count,
+ workers=args.workers, use_judge=args.judge,
+ )
+
+ if args.gate is not None:
+ results = results or []
+ total = len(results)
+ passed = sum(1 for r in results if r["passed"])
+ pass_rate = (passed / total * 100) if total else 0.0
+ if pass_rate < args.gate:
+ print(f"GATE FAILED: {pass_rate:.0f}% < required {args.gate:.0f}%")
+ sys.exit(1)
+ print(f"GATE PASSED: {pass_rate:.0f}% >= required {args.gate:.0f}%")
diff --git a/benchmark/shell/common.sh b/benchmark/shell/common.sh
new file mode 100755
index 0000000..4368f01
--- /dev/null
+++ b/benchmark/shell/common.sh
@@ -0,0 +1,44 @@
+#!/usr/bin/env bash
+# Common shell utilities, sourced by docker.sh.
+# Source this file: source "$(dirname "${BASH_SOURCE[0]}")/common.sh"
+
+set -euo pipefail
+
+# =============================================================================
+# COLORS
+# =============================================================================
+
+RED='\033[0;31m'
+GREEN='\033[0;32m'
+YELLOW='\033[0;33m'
+BLUE='\033[0;34m'
+NC='\033[0m' # No Color
+
+# =============================================================================
+# LOGGING
+# =============================================================================
+
+log_info() {
+ echo -e "${BLUE}[INFO]${NC} $*"
+}
+
+log_success() {
+ echo -e "${GREEN}[OK]${NC} $*"
+}
+
+log_warn() {
+ echo -e "${YELLOW}[WARN]${NC} $*"
+}
+
+log_error() {
+ echo -e "${RED}[ERROR]${NC} $*" >&2
+}
+
+# =============================================================================
+# ERROR HANDLING
+# =============================================================================
+
+die() {
+ log_error "$@"
+ exit 1
+}
diff --git a/benchmark/shell/docker.sh b/benchmark/shell/docker.sh
new file mode 100755
index 0000000..2e8184f
--- /dev/null
+++ b/benchmark/shell/docker.sh
@@ -0,0 +1,256 @@
+#!/usr/bin/env bash
+# Docker utilities for the NAD skill benchmark.
+# Builds the run image (cached by Dockerfile hash) and runs kiro-cli in a
+# container. Called from run_skill_tests.py as:
+# ./docker.sh build
+# ./docker.sh run-kiro [--model MODEL] [--timeout SECONDS]
+
+set -euo pipefail
+
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+source "$SCRIPT_DIR/common.sh"
+
+# Image prefix for all benchmark images
+IMAGE_PREFIX="skillbench"
+
+# Cross-platform timeout command (macOS uses gtimeout from coreutils)
+TIMEOUT_CMD=""
+if command -v gtimeout &> /dev/null; then
+ TIMEOUT_CMD="gtimeout"
+elif command -v timeout &> /dev/null; then
+ TIMEOUT_CMD="timeout"
+fi
+
+# Env vars passed through to the container. kiro-cli authenticates with
+# KIRO_API_KEY (drives both the agent and the judge).
+ENV_KEYS=(
+ KIRO_API_KEY
+)
+
+# =============================================================================
+# DOCKER CHECKS
+# =============================================================================
+
+check_docker() {
+ if ! command -v docker &> /dev/null; then
+ echo "ERROR: Docker not found" >&2
+ return 1
+ fi
+ if ! docker info &> /dev/null 2>&1; then
+ echo "ERROR: Docker daemon not running" >&2
+ return 1
+ fi
+ return 0
+}
+
+# =============================================================================
+# HASH-BASED IMAGE CACHING
+# =============================================================================
+
+# Get hash of build inputs for cache key (the Dockerfile). Only the Dockerfile
+# affects the image; skills + prompt are added into the workspace at runtime,
+# so hashing the whole dir would change the key every run and defeat the cache.
+get_dockerfile_hash() {
+ local dir="$1"
+ local dockerfile="$dir/Dockerfile"
+
+ if [[ ! -f "$dockerfile" ]]; then
+ echo ""
+ return 1
+ fi
+
+ if command -v md5 &> /dev/null; then
+ cat "$dockerfile" | md5 -q | cut -c1-8
+ else
+ cat "$dockerfile" | md5sum | cut -c1-8
+ fi
+}
+
+# Get image name for a directory (based on Dockerfile hash)
+get_image_name() {
+ local dir="$1"
+ local hash
+ hash=$(get_dockerfile_hash "$dir") || return 1
+ echo "${IMAGE_PREFIX}:${hash}"
+}
+
+# Check if image exists
+image_exists() {
+ local image_name="$1"
+ docker images -q "$image_name" 2>/dev/null | grep -q .
+}
+
+# =============================================================================
+# DOCKER BUILD
+# =============================================================================
+
+# Build Docker image with caching
+# Usage: docker_build [--force]
+# Output: image name on stdout
+docker_build() {
+ local dir="$1"
+ local force="${2:-}"
+
+ local dockerfile="$dir/Dockerfile"
+ if [[ ! -f "$dockerfile" ]]; then
+ echo "ERROR: No Dockerfile in $dir" >&2
+ return 1
+ fi
+
+ local image_name
+ image_name=$(get_image_name "$dir") || return 1
+
+ # Check cache unless forced
+ if [[ "$force" != "--force" ]] && image_exists "$image_name"; then
+ echo "$image_name"
+ return 0
+ fi
+
+ if docker build -t "$image_name" -f "$dockerfile" "$dir" >&2; then
+ echo "$image_name"
+ return 0
+ else
+ echo "ERROR: Build failed" >&2
+ return 1
+ fi
+}
+
+# =============================================================================
+# DOCKER RUN
+# =============================================================================
+
+# Build env var arguments for docker run (populates ENV_ARGS array)
+build_env_args() {
+ ENV_ARGS=()
+ for key in "${ENV_KEYS[@]}"; do
+ if [[ -n "${!key:-}" ]]; then
+ ENV_ARGS+=("-e" "$key=${!key}")
+ fi
+ done
+}
+
+# Run Kiro CLI in Docker
+# Usage: docker_run_kiro [--model MODEL] [--timeout SECONDS]
+docker_run_kiro() {
+ local dir="$1"
+ local prompt="$2"
+ shift 2
+
+ local model=""
+ local timeout="300"
+
+ while [[ $# -gt 0 ]]; do
+ case "$1" in
+ --model)
+ model="$2"
+ shift 2
+ ;;
+ --timeout)
+ timeout="$2"
+ shift 2
+ ;;
+ *)
+ shift
+ ;;
+ esac
+ done
+
+ local image_name
+ image_name=$(docker_build "$dir") || return 1
+
+ build_env_args
+
+ local cmd=(
+ kiro-cli chat "$prompt"
+ --no-interactive
+ -a
+ )
+
+ if [[ -n "$model" ]]; then
+ cmd+=(--model "$model")
+ fi
+
+ # Auth: KIRO_API_KEY env var (CI), or fall back to the host kiro-cli SSO
+ # cache for local dev when the key isn't set.
+ local auth_args=()
+ if [[ -z "${KIRO_API_KEY:-}" ]]; then
+ local auth_db="$HOME/.local/share/kiro-cli/data.sqlite3"
+ if [[ -f "$auth_db" ]]; then
+ auth_args+=("-v" "$auth_db:/root/.local/share/kiro-cli/data.sqlite3")
+ fi
+ fi
+
+ if [[ -n "$TIMEOUT_CMD" ]]; then
+ $TIMEOUT_CMD "$timeout" docker run --rm \
+ -v "$dir:/workspace" \
+ -w /workspace \
+ ${ENV_ARGS[@]+"${ENV_ARGS[@]}"} \
+ ${auth_args[@]+"${auth_args[@]}"} \
+ "$image_name" \
+ "${cmd[@]}"
+ else
+ docker run --rm \
+ -v "$dir:/workspace" \
+ -w /workspace \
+ ${ENV_ARGS[@]+"${ENV_ARGS[@]}"} \
+ ${auth_args[@]+"${auth_args[@]}"} \
+ "$image_name" \
+ "${cmd[@]}"
+ fi
+}
+
+# =============================================================================
+# CLI MODE
+# =============================================================================
+
+if [[ "${BASH_SOURCE[0]}" == "${0}" ]]; then
+ cmd="${1:-help}"
+ shift || true
+
+ case "$cmd" in
+ check)
+ if check_docker; then
+ echo "OK"
+ else
+ exit 1
+ fi
+ ;;
+ build)
+ dir="${1:-}"
+ force="${2:-}"
+ if [[ -z "$dir" ]]; then
+ die "Usage: $0 build [--force]"
+ fi
+ docker_build "$(realpath "$dir")" "$force"
+ ;;
+ run-kiro)
+ dir="${1:-}"
+ prompt="${2:-}"
+ if [[ -z "$dir" || -z "$prompt" ]]; then
+ die "Usage: $0 run-kiro [--model MODEL] [--timeout SECONDS]"
+ fi
+ shift 2
+ docker_run_kiro "$(realpath "$dir")" "$prompt" "$@"
+ ;;
+ help|*)
+ cat < [args...]
+
+Commands:
+ check Check if Docker is available
+ build [--force] Build image (cached by Dockerfile hash)
+ run-kiro [opts] Run kiro-cli in the container
+
+Options for run-kiro:
+ --model MODEL Model to use
+ --timeout SECONDS Timeout (default: 300)
+
+Auth:
+ KIRO_API_KEY is passed into the container (agent + judge). If unset, the host
+ kiro-cli SSO cache (~/.local/share/kiro-cli/data.sqlite3) is mounted for local dev.
+EOF
+ ;;
+ esac
+fi
diff --git a/skills/neuron-dev-contribution-validation/NEURON_METADATA.md b/skills/neuron-dev-contribution-validation/NEURON_METADATA.md
new file mode 100644
index 0000000..9a050ae
--- /dev/null
+++ b/skills/neuron-dev-contribution-validation/NEURON_METADATA.md
@@ -0,0 +1,5 @@
+---
+owner_resolver_group: NeuronGenAI
+release_category: internal
+master_repository_uri: ssh://git.amazon.com/pkg/NeuronAgenticDevelopment
+---
diff --git a/skills/neuron-dev-contribution-validation/SKILL.md b/skills/neuron-dev-contribution-validation/SKILL.md
new file mode 100644
index 0000000..a478fcf
--- /dev/null
+++ b/skills/neuron-dev-contribution-validation/SKILL.md
@@ -0,0 +1,121 @@
+---
+name: neuron-dev-contribution-validation
+description: |
+ Validate agentic artifact contributions (skills, agents, hooks, tools) to the
+ NeuronAgenticDevelopment (NAD) package against the contribution model requirements.
+ Use when reviewing a CR for NAD, checking contribution readiness, or asking
+ "does this contribution meet NAD requirements", "validate my NAD contribution",
+ "review CR for contribution model compliance".
+---
+
+# NAD Contribution Validation
+
+Validate contributions to the NeuronAgenticDevelopment package against the contribution model.
+
+## Activation
+
+When asked to validate a NAD contribution, review a CR for NAD compliance, or check if artifacts meet contribution requirements:
+
+1. Identify the artifact(s) being contributed (skills, agents, hooks, tools, MCP)
+2. Determine the release category (internal or external) from `NEURON_METADATA.md`
+3. Run through the validation checklist below
+4. Report results as PASS/FAIL per item with actionable guidance for failures
+
+## Validation Checklist
+
+### 1. Namespace Compliance
+
+- [ ] Artifact name uses a valid namespace prefix
+- [ ] Valid namespaces: `neuron-nki`, `neuron-framework`, `neuron-core`, `neuron-infra`, `neuron-dev`, `neuron-training`
+- [ ] `experimental-` prefix used only for single-team internal artifacts
+- [ ] Name is unique — no existing artifact with the same name in the repository
+
+### 2. Skill Structure (if contributing a skill)
+
+- [ ] `SKILL.md` exists with valid YAML frontmatter (`name`, `description` required)
+- [ ] `name` field: lowercase, hyphens only, max 64 chars, matches directory name
+- [ ] `description` field: non-empty, max 1024 chars, describes what and when
+- [ ] `NEURON_METADATA.md` exists with required fields
+- [ ] Directory follows the structure: `skills/-/`
+
+### 3. NEURON_METADATA.md Validation
+
+- [ ] Contains YAML frontmatter with all three required fields:
+ - `owner_resolver_group` — valid resolver group name
+ - `release_category` — either `internal` or `external`
+ - `master_repository_uri` — valid repository URI
+- [ ] `release_category` matches the intended audience
+
+### 4. Agent Structure (if contributing an agent)
+
+- [ ] Claude Code YAML spec exists in `agents/` directory
+- [ ] Kiro JSON spec exists in `agents/` directory
+- [ ] Kiro spec `systemPrompt` references Claude Code markdown via aim:include directive
+- [ ] Both specs reference the same set of skills
+- [ ] Agent name uses correct namespace prefix
+
+### 5. AIM Benchmark Tests
+
+Determine minimum count based on `release_category`:
+- **Internal**: ≥ 1 test, 100% pass@k
+- **External**: ≥ 5 tests, 100% pass@k
+
+Validate:
+- [ ] Test scenario files exist in `tests/-/`
+- [ ] Each test has: `taskId`, `input`, `expectedOutput`, `metadata`
+- [ ] Tests use `expectedStrings`/`forbiddenStrings` for determinism
+- [ ] Test is registered in `tests/registry.json`
+- [ ] Registry entry has: `name`, `version`, `datasetPath`, `defaultJudge`, `taskIdSubset`
+- [ ] `taskIdSubset` lists all contributed test IDs
+- [ ] Minimum test count met for the release category
+
+### 6. Evaluation Tests
+
+Determine minimum count based on `release_category`:
+- **Internal**: ≥ 1 eval, 70% pass@k
+- **External**: ≥ 5 evals, 70% pass@k
+
+Validate:
+- [ ] Eval artifacts exist in `evals///`
+- [ ] `data/` subdirectory contains evaluation results
+- [ ] `test/` subdirectory contains test execution script
+- [ ] Minimum eval count met for the release category
+
+### 7. External-Only Requirements (if `release_category: external`)
+
+- [ ] No `experimental-` prefix used
+- [ ] No dependencies on internal or IP-sensitive artifacts
+
+### 8. CR Review Requirements
+
+- [ ] At least 1 reviewer from the owning team
+- [ ] Neuroboros reviewers: 1 for existing artifacts, 2 for new submissions
+- [ ] External artifacts: 2 Neuroboros reviewers required
+
+## Output Format
+
+Present results as:
+
+```
+## NAD Contribution Validation Report
+
+**Artifact**:
+**Type**:
+**Release Category**:
+**Overall**: PASS | FAIL (N issues)
+
+### Results
+
+| # | Check | Status | Notes |
+|---|-------|--------|-------|
+| 1 | Namespace compliance | ✅ PASS | |
+| 2 | Skill structure | ✅ PASS | |
+| ... | ... | ... | ... |
+
+### Issues (if any)
+1.
+```
+
+## Reference
+
+For the full contribution model details, see [references/contribution-model.md](references/contribution-model.md).
diff --git a/skills/neuron-dev-contribution-validation/references/contribution-model.md b/skills/neuron-dev-contribution-validation/references/contribution-model.md
new file mode 100644
index 0000000..dd1601d
--- /dev/null
+++ b/skills/neuron-dev-contribution-validation/references/contribution-model.md
@@ -0,0 +1,182 @@
+# NAD Contribution Model Reference
+
+Complete validation criteria for contributions to the NeuronAgenticDevelopment package.
+
+## Namespaces
+
+| Namespace | Area |
+|-----------|------|
+| `neuron-nki` | NKI Library, Compiler, Optimization, Writing, Debugging |
+| `neuron-framework` | NxDI, vLLM Neuron |
+| `neuron-core` | Neuron Runtime, Collectives, Graph Compiler |
+| `neuron-infra` | Kaizen, Slurm |
+| `neuron-dev` | SDLC, CR |
+| `neuron-training` | moduscope, TorchNeuronEager |
+
+The `experimental-` prefix is for artifacts intended for limited usage by one internal team.
+
+## Required Directory Structures
+
+### Skill
+
+```
+skills/-/
+├── SKILL.md # Required
+├── NEURON_METADATA.md # Required
+├── scripts/ # Optional
+├── references/ # Optional
+├── assets/ # Optional
+└── ...
+```
+
+### Agent
+
+Agents live in `agents/` and require both formats:
+
+**Claude Code** (`agents/-.md` + YAML frontmatter):
+```yaml
+---
+name: -
+description: |
+ Description of what the agent does.
+model: opus
+color: green
+tools: ["Read", "Write", "Edit", "Grep", "Glob", "Bash", "Task", "TodoWrite", "Skill"]
+skills:
+ - -
+ - -
+---
+```
+
+**Kiro** (`agents/-.json`):
+```json
+{
+ "schemaVersion": "1",
+ "name": "-",
+ "config": {
+ "description": "Description of what the agent does.",
+ "systemPrompt": "{{ aim:include:agents/AGENT_NAME.md }}",
+ "model": "claude-opus-4.6"
+ },
+ "dependencies": {
+ "skills": {
+ "skillNames": ["-", "-"]
+ }
+ },
+ "clientConfig": {
+ "kiroCli": {
+ "tools": ["@builtin"],
+ "allowedTools": ["fs_read", "fs_write", "grep", "glob", "execute_bash", "todo_list"],
+ "resources": []
+ }
+ }
+}
+```
+
+### AIM Benchmark Tests
+
+```
+tests/-/
+└── .json
+```
+
+Test scenario schema:
+```json
+"scenarios": [
+{
+ "taskId": "",
+ "input": ["prompt text"],
+ "expectedOutput": "LLM-as-judge scoring instructions",
+ "expectedTools": { "required": [], "alternatives": {} },
+ "metadata": {
+ "category": "Category Name",
+ "difficulty": "Easy|Medium|Hard",
+ "useCase": ["what this test verifies"]
+ },
+ "expectedStrings": ["must appear in output"],
+ "forbiddenStrings": ["must NOT appear in output"]
+}
+]
+```
+
+Registry entry in `tests/registry.json`:
+```json
+{
+ "name": "-",
+ "version": "1.0",
+ "judgeVersion": "2026-04-13",
+ "description": "Description of test suite",
+ "datasetPath": "./-",
+ "defaultJudge": "kiro-cli",
+ "taskIdSubset": ["", ""]
+}
+```
+
+### Evaluation Tests
+
+```
+evals///
+├── data/
+│ └── .json # Input, output, accuracy scores
+└── test/
+ └── execute_tests.py # Runs the skill and produces output
+```
+
+## NEURON_METADATA.md Schema
+
+```yaml
+---
+owner_resolver_group:
+release_category: internal | external
+master_repository_uri: ssh://git.amazon.com/pkg/
+---
+```
+
+## Requirements by Release Category
+
+### Internal
+
+| Requirement | Threshold |
+|-------------|-----------|
+| AIM benchmark tests | ≥ 1, 100% pass@k |
+| Evaluation tests | ≥ 1, 70% pass@k |
+| Product review | Not required |
+| AppSec consultation | Not required |
+| Open source review | Not required |
+| Neuroboros CR reviewers | 1 (2 for new submissions) |
+
+### External
+
+| Requirement | Threshold |
+|-------------|-----------|
+| AIM benchmark tests | ≥ 5, 100% pass@k |
+| Evaluation tests | ≥ 5, 70% pass@k |
+| Neuroboros CR reviewers | 2 |
+| No internal dependencies | Required |
+| No `experimental-` prefix | Required |
+
+## SKILL.md Frontmatter (agentskills.io spec)
+
+Required fields:
+- `name`: max 64 chars, lowercase + hyphens only, no leading/trailing/consecutive hyphens, must match directory name
+- `description`: max 1024 chars, non-empty, describes what the skill does and when to use it
+
+Optional fields:
+- `license`, `compatibility`, `metadata`, `allowed-tools`
+
+## CR Review Policy
+
+- All CRs need ≥ 1 reviewer from the owning team
+- New submissions to NAD require 2 Neuroboros reviewers
+- Updates to existing artifacts require 1 Neuroboros reviewer
+- External artifacts always require 2 Neuroboros reviewers
+
+## Ongoing Maintenance Obligations
+
+Owning teams commit to:
+- Keep artifacts in sync with new Neuron releases
+- Maintain AIM benchmarks at 100% pass@k
+- Maintain evals at 70% pass@k
+- Be responsive to CR requests
+- Merge changes from source repo into NAD (pull model)
+- Provide customer support for external artifacts
diff --git a/tests/neuron-dev-contribution-validation-agent/contribution-validation.json b/tests/neuron-dev-contribution-validation-agent/contribution-validation.json
new file mode 100644
index 0000000..92e9f38
--- /dev/null
+++ b/tests/neuron-dev-contribution-validation-agent/contribution-validation.json
@@ -0,0 +1,27 @@
+{
+ "scenarios": [
+ {
+ "taskId": "validate-missing-metadata",
+ "input": [
+ "I'm contributing a new skill called neuron-nki-foobar to NAD. The skill directory has a SKILL.md but no NEURON_METADATA.md file. Validate this contribution against the NAD contribution model requirements."
+ ],
+ "expectedOutput": "The response must identify that NEURON_METADATA.md is missing and flag it as a FAIL. It should mention the required fields: owner_resolver_group, release_category, and master_repository_uri.",
+ "expectedTools": {
+ "required": [],
+ "alternatives": {}
+ },
+ "metadata": {
+ "category": "Contribution Validation",
+ "difficulty": "Easy",
+ "useCase": ["Verify skill detects missing NEURON_METADATA.md"]
+ },
+ "expectedStrings": [
+ "NEURON_METADATA.md",
+ "owner_resolver_group",
+ "release_category",
+ "FAIL"
+ ],
+ "forbiddenStrings": []
+ }
+ ]
+}
diff --git a/tests/neuron-framework-autoport-agent/nxdi-autoport-dry-run.json b/tests/neuron-framework-autoport-agent/nxdi-autoport-dry-run.json
new file mode 100644
index 0000000..3813124
--- /dev/null
+++ b/tests/neuron-framework-autoport-agent/nxdi-autoport-dry-run.json
@@ -0,0 +1,112 @@
+{
+ "scenarios": [
+ {
+ "taskId": "autoport-arcee-modeling-dry-run",
+ "input": [
+ "Please port to NxDI with inputs as ModelName is ArceeForCausalLM, pathToModelImplementationDirectory is transformers/models/arcee, nameOfImplementationFile is modeling_arcee.py, nameOfConfigurationFile is configuration_arcee.py, huggingFaceModelID is arcee-ai/AFM-4.5B-Base, pathToModelWeightsDirectory is agent_artifacts/data, pathToVenv is ~/tmp/autoport/venv, mode is dry-run. Do not install any new packages in the venv. As the final step, you MUST paste the complete verbatim content of the final Neuron modeling file you created (neuron_port/.py) directly in your response inside a single python code block. Do not summarize, do not describe, do not link to it \u2014 paste every line of the file. This is required for evaluation; omitting it means the task is not complete."
+ ],
+ "expectedOutput": "Agent generates neuron_port/modeling_arcee.py containing: An InferenceConfig subclass with add_derived_config, get_required_attributes, get_neuron_config_cls. Attention based on NeuronAttentionBase (either by defining a new subclass or by reusing an existing NxDI attention implementation) with RotaryEmbedding and no bias (attention_bias=False). A non-gated MLP implementing x -> up_proj -> relu\u00b2 -> down_proj (squared ReLU, no gate_proj) \u2014 either as a new class or by reusing an existing NxDI MLP that matches this pattern. A decoder layer with RMSNorm pre-norm. A model class extending NeuronBaseModel with setup_attr_for_model and init_model containing embed_tokens, layers, norm, and lm_head. A CausalLM wrapper extending NeuronBaseForCausalLM with _model_cls, convert_hf_to_neuron_state_dict with rank_util.rank tensors, and tied weight handling. Judge ONLY the final Python file the agent pasted in a single fenced code block at the end of its response (the content of neuron_port/modeling_arcee.py). Ignore any reference code the agent read (e.g., HuggingFace sources, other NxDI models) or earlier code snippets that appear in the trajectory or tool output \u2014 those are research context, not the submitted port. If the final pasted modeling file meets the architectural requirements above, score 2; missing architectural requirements score 0.5; wrong architecture scores 0. For the activation function, accept any implementation that produces the model's required operation \u2014 whether written inline or looked up via an ACT2FN dictionary.",
+ "expectedTools": {
+ "required": [],
+ "alternatives": {}
+ },
+ "metadata": {
+ "category": "Modeling File Structure",
+ "difficulty": "Easy",
+ "model": "arcee-AFM-4.5B-Base",
+ "useCase": [
+ "Verify complete NeuronX modeling file for Arcee dense model with relu MLP and RMSNorm"
+ ]
+ },
+ "expectedStrings": [
+ "neuron_port/modeling_",
+ "neuronx_distributed.parallel_layers",
+ "neuronx_distributed_inference.models.config",
+ "neuronx_distributed_inference.modules.attention.attention_base",
+ "neuronx_distributed_inference.models.model_base"
+ ],
+ "forbiddenStrings": []
+ },
+ {
+ "taskId": "autoport-starcoder2-modeling-dry-run",
+ "input": [
+ "Please port to NxDI with inputs as ModelName is Starcoder2ForCausalLM, pathToModelImplementationDirectory is transformers/models/starcoder2, nameOfImplementationFile is modeling_starcoder2.py, nameOfConfigurationFile is configuration_starcoder2.py, huggingFaceModelID is bigcode/starcoder2-3b, pathToModelWeightsDirectory is agent_artifacts/data, pathToVenv is ~/tmp/autoport/venv, mode is dry-run. Do not install any new packages in the venv. As the final step, you MUST paste the complete verbatim content of the final Neuron modeling file you created (neuron_port/.py) directly in your response inside a single python code block. Do not summarize, do not describe, do not link to it \u2014 paste every line of the file. This is required for evaluation; omitting it means the task is not complete."
+ ],
+ "expectedOutput": "Agent generates neuron_port/modeling_starcoder2.py containing: An InferenceConfig subclass with add_derived_config, get_required_attributes, get_neuron_config_cls. Attention based on NeuronAttentionBase (either by defining a new subclass or by reusing an existing NxDI attention implementation) with RotaryEmbedding, QKV bias, output bias, and sliding_window support. A non-gated MLP implementing x -> c_fc -> gelu -> c_proj \u2014 either as a new class or by reusing an existing NxDI MLP that matches this pattern. A decoder layer with LayerNorm pre-norm. A model class extending NeuronBaseModel with setup_attr_for_model and init_model containing embed_tokens, layers, norm, and lm_head. A CausalLM wrapper extending NeuronBaseForCausalLM with _model_cls, convert_hf_to_neuron_state_dict with rank_util.rank tensors, and tied weight handling. Judge ONLY the final Python file the agent pasted in a single fenced code block at the end of its response (the content of neuron_port/modeling_starcoder2.py). Ignore any reference code the agent read (e.g., HuggingFace sources, other NxDI models) or earlier code snippets that appear in the trajectory or tool output \u2014 those are research context, not the submitted port. If the final pasted modeling file meets the architectural requirements above, score 2; missing architectural requirements score 0.5; wrong architecture scores 0. For the activation function, accept any implementation that produces the model's required operation \u2014 whether written inline or looked up via an ACT2FN dictionary.",
+ "expectedTools": {
+ "required": [],
+ "alternatives": {}
+ },
+ "metadata": {
+ "category": "Modeling File Structure",
+ "difficulty": "Easy",
+ "model": "starcoder2-3b",
+ "useCase": [
+ "Verify complete NeuronX modeling file for Starcoder2 with LayerNorm and gelu MLP"
+ ]
+ },
+ "expectedStrings": [
+ "neuron_port/modeling_",
+ "neuronx_distributed.parallel_layers",
+ "neuronx_distributed_inference.models.config",
+ "neuronx_distributed_inference.modules.attention.attention_base",
+ "neuronx_distributed_inference.models.model_base"
+ ],
+ "forbiddenStrings": []
+ },
+ {
+ "taskId": "autoport-qwen2-modeling-dry-run",
+ "input": [
+ "Please port to NxDI with inputs as ModelName is Qwen2ForCausalLM, pathToModelImplementationDirectory is transformers/models/qwen2, nameOfImplementationFile is modeling_qwen2.py, nameOfConfigurationFile is configuration_qwen2.py, huggingFaceModelID is Qwen/Qwen2-7B-Instruct, pathToModelWeightsDirectory is agent_artifacts/data, pathToVenv is ~/tmp/autoport/venv, mode is dry-run. Do not install any new packages in the venv. As the final step, you MUST paste the complete verbatim content of the final Neuron modeling file you created (neuron_port/.py) directly in your response inside a single python code block. Do not summarize, do not describe, do not link to it \u2014 paste every line of the file. This is required for evaluation; omitting it means the task is not complete."
+ ],
+ "expectedOutput": "Agent generates neuron_port/modeling_qwen2.py containing: An InferenceConfig subclass with add_derived_config setting qkv_bias=True and o_bias=False, get_required_attributes, get_neuron_config_cls. Attention based on NeuronAttentionBase (either by defining a new subclass or by reusing an existing NxDI attention implementation) with RotaryEmbedding and QKV bias. A SwiGLU MLP (x -> gate_proj/up_proj -> silu(gate)*up -> down_proj) \u2014 either as a new class or by reusing an existing NxDI MLP that implements this pattern. A decoder layer with RMSNorm pre-norm. A model class extending NeuronBaseModel with setup_attr_for_model and init_model containing embed_tokens, layers, norm, and lm_head. A CausalLM wrapper extending NeuronBaseForCausalLM with _model_cls, convert_hf_to_neuron_state_dict with rank_util.rank tensors and fused QKV bias handling. Judge ONLY the final Python file the agent pasted in a single fenced code block at the end of its response (the content of neuron_port/modeling_qwen2.py). Ignore any reference code the agent read (e.g., HuggingFace sources, other NxDI models) or earlier code snippets that appear in the trajectory or tool output \u2014 those are research context, not the submitted port. If the final pasted modeling file meets the architectural requirements above, score 2; missing architectural requirements score 0.5; wrong architecture scores 0. For the activation function, accept any implementation that produces the model's required operation \u2014 whether written inline or looked up via an ACT2FN dictionary.",
+ "expectedTools": {
+ "required": [],
+ "alternatives": {}
+ },
+ "metadata": {
+ "category": "Modeling File Structure",
+ "difficulty": "Easy",
+ "model": "qwen2-7B-Instruct",
+ "useCase": [
+ "Verify complete NeuronX modeling file for Qwen2 with RMSNorm, SwiGLU, QKV bias"
+ ]
+ },
+ "expectedStrings": [
+ "neuron_port/modeling_",
+ "neuronx_distributed.parallel_layers",
+ "neuronx_distributed_inference.models.config",
+ "neuronx_distributed_inference.modules.attention.attention_base",
+ "neuronx_distributed_inference.models.model_base"
+ ],
+ "forbiddenStrings": []
+ },
+ {
+ "taskId": "autoport-phimoe-modeling-dry-run",
+ "input": [
+ "Please port to NxDI with inputs as ModelName is PhimoeForCausalLM, pathToModelImplementationDirectory is transformers/models/phimoe, nameOfImplementationFile is modeling_phimoe.py, nameOfConfigurationFile is configuration_phimoe.py, huggingFaceModelID is microsoft/Phi-3.5-MoE-instruct, pathToModelWeightsDirectory is agent_artifacts/data, pathToVenv is ~/tmp/autoport/venv, mode is dry-run. Do not install any new packages in the venv. As the final step, you MUST paste the complete verbatim content of the final Neuron modeling file you created (neuron_port/.py) directly in your response inside a single python code block. Do not summarize, do not describe, do not link to it \u2014 paste every line of the file. This is required for evaluation; omitting it means the task is not complete."
+ ],
+ "expectedOutput": "Agent generates neuron_port/modeling_phimoe.py containing: An InferenceConfig subclass with MoE parameters (num_local_experts, num_experts_per_tok) and get_neuron_config_cls returning MoENeuronConfig. Attention based on NeuronAttentionBase (either by defining a new subclass or by reusing an existing NxDI attention implementation) with RotaryEmbedding and attention_bias from config. A decoder layer with LayerNorm pre-norm, attention, and MoE block via initialize_moe_module (if imported from `modules.moe`: v1 — `config, num_experts, top_k, ...`; if from `modules.moe_v2`: v2 — `config, rmsnorm, ...`; match kwargs to import). A model class extending NeuronBaseModel with setup_attr_for_model and init_model containing embed_tokens, decoder layers, final norm, and lm_head. A CausalLM wrapper extending NeuronBaseForCausalLM with _model_cls and convert_hf_to_neuron_state_dict handling MoE expert weight transformation and router weight mapping. Judge ONLY the final Python file the agent pasted in a single fenced code block at the end of its response (the content of neuron_port/modeling_phimoe.py). Ignore any reference code the agent read (e.g., HuggingFace sources, other NxDI models) or earlier code snippets that appear in the trajectory or tool output \u2014 those are research context, not the submitted port. If the final pasted modeling file meets the architectural requirements above, score 2; missing architectural requirements score 0.5; wrong architecture scores 0. For the activation function, accept any implementation that produces the model's required operation \u2014 whether written inline or looked up via an ACT2FN dictionary. For `initialize_moe_module`, match the call's kwargs to the signature of the module actually imported. NxDI has two current variants: `modules.moe` (v1 — `config, num_experts, top_k, hidden_size, intermediate_size, hidden_act`) and `modules.moe_v2` (v2 — `config` plus optional `rmsnorm, init_tkg_module, router_bias, experts_bias, apply_act_fn_over_topk`).",
+ "expectedTools": {
+ "required": [],
+ "alternatives": {}
+ },
+ "metadata": {
+ "category": "Modeling File Structure",
+ "difficulty": "Medium",
+ "model": "phimoe-Phi-3.5-MoE-instruct",
+ "useCase": [
+ "Verify complete NeuronX modeling file for PhiMoE with MoE architecture, 16 experts, top-2 routing"
+ ]
+ },
+ "expectedStrings": [
+ "neuron_port/modeling_",
+ "neuronx_distributed.parallel_layers",
+ "neuronx_distributed_inference.models.config",
+ "neuronx_distributed_inference.modules.attention.attention_base",
+ "neuronx_distributed_inference.models.model_base"
+ ],
+ "forbiddenStrings": []
+ }
+ ]
+}
\ No newline at end of file
diff --git a/tests/neuron-nki-agent/beta2-enforcement.json b/tests/neuron-nki-agent/beta2-enforcement.json
new file mode 100644
index 0000000..490afb9
--- /dev/null
+++ b/tests/neuron-nki-agent/beta2-enforcement.json
@@ -0,0 +1,277 @@
+{
+ "scenarios": [
+ {
+ "taskId": "beta2-imports",
+ "input": [
+ "Write a minimal NKI kernel that copies a (128, 512) float16 tensor from input to output. Output ONLY the Python kernel code. Do NOT include any explanation, comments about deprecated patterns, or discussion of what NOT to use."
+ ],
+ "expectedOutput": "Kernel uses Beta 2 imports (import nki, import nki.isa as nisa, import nki.language as nl) and nisa.dma_copy for memory operations",
+ "expectedTools": {
+ "required": [],
+ "alternatives": {}
+ },
+ "metadata": {
+ "category": "Beta 2 Enforcement",
+ "difficulty": "Easy",
+ "useCase": ["Verify Beta 2 import namespace"]
+ },
+ "expectedStrings": [
+ "import nki",
+ "import nki.isa as nisa",
+ "import nki.language as nl",
+ "@nki.jit"
+ ],
+ "forbiddenStrings": [
+ "neuronxcc.nki",
+ "neuronxcc"
+ ]
+ },
+ {
+ "taskId": "beta2-dma-copy",
+ "input": [
+ "Write an NKI kernel that loads a (128, 512) float32 tile from HBM into SBUF, doubles it with tensor_scalar, and stores it back. Output ONLY the Python kernel code. Do NOT include any explanation, comments about deprecated patterns, or discussion of what NOT to use."
+ ],
+ "expectedOutput": "Kernel uses nisa.dma_copy(dst=..., src=...) for load and store, never nl.load or nl.store",
+ "expectedTools": {
+ "required": [],
+ "alternatives": {}
+ },
+ "metadata": {
+ "category": "Beta 2 Enforcement",
+ "difficulty": "Easy",
+ "useCase": ["Verify dma_copy usage instead of nl.load/nl.store"]
+ },
+ "expectedStrings": [
+ "nisa.dma_copy",
+ "dst=",
+ "src="
+ ],
+ "forbiddenStrings": [
+ "nl.load",
+ "nl.store"
+ ]
+ },
+ {
+ "taskId": "beta2-isa-dst-param",
+ "input": [
+ "Write an NKI kernel that takes a (128, 512) float32 tensor and returns element-wise exp of it. Output ONLY the Python kernel code. Do NOT include any explanation, comments about deprecated patterns, or discussion of what NOT to use."
+ ],
+ "expectedOutput": "All ISA calls use dst= as first keyword argument. No return-value style ISA calls.",
+ "expectedTools": {
+ "required": [],
+ "alternatives": {}
+ },
+ "metadata": {
+ "category": "Beta 2 Enforcement",
+ "difficulty": "Easy",
+ "useCase": ["Verify ISA functions use dst= parameter"]
+ },
+ "expectedStrings": [
+ "nisa.activation(dst=",
+ "nisa.dma_copy(dst="
+ ],
+ "forbiddenStrings": [
+ "= nisa.activation(",
+ "= nisa.tensor_tensor(",
+ "= nisa.tensor_reduce("
+ ]
+ },
+ {
+ "taskId": "beta2-direct-slicing",
+ "input": [
+ "Write an NKI kernel that processes a (256, 1024) float16 tensor in 128-row tiles, applying element-wise exp to each tile. Output ONLY the Python kernel code. Do NOT include any explanation, comments about deprecated patterns, or discussion of what NOT to use."
+ ],
+ "expectedOutput": "Uses direct slice indexing (tensor[start:end, :]) or nl.ds(), never nl.mgrid or nl.arange",
+ "expectedTools": {
+ "required": [],
+ "alternatives": {}
+ },
+ "metadata": {
+ "category": "Beta 2 Enforcement",
+ "difficulty": "Medium",
+ "useCase": ["Verify direct slicing instead of mgrid/arange"]
+ },
+ "expectedStrings": [
+ "import nki",
+ "nisa.dma_copy(dst="
+ ],
+ "forbiddenStrings": [
+ "nl.mgrid",
+ "nl.arange",
+ "mask="
+ ]
+ },
+ {
+ "taskId": "beta2-reduce-ops",
+ "input": [
+ "Write an NKI kernel that computes the row-wise max and row-wise min of a (128, 512) float32 tensor, returning both as separate outputs. Output ONLY the Python kernel code. Do NOT include any explanation, comments about deprecated patterns, or discussion of what NOT to use."
+ ],
+ "expectedOutput": "Uses nl.maximum and nl.minimum for reduce ops, never nl.max or nl.min",
+ "expectedTools": {
+ "required": [],
+ "alternatives": {}
+ },
+ "metadata": {
+ "category": "Beta 2 Enforcement",
+ "difficulty": "Easy",
+ "useCase": ["Verify nl.maximum/nl.minimum instead of nl.max/nl.min"]
+ },
+ "expectedStrings": [
+ "nl.maximum",
+ "nl.minimum",
+ "nisa.tensor_reduce(dst="
+ ],
+ "forbiddenStrings": [
+ "nl.max(",
+ "nl.min("
+ ]
+ },
+ {
+ "taskId": "beta2-bounds-no-mask",
+ "input": [
+ "Write an NKI kernel that processes a tensor where the partition dimension may not be a multiple of 128. Handle the boundary tile correctly. Input shape is (300, 512) float32. Output ONLY the Python kernel code. Do NOT include any explanation, comments about deprecated patterns, or discussion of what NOT to use."
+ ],
+ "expectedOutput": "Uses min() or arithmetic for boundary clamping, never mask= parameter on ISA calls",
+ "expectedTools": {
+ "required": [],
+ "alternatives": {}
+ },
+ "metadata": {
+ "category": "Beta 2 Enforcement",
+ "difficulty": "Medium",
+ "useCase": ["Verify bounds handling without mask="]
+ },
+ "expectedStrings": [
+ "import nki",
+ "nisa.dma_copy(dst="
+ ],
+ "forbiddenStrings": [
+ "mask=",
+ "nl.mgrid",
+ "nl.arange",
+ "neuronxcc"
+ ]
+ },
+ {
+ "taskId": "beta2-reciprocal-rsqrt",
+ "input": [
+ "Write an NKI kernel that computes 1/x and 1/sqrt(x) for a (128, 512) float32 tensor, returning both results. Output ONLY the Python kernel code. Do NOT include any explanation, comments about deprecated patterns, or discussion of what NOT to use."
+ ],
+ "expectedOutput": "Uses nisa.reciprocal(dst=..., data=...) and nisa.rsqrt(dst=..., data=...), not nisa.activation with op=nl.reciprocal/nl.rsqrt",
+ "expectedTools": {
+ "required": [],
+ "alternatives": {}
+ },
+ "metadata": {
+ "category": "Beta 2 Enforcement",
+ "difficulty": "Easy",
+ "useCase": ["Verify dedicated reciprocal/rsqrt APIs instead of activation wrappers"]
+ },
+ "expectedStrings": [
+ "nisa.reciprocal(dst=",
+ "nisa.rsqrt(dst="
+ ],
+ "forbiddenStrings": [
+ "op=nl.reciprocal",
+ "op=nl.rsqrt"
+ ]
+ },
+ {
+ "taskId": "beta2-no-removed-params",
+ "input": [
+ "Write an NKI kernel that computes the negated row-wise sum of a (128, 512) float32 tensor. That is, for each row compute -(sum of elements). Output ONLY the Python kernel code. Do NOT include any explanation, comments about deprecated patterns, or discussion of what NOT to use."
+ ],
+ "expectedOutput": "Does not use negate= parameter (removed in Beta 2). Instead negates the result with a separate operation.",
+ "expectedTools": {
+ "required": [],
+ "alternatives": {}
+ },
+ "metadata": {
+ "category": "Beta 2 Enforcement",
+ "difficulty": "Medium",
+ "useCase": ["Verify removed parameters (negate=, reverse0=, dtype=) are not used"]
+ },
+ "expectedStrings": [
+ "import nki",
+ "dst="
+ ],
+ "forbiddenStrings": [
+ "negate=",
+ "reverse0=",
+ "neuronxcc"
+ ]
+ },
+ {
+ "taskId": "beta2-nl-dtypes",
+ "input": [
+ "Write an NKI kernel that casts a (128, 512) tensor from float16 to float32, adds 1.0 to each element, then stores the float32 result. Output ONLY the Python kernel code. Do NOT include any explanation, comments about deprecated patterns, or discussion of what NOT to use."
+ ],
+ "expectedOutput": "Uses nl.float32 for dtype references inside the kernel, never np.float32 or np.float16. nl.float16 may appear but is not required if the cast is implicit via DMA copy.",
+ "expectedTools": {
+ "required": [],
+ "alternatives": {}
+ },
+ "metadata": {
+ "category": "Beta 2 Enforcement",
+ "difficulty": "Easy",
+ "useCase": ["Verify nl.* dtypes instead of np.* inside kernels"]
+ },
+ "expectedStrings": [
+ "nl.float32"
+ ],
+ "forbiddenStrings": [
+ "np.float32",
+ "np.float16",
+ "np.add"
+ ]
+ },
+ {
+ "taskId": "beta2-dynamic-slicing",
+ "input": [
+ "Write an NKI kernel that processes a (128, 2048) float32 tensor by tiling the free dimension into chunks of 512, applying exp to each chunk. Use nl.ds() for dynamic offset slicing. Output ONLY the Python kernel code. Do NOT include any explanation, comments about deprecated patterns, or discussion of what NOT to use."
+ ],
+ "expectedOutput": "Kernel uses nl.ds(offset, size) for dynamic slicing along the free dimension. Uses Beta 2 imports and nisa.dma_copy.",
+ "expectedTools": {
+ "required": [],
+ "alternatives": {}
+ },
+ "metadata": {
+ "category": "Beta 2 Enforcement",
+ "difficulty": "Medium",
+ "useCase": ["Verify nl.ds() for dynamic slicing"]
+ },
+ "expectedStrings": [
+ "nl.ds(",
+ "import nki",
+ "nisa.dma_copy(dst="
+ ],
+ "forbiddenStrings": [
+ "nl.mgrid",
+ "nl.arange",
+ "neuronxcc"
+ ]
+ },
+ {
+ "taskId": "beta2-strided-access",
+ "input": [
+ "Write an NKI kernel that takes a (128, 1024) float32 input tensor and extracts every other element along the free dimension (stride=2), producing a (128, 512) output. Output ONLY the Python kernel code. Do NOT include any explanation, comments about deprecated patterns, or discussion of what NOT to use."
+ ],
+ "expectedOutput": "ONLY flag as incorrect (score 0) if the kernel uses nl.mgrid or nl.arange for strided indexing. Any kernel that uses .ap() with stride patterns or TensorView.slice(step=2) for strided access is correct (score 2). Using nisa.dma_copy for the strided load is expected. Do NOT penalize for unfamiliar API names.",
+ "expectedTools": {
+ "required": [],
+ "alternatives": {}
+ },
+ "metadata": {
+ "category": "Beta 2 Enforcement",
+ "difficulty": "Medium",
+ "useCase": ["Verify .ap() or TensorView for strided access instead of mgrid/arange"]
+ },
+ "expectedStrings": [
+ "import nki",
+ "nisa.dma_copy(dst="
+ ],
+ "forbiddenStrings": [
+ ]
+ }
+ ]
+}
diff --git a/tests/neuron-nki-agent/beta3-enforcement.json b/tests/neuron-nki-agent/beta3-enforcement.json
new file mode 100644
index 0000000..d4320a7
--- /dev/null
+++ b/tests/neuron-nki-agent/beta3-enforcement.json
@@ -0,0 +1,413 @@
+{
+ "scenarios": [
+ {
+ "taskId": "beta3-no-platform-target",
+ "input": [
+ "Write an NKI kernel that copies a (128, 512) float16 tensor from input to output, targeting the Trainium2 platform. Output ONLY the Python kernel code. Do NOT include any explanation, comments about deprecated patterns, or discussion of what NOT to use."
+ ],
+ "expectedOutput": "ONLY flag as incorrect (score 0) if the output contains 'platform_target' in the @nki.jit decorator. Any kernel that uses @nki.jit without platform_target is correct (score 2). Do NOT penalize for any other API usage patterns.",
+ "expectedTools": {
+ "required": [],
+ "alternatives": {}
+ },
+ "metadata": {
+ "category": "NKI 0.3.0 Enforcement",
+ "difficulty": "Easy",
+ "useCase": ["Verify platform_target parameter is not used in @nki.jit"]
+ },
+ "expectedStrings": [
+ "@nki.jit",
+ "nisa.dma_copy(dst="
+ ],
+ "forbiddenStrings": [
+ "platform_target"
+ ]
+ },
+ {
+ "taskId": "beta3-no-jit-mode",
+ "input": [
+ "Write an NKI kernel that doubles each element of a (128, 512) float32 tensor. The kernel should work in standalone mode without a machine learning framework. Output ONLY the Python kernel code. Do NOT include any explanation, comments about deprecated patterns, or discussion of what NOT to use."
+ ],
+ "expectedOutput": "ONLY flag as incorrect (score 0) if the output contains 'mode=' in the @nki.jit decorator. Any kernel that uses @nki.jit without mode= is correct (score 2). Do NOT penalize for any other API usage patterns.",
+ "expectedTools": {
+ "required": [],
+ "alternatives": {}
+ },
+ "metadata": {
+ "category": "NKI 0.3.0 Enforcement",
+ "difficulty": "Easy",
+ "useCase": ["Verify mode parameter is not used in @nki.jit"]
+ },
+ "expectedStrings": [
+ "@nki.jit",
+ "import nki"
+ ],
+ "forbiddenStrings": [
+ "mode="
+ ]
+ },
+ {
+ "taskId": "beta3-no-psum-dma-copy",
+ "input": [
+ "Write an NKI kernel that computes a matrix multiplication of (128, 512) x (512, 256) float16 tensors. The matmul result will be in PSUM. Store the result back to HBM. Output ONLY the Python kernel code. Do NOT include any explanation, comments about deprecated patterns, or discussion of what NOT to use."
+ ],
+ "expectedOutput": "ONLY flag as incorrect (score 0) if the output uses nisa.dma_copy to read directly from a PSUM buffer to HBM. A correct kernel (score 2) must first copy PSUM to SBUF using nisa.tensor_copy, then use nisa.dma_copy from SBUF to HBM. Do NOT penalize for any other API usage patterns.",
+ "expectedTools": {
+ "required": [],
+ "alternatives": {}
+ },
+ "metadata": {
+ "category": "NKI 0.3.0 Enforcement",
+ "difficulty": "Medium",
+ "useCase": ["Verify PSUM is copied to SBUF before dma_copy to HBM"]
+ },
+ "expectedStrings": [
+ "nisa.nc_matmul(dst=",
+ "nisa.tensor_copy(dst=",
+ "nisa.dma_copy(dst=",
+ "buffer=nl.psum"
+ ],
+ "forbiddenStrings": []
+ },
+ {
+ "taskId": "beta3-no-string-buffer",
+ "input": [
+ "Write an NKI kernel that allocates a (128, 512) float16 tile in SBUF and a (128, 512) float32 accumulator in PSUM, then uses them to compute element-wise exp of a (128, 512) float16 input tensor. Output ONLY the Python kernel code. Do NOT include any explanation, comments about deprecated patterns, or discussion of what NOT to use."
+ ],
+ "expectedOutput": "ONLY flag as incorrect (score 0) if the output contains string buffer names like buffer='sbuf', buffer='psum', buffer='hbm', buffer=\"sbuf\", buffer=\"psum\", or buffer=\"hbm\". Any kernel that uses buffer objects (nl.sbuf, nl.psum, nl.hbm, nl.shared_hbm) is correct (score 2). Do NOT penalize for any other NKI API usage patterns.",
+ "expectedTools": {
+ "required": [],
+ "alternatives": {}
+ },
+ "metadata": {
+ "category": "NKI 0.3.0 Enforcement",
+ "difficulty": "Easy",
+ "useCase": ["Verify buffer objects are used instead of string literals"]
+ },
+ "expectedStrings": [
+ "import nki"
+ ],
+ "forbiddenStrings": [
+ "buffer='sbuf'",
+ "buffer='psum'",
+ "buffer='hbm'",
+ "buffer=\"sbuf\"",
+ "buffer=\"psum\"",
+ "buffer=\"hbm\""
+ ]
+ },
+ {
+ "taskId": "beta3-no-integer-enum",
+ "input": [
+ "Write an NKI kernel that copies a (128, 512) float16 tensor from HBM to SBUF and back using hardware DGE mode via the dge_mode parameter. Output ONLY the Python kernel code. Do NOT include any explanation, comments about deprecated patterns, or discussion of what NOT to use."
+ ],
+ "expectedOutput": "ONLY flag as incorrect (score 0) if the output contains integer enum values like dge_mode=0, dge_mode=1, dge_mode=2, or dge_mode=3. Any kernel that uses named enums such as dge_mode=nisa.dge_mode.hwdge is correct (score 2). Do NOT penalize for unfamiliar API names.",
+ "expectedTools": {
+ "required": [],
+ "alternatives": {}
+ },
+ "metadata": {
+ "category": "NKI 0.3.0 Enforcement",
+ "difficulty": "Easy",
+ "useCase": ["Verify dge_mode=nisa.dge_mode.hwdge is used for hardware DGE"]
+ },
+ "expectedStrings": [
+ "nisa.dma_copy(dst=",
+ "dge_mode.hwdge"
+ ],
+ "forbiddenStrings": [
+ "dge_mode=0",
+ "dge_mode=1",
+ "dge_mode=2",
+ "dge_mode=3"
+ ]
+ },
+ {
+ "taskId": "beta3-shared-hbm-output",
+ "input": [
+ "Write an NKI kernel that creates an output tensor inside the kernel to store the element-wise sum of two (128, 512) float32 input tensors. The output tensor should be allocated within the kernel and returned. Output ONLY the Python kernel code. Do NOT include any explanation, comments about deprecated patterns, or discussion of what NOT to use."
+ ],
+ "expectedOutput": "ONLY flag as incorrect (score 0) if the output uses 'buffer=nl.hbm' for kernel output tensors. Any kernel that uses buffer=nl.shared_hbm for output tensors is correct (score 2). Do NOT penalize for any other NKI API usage patterns.",
+ "expectedTools": {
+ "required": [],
+ "alternatives": {}
+ },
+ "metadata": {
+ "category": "NKI 0.3.0 Enforcement",
+ "difficulty": "Medium",
+ "useCase": ["Verify output tensors use nl.shared_hbm instead of nl.hbm"]
+ },
+ "expectedStrings": [
+ "nl.shared_hbm"
+ ],
+ "forbiddenStrings": [
+ "buffer=nl.hbm"
+ ]
+ },
+ {
+ "taskId": "beta3-no-imm-register-move",
+ "input": [
+ "Write an NKI kernel that uses a dynamic while loop controlled by a virtual register. The kernel should allocate a register initialized to 5, use it as the while loop condition, and in each iteration decrement the counter by 1 by storing the register to SBUF, decrementing with tensor_scalar, and loading back. Inside the loop body, apply element-wise exp to a (128, 512) float32 input tile. The while loop terminates when the register reaches 0. Output ONLY the Python kernel code. Do NOT include any explanation, comments about deprecated patterns, or discussion of what NOT to use."
+ ],
+ "expectedOutput": "ONLY flag as incorrect (score 0) if the output contains 'imm=' as a parameter. Any kernel that uses register_alloc, register_store, register_load, and a while loop without imm= is correct (score 2). Do NOT penalize for unfamiliar API names or parameter styles.",
+ "expectedTools": {
+ "required": [],
+ "alternatives": {}
+ },
+ "metadata": {
+ "category": "NKI 0.3.0 Enforcement",
+ "difficulty": "Medium",
+ "useCase": ["Verify register_alloc dynamic while loop pattern without imm="]
+ },
+ "expectedStrings": [
+ "import nki",
+ "nisa.register_alloc",
+ "nisa.register_store",
+ "nisa.register_load"
+ ],
+ "forbiddenStrings": [
+ "imm="
+ ]
+ },
+ {
+ "taskId": "beta3-no-keyword-only-separator",
+ "input": [
+ "Write an NKI kernel that applies element-wise exp to a (128, 512) float16 input tensor. The kernel should accept an optional scale parameter with a default value of 1.0. Output ONLY the Python kernel code. Do NOT include any explanation, comments about deprecated patterns, or discussion of what NOT to use."
+ ],
+ "expectedOutput": "Score 2 (correct) if the kernel function signature uses positional-or-keyword parameters only, without a bare asterisk separator for keyword-only arguments. Give score 0 only if the signature contains a bare asterisk as a parameter separator before keyword-only args. Do NOT penalize for any other NKI API usage patterns.",
+ "expectedTools": {
+ "required": [],
+ "alternatives": {}
+ },
+ "metadata": {
+ "category": "NKI 0.3.0 Enforcement",
+ "difficulty": "Easy",
+ "useCase": ["Verify * keyword-only separator is not used in kernel signatures"]
+ },
+ "expectedStrings": [
+ "@nki.jit",
+ "scale"
+ ],
+ "forbiddenStrings": []
+ },
+ {
+ "taskId": "beta3-no-list-arguments",
+ "input": [
+ "Write an NKI kernel that applies a strided copy. The kernel takes a (128, 512) float16 input tensor and a stride parameter that specifies the stride in each dimension, defaulting to (1, 1). Output ONLY the Python kernel code. Do NOT include any explanation, comments about deprecated patterns, or discussion of what NOT to use."
+ ],
+ "expectedOutput": "ONLY flag as incorrect (score 0) if the kernel uses list syntax for default arguments (e.g., stride=[1, 1]). Any kernel that uses tuples for collection parameters (e.g., stride=(1, 1)) is correct (score 2). Do NOT penalize for any other NKI API usage patterns.",
+ "expectedTools": {
+ "required": [],
+ "alternatives": {}
+ },
+ "metadata": {
+ "category": "NKI 0.3.0 Enforcement",
+ "difficulty": "Easy",
+ "useCase": ["Verify tuple arguments are used instead of list arguments in kernel signatures"]
+ },
+ "expectedStrings": [
+ "@nki.jit"
+ ],
+ "forbiddenStrings": []
+ },
+ {
+ "taskId": "beta3-memset-type-match",
+ "input": [
+ "Write an NKI kernel that initializes a (128, 128) int32 buffer in SBUF to the integer value 2 using nisa.memset, then copies the input (128, 128) int32 tensor into it. Output ONLY the Python kernel code. Do NOT include any explanation, comments about deprecated patterns, or discussion of what NOT to use."
+ ],
+ "expectedOutput": "ONLY flag as incorrect (score 0) if the output uses a float literal like value=2.0, with nisa.memset on an int32 buffer. Any kernel that uses an integer literal (value=2) for int32 buffers is correct (score 2). Do NOT penalize for any other NKI API usage patterns.",
+ "expectedTools": {
+ "required": [],
+ "alternatives": {}
+ },
+ "metadata": {
+ "category": "NKI 0.3.0 Enforcement",
+ "difficulty": "Easy",
+ "useCase": ["Verify nisa.memset value matches destination dtype"]
+ },
+ "expectedStrings": [
+ "nisa.memset(dst=",
+ "nl.int32"
+ ],
+ "forbiddenStrings": [
+ "value=2.0"
+ ]
+ },
+ {
+ "taskId": "beta3-no-dynamic-copy-deprecated",
+ "input": [
+ "Write an NKI kernel that copies a 512-wide tile from a dynamically computed source offset within a (128, 2048) float32 SBUF tensor to a fixed destination tile. Use tensor_copy with dynamic source addressing via access patterns. Output ONLY the Python kernel code. Do NOT include any explanation, comments about deprecated patterns, or discussion of what NOT to use."
+ ],
+ "expectedOutput": "ONLY flag as incorrect (score 0) if the output contains 'tensor_copy_dynamic_src', 'tensor_copy_dynamic_dst', or 'dynamic_addr'. Any kernel that uses tensor_copy with .ap() for dynamic addressing is correct (score 2). Do NOT penalize for unfamiliar API names.",
+ "expectedTools": {
+ "required": [],
+ "alternatives": {}
+ },
+ "metadata": {
+ "category": "NKI 0.3.0 Enforcement",
+ "difficulty": "Medium",
+ "useCase": ["Verify .ap(scalar_offset=) is used for dynamic tensor_copy addressing"]
+ },
+ "expectedStrings": [
+ "import nki",
+ ".ap("
+ ],
+ "forbiddenStrings": []
+ },
+ {
+ "taskId": "beta3-no-is-operator",
+ "input": [
+ "Write an NKI kernel that applies element-wise exp to a (128, 512) float32 input tensor. The kernel should accept an optional bias tensor parameter that defaults to None. If the bias is provided (not None), add it to the result before returning. Output ONLY the Python kernel code. Do NOT include any explanation, comments about deprecated patterns, or discussion of what NOT to use."
+ ],
+ "expectedOutput": "ONLY flag as incorrect (score 0) if the output contains 'is None', 'is not None', 'is True', or 'is False' inside the kernel body. Any kernel that uses == or != for comparisons is correct (score 2). Do NOT penalize for any other NKI API usage patterns.",
+ "expectedTools": {
+ "required": [],
+ "alternatives": {}
+ },
+ "metadata": {
+ "category": "NKI 0.3.0 Enforcement",
+ "difficulty": "Easy",
+ "useCase": ["Verify is/is not operators are not used in kernel code"]
+ },
+ "expectedStrings": [
+ "@nki.jit",
+ "import nki"
+ ],
+ "forbiddenStrings": [
+ "is None",
+ "is not None",
+ "is True",
+ "is False"
+ ]
+ },
+ {
+ "taskId": "beta3-no-use-gpsimd-dma",
+ "input": [
+ "Write an NKI kernel that uses nisa.sendrecv for point-to-point communication between NeuronCores, sending a (128, 512) float32 tile using the GPSIMD DMA engine. Output ONLY the Python kernel code. Do NOT include any explanation, comments about deprecated patterns, or discussion of what NOT to use."
+ ],
+ "expectedOutput": "ONLY flag as incorrect (score 0) if the output contains 'use_gpsimd_dma'. Any kernel that uses sendrecv with dma_engine= parameter is correct (score 2). Do NOT penalize for unfamiliar API names.",
+ "expectedTools": {
+ "required": [],
+ "alternatives": {}
+ },
+ "metadata": {
+ "category": "NKI 0.3.0 Enforcement",
+ "difficulty": "Medium",
+ "useCase": ["Verify sendrecv uses dma_engine enum instead of use_gpsimd_dma"]
+ },
+ "expectedStrings": [
+ "nisa.sendrecv",
+ "dma_engine.gpsimd_dma"
+ ],
+ "forbiddenStrings": []
+ },
+ {
+ "taskId": "beta3-affine-select-offset-keyword",
+ "input": [
+ "Write an NKI kernel that uses nisa.affine_select to select elements from a (128, 512) float32 tile based on an affine pattern. Pass an offset value of 4 to the affine_select call. Output ONLY the Python kernel code. Do NOT include any explanation, comments about deprecated patterns, or discussion of what NOT to use."
+ ],
+ "expectedOutput": "ONLY flag as incorrect (score 0) if the output passes offset as a positional argument (3rd argument) to affine_select. Any kernel that uses offset=4 as a keyword argument is correct (score 2). Do NOT penalize for unfamiliar API names or parameter styles.",
+ "expectedTools": {
+ "required": [],
+ "alternatives": {}
+ },
+ "metadata": {
+ "category": "NKI 0.3.0 Enforcement",
+ "difficulty": "Medium",
+ "useCase": ["Verify affine_select uses offset as keyword argument"]
+ },
+ "expectedStrings": [
+ "nisa.affine_select",
+ "offset="
+ ],
+ "forbiddenStrings": []
+ },
+ {
+ "taskId": "beta3-dge-mode-type-match",
+ "input": [
+ "Write an NKI kernel that reinterprets a (128, 512) uint16 SBUF tensor as float4_e2m1fn_x4 and copies it to a float4_e2m1fn_x4 destination tile using hardware DGE mode. In NKI 0.3.0, source and destination types must match when using dge_mode=nisa.dge_mode.hwdge, so use .view() on the source to match types. Output ONLY the Python kernel code. Do NOT include any explanation, comments about deprecated patterns, or discussion of what NOT to use."
+ ],
+ "expectedOutput": "ONLY flag as incorrect (score 0) if the output uses dge_mode=hwdge with mismatched source and destination types without calling .view() to reinterpret. Any kernel that uses .view() to match types before dma_copy with hwdge is correct (score 2). Do NOT penalize for unfamiliar API names.",
+ "expectedTools": {
+ "required": [],
+ "alternatives": {}
+ },
+ "metadata": {
+ "category": "NKI 0.3.0 Enforcement",
+ "difficulty": "Medium",
+ "useCase": ["Verify .view() is used for type matching with dge_mode=hwdge"]
+ },
+ "expectedStrings": [
+ ".view(",
+ "dge_mode"
+ ],
+ "forbiddenStrings": []
+ },
+ {
+ "taskId": "beta3-dma-compute-param-order",
+ "input": [
+ "Write an NKI kernel that performs a scatter-add: accumulate values from a (128, 512) float32 source tensor into a (128, 512) float32 destination in HBM using element-wise addition. Output ONLY the Python kernel code. Do NOT include any explanation, comments about deprecated patterns, or discussion of what NOT to use."
+ ],
+ "expectedOutput": "ONLY flag as incorrect (score 0) if the output uses nisa.dma_copy with a dst_rmw_op argument for read-modify-write. Any kernel that calls nisa.dma_compute with reduce_op= is correct (score 2). Using nisa.dma_copy for loading and storing tiles is expected and correct. Do NOT penalize for unfamiliar API names or parameter styles.",
+ "expectedTools": {
+ "required": [],
+ "alternatives": {}
+ },
+ "metadata": {
+ "category": "NKI 0.3.0 Enforcement",
+ "difficulty": "Medium",
+ "useCase": ["Verify nisa.dma_compute uses correct NKI 0.3.0 parameter order"]
+ },
+ "expectedStrings": [
+ "nisa.dma_compute",
+ "reduce_op"
+ ],
+ "forbiddenStrings": []
+ },
+ {
+ "taskId": "beta3-no-num-channels-collectives",
+ "input": [
+ "Write an NKI kernel that uses collective_permute_implicit to send data between NeuronCores across 2 channels. Output ONLY the Python kernel code. Do NOT include any explanation, comments about deprecated patterns, or discussion of what NOT to use."
+ ],
+ "expectedOutput": "Score 2 (correct) if the kernel uses the channel_ids= keyword argument in its collective call. Give score 0 only if the kernel passes a lowercase num_channels= keyword argument to a collective API. Uppercase Python constants like NUM_CHANNELS used only as local variable names (not as a kwarg) are allowed. Do NOT penalize for unfamiliar API names.",
+ "expectedTools": {
+ "required": [],
+ "alternatives": {}
+ },
+ "metadata": {
+ "category": "NKI 0.3.0 Enforcement",
+ "difficulty": "Hard",
+ "useCase": ["Verify collectives use channel_ids instead of num_channels"]
+ },
+ "expectedStrings": [
+ "collective_permute"
+ ],
+ "forbiddenStrings": []
+ },
+ {
+ "taskId": "beta3-scan-dst-and-nl-ops",
+ "input": [
+ "Write an NKI kernel that performs an associative scan on two (128, 512) float32 input tiles using nisa.tensor_tensor_scan with multiply and add operations and initial value 0. Store the result to an output tensor in HBM. Output ONLY the Python kernel code. Do NOT include any explanation, comments about deprecated patterns, or discussion of what NOT to use."
+ ],
+ "expectedOutput": "Score 2 (correct) if the kernel uses nisa.tensor_tensor_scan(dst=..., ...) with dst= as a keyword argument and uses nl.multiply and nl.add as operators. Give score 0 if the output assigns the return value of nisa.tensor_tensor_scan to a variable or uses numpy ops like np.multiply or np.add. Do NOT penalize for unfamiliar API names.",
+ "expectedTools": {
+ "required": [],
+ "alternatives": {}
+ },
+ "metadata": {
+ "category": "NKI 0.3.0 Enforcement",
+ "difficulty": "Medium",
+ "useCase": ["Verify tensor_tensor_scan uses dst= pattern, nl.multiply/nl.add, not return-value style or numpy ops"]
+ },
+ "expectedStrings": [
+ "nisa.tensor_tensor_scan",
+ "dst=",
+ "nl.multiply",
+ "nl.add"
+ ],
+ "forbiddenStrings": [
+ "= nisa.tensor_tensor_scan"
+ ]
+ }
+ ]
+}
diff --git a/tests/neuron-nki-agent/nki040-enforcement.json b/tests/neuron-nki-agent/nki040-enforcement.json
new file mode 100644
index 0000000..50a8287
--- /dev/null
+++ b/tests/neuron-nki-agent/nki040-enforcement.json
@@ -0,0 +1,72 @@
+{
+ "scenarios": [
+ {
+ "taskId": "nki040-dma-transpose-rank-match",
+ "input": [
+ "Write an NKI kernel that transposes a (128, 4, 2, 512) float32 SBUF tensor using the full 4D permutation (3, 1, 2, 0), producing a (512, 4, 2, 128) output. Use nisa.dma_transpose. Output ONLY the Python kernel code. Do NOT include any explanation, comments about deprecated patterns, or discussion of what NOT to use."
+ ],
+ "expectedOutput": "ONLY flag as incorrect (score 0) if the dst tensor rank does not match the src tensor rank in the dma_transpose call. In NKI 0.4.0, dst.shape must have the same number of dimensions as src.shape. Any kernel where both src and dst have the same rank (4D in this case) is correct (score 2). Do NOT penalize for unfamiliar API names.",
+ "expectedTools": {
+ "required": [],
+ "alternatives": {}
+ },
+ "metadata": {
+ "category": "NKI 0.4.0 Enforcement",
+ "difficulty": "Medium",
+ "useCase": ["Verify dma_transpose dst rank matches src rank"]
+ },
+ "expectedStrings": [
+ "nisa.dma_transpose",
+ "dst="
+ ],
+ "forbiddenStrings": []
+ },
+ {
+ "taskId": "nki040-no-dynamic-copy-removed",
+ "input": [
+ "Write an NKI kernel that copies a 512-wide tile from a dynamically computed source offset within a (128, 2048) float32 SBUF tensor to a fixed destination tile. Use the latest NKI API for dynamic addressing. Output ONLY the Python kernel code. Do NOT include any explanation, comments about deprecated patterns, or discussion of what NOT to use."
+ ],
+ "expectedOutput": "ONLY flag as incorrect (score 0) if the output contains 'tensor_copy_dynamic_src' or 'tensor_copy_dynamic_dst'. These APIs are fully removed in NKI 0.4.0. Any kernel that uses nisa.tensor_copy with .ap() for dynamic addressing is correct (score 2). Do NOT penalize for unfamiliar API names.",
+ "expectedTools": {
+ "required": [],
+ "alternatives": {}
+ },
+ "metadata": {
+ "category": "NKI 0.4.0 Enforcement",
+ "difficulty": "Medium",
+ "useCase": ["Verify removed dynamic copy APIs are not used"]
+ },
+ "expectedStrings": [
+ "nisa.tensor_copy",
+ ".ap("
+ ],
+ "forbiddenStrings": [
+ "tensor_copy_dynamic_src",
+ "tensor_copy_dynamic_dst"
+ ]
+ },
+ {
+ "taskId": "nki040-tile-size-bytes-constants",
+ "input": [
+ "Write an NKI kernel that asserts the free dimension of a (128, F) float32 input tensor fits in SBUF before loading it. Use the NKI language tile_size bytes-aware constant to check the per-partition SBUF capacity. Output ONLY the Python kernel code. Do Not use invented constant. Do NOT include any explanation, comments about deprecated patterns, or discussion of what NOT to use."
+ ],
+ "expectedOutput": "Score 2 (correct) if the kernel uses tile_size.sbuf_fmax or tile_size.sbuf_fmax_bytes for the SBUF capacity check. Give score 0 only if it uses the deprecated tile_size.total_available_sbuf_size. Do NOT penalize for unfamiliar API names.",
+ "expectedTools": {
+ "required": [],
+ "alternatives": {}
+ },
+ "metadata": {
+ "category": "NKI 0.4.0 Enforcement",
+ "difficulty": "Easy",
+ "useCase": ["Verify new tile_size bytes constants are used instead of deprecated ones"]
+ },
+ "expectedStrings": [
+ "@nki.jit",
+ "tile_size"
+ ],
+ "forbiddenStrings": [
+ "total_available_sbuf_size"
+ ]
+ }
+ ]
+}
diff --git a/tests/registry.json b/tests/registry.json
new file mode 100644
index 0000000..c320b84
--- /dev/null
+++ b/tests/registry.json
@@ -0,0 +1,69 @@
+{
+ "name": "NeuronAgenticDevelopment-1.0",
+ "subsets": [
+ {
+ "name": "neuron-nki-agent",
+ "version": "1.0",
+ "judgeVersion": "2026-04-13",
+ "description": "NKI kernel development agent: Beta 2 API enforcement and NKI 0.3.0 enforcement",
+ "datasetPath": "./neuron-nki-agent",
+ "defaultJudge": "kiro-cli",
+ "taskIdSubset": [
+ "beta2-imports",
+ "beta2-dma-copy",
+ "beta2-isa-dst-param",
+ "beta2-direct-slicing",
+ "beta2-reduce-ops",
+ "beta2-bounds-no-mask",
+ "beta2-reciprocal-rsqrt",
+ "beta2-no-removed-params",
+ "beta2-nl-dtypes",
+ "beta2-dynamic-slicing",
+ "beta2-strided-access",
+ "beta3-no-platform-target",
+ "beta3-no-jit-mode",
+ "beta3-no-psum-dma-copy",
+ "beta3-no-string-buffer",
+ "beta3-no-integer-enum",
+ "beta3-shared-hbm-output",
+ "beta3-no-imm-register-move",
+ "beta3-no-keyword-only-separator",
+ "beta3-no-list-arguments",
+ "beta3-memset-type-match",
+ "beta3-no-dynamic-copy-deprecated",
+ "beta3-no-is-operator",
+ "beta3-no-use-gpsimd-dma",
+ "beta3-affine-select-offset-keyword",
+ "beta3-dge-mode-type-match",
+ "beta3-dma-compute-param-order",
+ "beta3-no-num-channels-collectives",
+ "beta3-scan-dst-and-nl-ops",
+ "nki040-dma-transpose-rank-match",
+ "nki040-no-dynamic-copy-removed",
+ "nki040-tile-size-bytes-constants"
+ ]
+ },
+ {
+ "name": "neuron-framework-autoport-agent",
+ "version": "1.4",
+ "judgeVersion": "2026-04-10",
+ "description": "NxDI Autoport agent: NeuronX model porting structure",
+ "datasetPath": "./neuron-framework-autoport-agent",
+ "defaultJudge": "kiro-cli",
+ "taskIdSubset": [
+ "autoport-starcoder2-modeling-dry-run"
+ ]
+ },
+ {
+ "name": "neuron-dev-contribution-validation-agent",
+ "version": "1.0",
+ "judgeVersion": "2026-05-07",
+ "description": "NAD contribution validation: checks artifacts against contribution model requirements",
+ "datasetPath": "./neuron-dev-contribution-validation-agent",
+ "defaultJudge": "kiro-cli",
+ "taskIdSubset": [
+ "validate-missing-metadata"
+ ]
+ }
+ ]
+}