From 175c8e9a8bce19660e2f94ff68d2a99a8c662a36 Mon Sep 17 00:00:00 2001 From: hyeonsang Date: Tue, 8 Sep 2026 01:19:23 +0900 Subject: [PATCH] fix(plugin): refine Astra contracts and harden WorkGraph for v0.5.4 --- README.md | 35 ++- docs/EVALS.md | 48 ++- docs/RELEASE-v0.5.4-KO.md | 33 ++ .../.codex-plugin/plugin.json | 2 +- .../.codex-plugin/plugin.json | 2 +- .../references/evidence-contract.md | 20 +- .../schemas/evidence-contract.schema.json | 2 +- .../schemas/workgraph.schema.json | 2 +- .../scripts/import-requirements.py | 25 +- .../scripts/inspect-release-state.py | 55 ++++ .../scripts/validate-evidence.py | 2 +- .../scripts/validate-screen-spec.py | 19 +- .../wigtn-plugins-with-codex/scripts/wigtn.py | 67 ++++- .../scripts/workgraph_core.py | 37 ++- .../skills/acceptance-verifier/SKILL.md | 60 ++-- .../references/evidence-matrix.md | 23 +- .../references/saved-evidence.md | 5 +- .../skills/design-direction/SKILL.md | 2 +- .../skills/handdrawn-diagram/SKILL.md | 2 +- .../handdrawn-diagram/references/rendering.md | 4 +- .../skills/product-spec/SKILL.md | 48 ++- .../references/create-contract.md | 39 +-- .../references/review-contract.md | 12 +- .../skills/release-readiness/SKILL.md | 11 +- .../references/git-safety.md | 2 +- .../skills/screen-spec/SKILL.md | 14 +- .../skills/verified-delivery/SKILL.md | 64 ++-- .../references/delivery-evidence.md | 83 ++--- .../skills/work-planner/SKILL.md | 75 ++--- .../references/workgraph-contract.md | 13 +- scripts/check-codex-usage-missing.py | 41 +++ scripts/check-codex-usage.py | 73 +++++ scripts/check-model-migration-eval.py | 284 ++++++++++++++++++ scripts/check-release-state.py | 45 ++- scripts/check-screen-contract.py | 25 +- scripts/check-symbolic-requirement-ids.py | 118 ++++++++ scripts/check-token-ablation.py | 94 ++++++ scripts/check-verified-delivery-routing.py | 2 +- scripts/check-workgraph-concurrency.py | 54 ++++ scripts/check-workgraph-import-root.py | 22 ++ scripts/check-workgraph-inspect.py | 30 ++ scripts/check-workgraph-project-guard.py | 16 + scripts/codex_usage.py | 149 +++++++++ scripts/export-eval-packet.py | 2 +- scripts/migration_cases.py | 64 ++++ scripts/migration_evidence.py | 65 ++++ scripts/run-behavior-evals.sh | 27 +- scripts/run-model-migration-eval.py | 174 +++++++++++ scripts/run-ordinary-gate.sh | 2 +- scripts/run-package-ablation.sh | 2 +- scripts/run-static-contracts.sh | 13 + scripts/run-token-ablation.sh | 110 +++++++ scripts/run-workgraph-pilot.sh | 2 +- scripts/score-behavior-smoke.py | 24 +- scripts/score-model-migration-eval.py | 246 +++++++++++++++ scripts/score-token-ablation.py | 167 ++++++++++ scripts/summarize-token-efficiency.py | 133 ++++++++ scripts/validate.sh | 1 - tests/behavior/token-efficiency-cases.tsv | 3 + 59 files changed, 2474 insertions(+), 320 deletions(-) create mode 100644 docs/RELEASE-v0.5.4-KO.md create mode 100644 scripts/check-codex-usage-missing.py create mode 100755 scripts/check-codex-usage.py create mode 100644 scripts/check-model-migration-eval.py create mode 100644 scripts/check-symbolic-requirement-ids.py create mode 100755 scripts/check-token-ablation.py create mode 100644 scripts/check-workgraph-concurrency.py create mode 100644 scripts/check-workgraph-import-root.py create mode 100644 scripts/check-workgraph-inspect.py create mode 100644 scripts/check-workgraph-project-guard.py create mode 100755 scripts/codex_usage.py create mode 100644 scripts/migration_cases.py create mode 100644 scripts/migration_evidence.py create mode 100644 scripts/run-model-migration-eval.py create mode 100755 scripts/run-token-ablation.sh create mode 100644 scripts/score-model-migration-eval.py create mode 100755 scripts/score-token-ablation.py create mode 100755 scripts/summarize-token-efficiency.py create mode 100644 tests/behavior/token-efficiency-cases.tsv diff --git a/README.md b/README.md index 32709c5..41ec8c5 100644 --- a/README.md +++ b/README.md @@ -2,9 +2,9 @@ # WIGTN Plugin for Codex -**Codex의 자율성은 그대로. PRD·작업 계획·검증·Git 권한은 필요한 순간에만.** +**PRD·저장 가능한 작업 계획·요구사항 검증·Git 작업을 위한 선택형 스킬 모음.** -![Version](https://img.shields.io/badge/version-0.5.3-6C5CE7?style=for-the-badge) +![Version](https://img.shields.io/badge/version-0.5.4-6C5CE7?style=for-the-badge) ![Skills](https://img.shields.io/badge/core_skills-9-00B894?style=for-the-badge) ![Platform](https://img.shields.io/badge/platform-Codex-111827?style=for-the-badge) ![License](https://img.shields.io/badge/license-Apache--2.0-0984E3?style=for-the-badge) @@ -120,6 +120,18 @@ release-readiness ──── 안전한 커밋·푸시·PR --- +## v0.5.4: Astra 지침 정리·검증 정확성·WorkGraph 복구 + +- 요청 범위에 맞게 스킬·참조 문서를 정리하고, 명시 호출형 Verified Delivery와 일반 코딩의 경계를 유지했습니다. +- 요구사항 ID·Markdown 목록 처리, 실제 제약 위반 판정, 저장 증거와 보고서 충돌 처리를 개선했습니다. +- WorkGraph inspect와 import 경로 처리를 개선하고, 동시 CLI 저장의 변경 유실과 잘못된 프로젝트 설정의 계획 저장을 수정했습니다. +- CLI usage의 미보고 항목을 0으로 채우지 않고, 측정 가능한 항목과 비용 추정의 한계를 구분합니다. +- 추가 축약 실험본은 채택하지 않았습니다. Knowledge Wiki는 공동 버전 정책에 따른 manifest 변경이며 기능은 그대로입니다. + +[변경 내용과 검증 범위](docs/RELEASE-v0.5.4-KO.md). 모델 비교 결과와 최종 runtime의 로컬 검증을 구분하며 일반 성능·비용 우위를 주장하지 않습니다. + +--- + ## v0.5.3: 하네스 경량화·스택 중립화 - 항상 노출되는 Core 스킬 설명 합계를 2,980자에서 2,162자로 줄여 @@ -230,12 +242,15 @@ python3 plugins/wigtn-plugins-with-codex/scripts/wigtn.py --json init python3 plugins/wigtn-plugins-with-codex/scripts/wigtn.py --json init --apply python3 plugins/wigtn-plugins-with-codex/scripts/wigtn.py --json import docs/PRD.md --apply python3 plugins/wigtn-plugins-with-codex/scripts/wigtn.py --json plan --apply +python3 plugins/wigtn-plugins-with-codex/scripts/wigtn.py --json inspect python3 plugins/wigtn-plugins-with-codex/scripts/wigtn.py --json status python3 plugins/wigtn-plugins-with-codex/scripts/wigtn.py --json next python3 plugins/wigtn-plugins-with-codex/scripts/wigtn.py --json diff --check python3 plugins/wigtn-plugins-with-codex/scripts/wigtn.py --json doctor ``` +`inspect`는 저장 파일을 검증하고 source drift와 다음 작업을 읽기 전용으로 조회합니다. 저장된 검사 명령을 실행하거나 변경을 적용하지 않습니다. `import`의 상대경로는 `--root` 기준입니다. + mutation 명령은 `--apply`가 없으면 dry-run이며, 동일한 import·plan·drift 적용은 revision을 올리지 않습니다. 자세한 상태 의미는 [WorkGraph lifecycle](docs/WORKGRAPH-LIFECYCLE.md)을 참고하세요. @@ -345,8 +360,10 @@ Claude 전용 도구 이름, 고정 서브에이전트 fan-out, 자동 모델 ``` 이 suite에는 plugin resource 무결성, 다이어그램·HTML 발표 계약, -responsive wireframe portability, WorkGraph lifecycle·drift·migration·CLI의 -67개 결정론적 케이스와 paired schedule 무결성 검사도 포함됩니다. +한국어/영어 화면 제목과 artifact 선택 형식을 포함한 screen contract, +untracked 내용을 읽지 않는 bounded release diff, WorkGraph +lifecycle·drift·migration·CLI의 67개 결정론적 케이스와 paired schedule +무결성 검사도 포함됩니다. 신규 `work-planner`의 모델 기반 capability pilot은 12개 격리 저장소에서 별도로 실행합니다. @@ -382,6 +399,16 @@ effect는 [일반 코딩 비간섭 보고서](docs/ORDINARY-NONINTERFERENCE-GATE ./scripts/run-behavior-evals.sh --execute ``` +커밋된 `HEAD`와 현재 worktree의 Compact PRD·근거 부족 acceptance 경로를 +3회씩 비교하려면 focused ablation을 실행합니다. JSONL에서 input, cached +input, cache-write, output, reasoning, tool-item을 분리하며 비용은 실제 +구독 청구가 아닌 API 환산 비교치입니다. + +```bash +WIGTN_TOKEN_ABLATION_ROOT=/tmp/fresh-token-ablation \ + ./scripts/run-token-ablation.sh --execute +``` + Core 4의 내용 효과와 catalog 길이 효과를 분리하는 package ablation은: ```bash diff --git a/docs/EVALS.md b/docs/EVALS.md index c0696db..490f2d4 100644 --- a/docs/EVALS.md +++ b/docs/EVALS.md @@ -29,9 +29,11 @@ like proof of model quality. - optional project context rejects unsafe paths and unknown configuration - screen bundles reject missing artifacts, unresolved template values, broken wireframe anchors, remote resources, missing viewport metadata, and - cross-artifact requirement drift + cross-artifact requirement drift; validation accepts comma/space artifact + selections and English or Korean screen headings - release-state inspection distinguishes staged, unstaged, untracked, - conflicted, detached, and in-progress Git states without mutation + conflicted, detached, and in-progress Git states without mutation; optional + bounded patches include whitespace checks but never untracked contents - sanitized eval packets redact run roots and secrets, hash membership, exclude auth/work homes, and detect tampering - WorkGraph schema, ID/reference integrity, dependency cycles, false @@ -94,6 +96,37 @@ and a results file outside the repository by default. Its four prompts cover PRD creation, uncertain acceptance evidence, IA-only scope, and an ordinary coding request. This is deliberately not full skill-router coverage. +Runs use Codex JSONL events to report input, cached input, cache-write input, +output, reasoning output, and tool-item counts in `TOKEN-EFFICIENCY.md` and +`TOKEN-EFFICIENCY.json`. API-equivalent cost uses explicit, overridable rate +variables and is only a normalized comparison metric; it is not a ChatGPT +subscription charge. The report includes observed cache pricing and a cold +comparison that prices all input as uncached, so arm order and cache hits are +visible rather than silently deciding the conclusion. Keep quality gates +separate from this cost report. + +For a focused before/after check, compare the committed `HEAD` plugin with the +current worktree candidate. The default is three repetitions over Compact PRD +and evidence-poor acceptance prompts, for 12 model calls: + +```bash +./scripts/run-token-ablation.sh +WIGTN_TOKEN_ABLATION_ROOT=/tmp/fresh-token-ablation \ + ./scripts/run-token-ablation.sh --execute +``` + +The scorer requires preserved Compact PRD shape, exact `not-verifiable`, zero +tools where the prompt prohibits commands, and lower candidate PRD median tool +items and input tokens. This is a targeted development ablation, not a general +plugin-efficiency claim. + +For new token work, inspect command-execution traces before editing prose. +Prefer one bounded deterministic collector when the model repeatedly fetches +related state, while keeping judgment in the model. If a path already uses no +tools or combines its reads in one call, do not claim that shortening files +alone improves token efficiency; measure a quality-preserving alternative +first. + The smoke scorer checks execution health only. It deliberately does not turn output keywords into a quality score. Publication claims require frozen task-specific scorers, repeated runs, and human review. @@ -242,3 +275,14 @@ The current release policy is: - keep product/screen contracts and validators, - use the optional Evidence Contract for explicit cross-workflow handoffs, - block release on any unauthorized Git mutation. + +## GPT-6 Astra migration pilot + +Use `scripts/run-model-migration-eval.py --baseline SNAPSHOT --root FRESH_PATH` to inspect the 24-call plan; add `--execute` only with authorization for model execution and plugin-file transfer. This snapshots the pre-change dirty worktree instead of comparing against HEAD. Run `scripts/score-model-migration-eval.py RUN_ROOT` afterward. Completeness is separate from semantic quality review. The default is Astra/medium; no dollar costs are estimated. See [the release verification scope](RELEASE-v0.5.4-KO.md). The legacy token ablation runner remains Sol-only because its rates are pinned. + +The second-pass evaluator adds `--suite targeted` (30 calls at two repetitions) for explicit verified-delivery activation. It uses a fixed implementation oracle, records unexpected worktree changes, rejects empty schedules and failed/duplicate turns, and leaves unknown telemetry as null. This remains a development pilot; consult the report for untested authority and long-running scenarios. No additional external execution is authorized by choosing a suite. + + +## Schema v2 evidence and adjudication + +The current migration evaluator verifies sealed inputs and per-run evidence, retains measured consumption from failed attempts, and separates schedule, integrity, execution, review and task pass states. Use `--review-template` to export review forms and `--reviews FILE --require-pass` for the final gate. The new `--suite contracts` adds direct Compact/external-PRD/six-conflict cases (30 calls at repeat=2). See [the release verification scope](RELEASE-v0.5.4-KO.md). Old unsealed packets remain historical data, not v2-qualified results. diff --git a/docs/RELEASE-v0.5.4-KO.md b/docs/RELEASE-v0.5.4-KO.md new file mode 100644 index 0000000..eda10b7 --- /dev/null +++ b/docs/RELEASE-v0.5.4-KO.md @@ -0,0 +1,33 @@ +# v0.5.4 변경 내용과 검증 범위 + +GPT-6 Astra에 맞춰 선택형 지침을 정리하고, 요구사항 판정과 WorkGraph 저장의 실패 경계를 수정했다. Core와 Knowledge Wiki manifest를 공동 버전 정책에 따라 0.5.4로 맞췄다. Wiki 기능 변경은 없다. + +## 사용자에게 달라지는 동작 + +- 일반 코딩에 Verified Delivery를 자동 적용하지 않는다. 요청한 작업에 비례하는 검증을 유지하며 불필요한 고정 절차와 중복 참조를 줄였다. +- Compact PRD와 외부 PRD는 해당 문서의 계약으로 검토한다. 중대한 결함을 고정된 개수 제한 때문에 누락하지 않는다. +- acceptance 판정에서 실제 제약 위반과 검증 불가능을 구분하고, 저장된 증거와 보고서가 충돌하면 그 차이를 보존한다. +- 숫자형 요구사항 ID와 함께 `REQ-A` 같은 대문자 기호형 ID를 import·WorkGraph·evidence에서 일관되게 처리한다. Markdown 목록의 ID와 문장 속 참조를 구분한다. +- 화면 검증은 한국어 `화면:` 제목과 쉼표 또는 공백으로 구분한 artifact 선택을 받는다. +- `wigtn.py inspect`는 저장 상태를 검증하고 source drift와 다음 작업을 읽기 전용으로 보여준다. 저장된 명령은 실행하지 않는다. import의 상대경로는 `--root` 기준으로 해석한다. +- 동시 CLI 저장은 read→revision 검사→save 전체를 OS advisory lock으로 보호한다. 최신 상태를 읽어 두 변경을 보존하거나, 명시한 revision이 오래되면 충돌로 거부한다. +- 잘못된 프로젝트 설정은 plan 저장 전에 거부한다. 오류 응답은 exit 2이며 기존 그래프 내용은 보존한다. +- release-state의 선택적 diff 조회는 크기를 제한하고 잘림 여부를 표시한다. untracked 파일의 내용은 읽지 않는다. + +## 검증 + +필수 검증 명령은 `bash scripts/validate.sh`다. 모델 서비스를 호출하지 않는 저장소 계약·회귀 검사이며 CI에서도 실행한다. + +추가 검사는 기호형 ID, 화면 계약, bounded release diff, usage 누락, 평가 증거 무결성, WorkGraph inspect와 import 경로를 포함한다. 동시성 검사 네 개는 실제 CLI 프로세스를 사용해 변경 보존, 오래된 revision 거부, dry-run 무변경, 종료된 writer의 잠금 해제를 확인한다. 프로젝트 설정 검사는 잘못된 설정 네 종류를 preview/apply 각각에 적용한다. + +실험 실행기와 채점기를 포함하되 모델 호출은 명시 실행 시에만 수행한다. 기록되지 않은 usage는 0으로 추정하지 않는다. API 환산값과 실제 구독 청구액은 서로 다른 값이다. + +## 실측 해석과 남은 경계 + +개발 과정에서는 GPT-6 Astra/medium, Codex CLI 0.153.4에서 기본 Codex·선택형 C8·추가 축약 조건을 비교했다. 42개 작업 단위에 인계·재개 단계를 포함한 CLI 호출은 48회였다. 최종 C9 runtime 자체를 별도 모델 비교 조건으로 실행하지는 않았다. + +추가 축약 조건은 채택하지 않았다. 작업은 개발자가 구성한 작은 표본이며, 사후 채점 정정과 미완료 실행을 포함한다. 이 PR은 일반 코딩 우위, 보편적 비용 절감률, 최종 0.5.4의 독립적인 모델 성능 개선을 주장하지 않는다. + +잠금은 협력하는 CLI writer 사이에서 작동한다. 직접 JSON을 편집하는 프로그램은 참여하지 않으며 Windows 잠금 경로는 실측하지 않았다. inspect의 구조 검증은 실제 코드 실행이나 요구사항 충족의 증명이 아니다. + +상세 기술보고서 초안과 원시 세션 자료는 로컬 연구 자료로 별도 보존한다. 이 PR에는 공개용 정리가 끝나지 않은 원시 로그·실험 작업 폴더를 포함하지 않는다. 본 문서는 그 자료를 독립 재현할 수 있는 공개 증거 묶음이 아니다. diff --git a/plugins/wigtn-knowledge-wiki/.codex-plugin/plugin.json b/plugins/wigtn-knowledge-wiki/.codex-plugin/plugin.json index 7497ea6..151ec52 100644 --- a/plugins/wigtn-knowledge-wiki/.codex-plugin/plugin.json +++ b/plugins/wigtn-knowledge-wiki/.codex-plugin/plugin.json @@ -1,6 +1,6 @@ { "name": "wigtn-knowledge-wiki", - "version": "0.5.3", + "version": "0.5.4", "description": "Opt-in, gated Codex session knowledge capture for an explicitly authorized personal wiki namespace.", "author": { "name": "WIGTN" diff --git a/plugins/wigtn-plugins-with-codex/.codex-plugin/plugin.json b/plugins/wigtn-plugins-with-codex/.codex-plugin/plugin.json index 3933ea9..17c5e70 100644 --- a/plugins/wigtn-plugins-with-codex/.codex-plugin/plugin.json +++ b/plugins/wigtn-plugins-with-codex/.codex-plugin/plugin.json @@ -1,6 +1,6 @@ { "name": "wigtn-plugins-with-codex", - "version": "0.5.3", + "version": "0.5.4", "description": "Selective product contracts for Codex: compact PRDs, saved WorkGraphs, scoped screen artifacts, acceptance evidence, explicit verified delivery, and safe release actions.", "author": { "name": "WIGTN" diff --git a/plugins/wigtn-plugins-with-codex/references/evidence-contract.md b/plugins/wigtn-plugins-with-codex/references/evidence-contract.md index c63a665..a92bc33 100644 --- a/plugins/wigtn-plugins-with-codex/references/evidence-contract.md +++ b/plugins/wigtn-plugins-with-codex/references/evidence-contract.md @@ -17,18 +17,20 @@ python3 /scripts/validate-evidence.py - `verified`: precise implementation evidence and a referenced passing check - `implemented-not-executed`: implementation evidence exists; no relevant passing check was executed -- `partially-verified`: only part of the observable requirement is supported -- `not-satisfied`: evidence shows the requirement is absent or contradicted +- `partially-verified`: part is supported, the rest is unresolved, and no required + behavior is demonstrated false +- `not-satisfied`: evidence shows required behavior is absent or contradicted; + this takes precedence over passing other examples - `not-verifiable`: available evidence cannot support a conclusion - `not-applicable`: the requirement does not apply to this scope -Never mark a requirement `verified` from documentation, model narration, an -unexecuted test, a check that is not referenced by ID, or a green-only check -written by the same agent during the implementation. An agent-authored check -can support `verified` only when the artifact or accompanying trace records its -pre-change failure and post-change pass. Until check provenance is represented -in the machine-readable schema, record that distinction in `limitations` and -use `partially-verified` when independent evidence is absent. +Never mark a requirement `verified` from documentation, narration, an +unexecuted or unrelated test, or a check that is not referenced by ID. +Judge executed checks by their coverage of the current requirement. Record +whether they were pre-existing, supplied externally, or authored for this work; +after-change-only checks are not red/green evidence. Their authorship alone +neither proves coverage nor requires a downgrade. Use `partially-verified` for +an actual unresolved subclaim, and `not-satisfied` for a demonstrated violation. ## Portability and authority diff --git a/plugins/wigtn-plugins-with-codex/schemas/evidence-contract.schema.json b/plugins/wigtn-plugins-with-codex/schemas/evidence-contract.schema.json index 5c6ace3..fd36e63 100644 --- a/plugins/wigtn-plugins-with-codex/schemas/evidence-contract.schema.json +++ b/plugins/wigtn-plugins-with-codex/schemas/evidence-contract.schema.json @@ -64,7 +64,7 @@ "properties": { "id": { "type": "string", - "pattern": "^[A-Z][A-Z0-9_-]*-[0-9]{2,}$" + "pattern": "^[A-Z][A-Z0-9_]*(?:-[A-Z0-9_]+)+$" }, "text": { "type": "string", diff --git a/plugins/wigtn-plugins-with-codex/schemas/workgraph.schema.json b/plugins/wigtn-plugins-with-codex/schemas/workgraph.schema.json index 1dc25c4..1c4b2fd 100644 --- a/plugins/wigtn-plugins-with-codex/schemas/workgraph.schema.json +++ b/plugins/wigtn-plugins-with-codex/schemas/workgraph.schema.json @@ -111,7 +111,7 @@ "properties": { "id": { "type": "string", - "pattern": "^[A-Z][A-Z0-9_-]*-[0-9]{2,}$" + "pattern": "^[A-Z][A-Z0-9_]*(?:-[A-Z0-9_]+)+$" }, "source_id": { "type": "string", diff --git a/plugins/wigtn-plugins-with-codex/scripts/import-requirements.py b/plugins/wigtn-plugins-with-codex/scripts/import-requirements.py index 5f8ddd5..01ddbac 100755 --- a/plugins/wigtn-plugins-with-codex/scripts/import-requirements.py +++ b/plugins/wigtn-plugins-with-codex/scripts/import-requirements.py @@ -13,11 +13,14 @@ FORMATS = {"auto", "wigtn", "spec-kit", "openspec", "bmad", "generic"} -STABLE_ID = re.compile(r"^[A-Z][A-Z0-9_-]*-[0-9]{2,}$", re.I) +# Symbolic IDs are uppercase labels; legacy numeric IDs remain case-insensitive. +# Anchor labels to the start and require a text separator to avoid extracting +# references inside prose or splitting an ID into an ID plus requirement text. +ID_TOKEN = r"(?:[A-Z][A-Z0-9_]*(?:-[A-Z0-9_]+)+|(?i:[A-Z][A-Z0-9_]*(?:-[A-Z0-9_]+)*-[0-9]{2,}))" +STABLE_ID = re.compile(rf"^{ID_TOKEN}$") INLINE_ID = re.compile( - r"(?:\*\*|`)?(?P[A-Z][A-Z0-9_-]*-[0-9]{2,})(?:\*\*|`)?" - r"\s*(?::|[-–—])?\s*(?P.+)", - re.I, + rf"^(?:\*\*|`)?(?P{ID_TOKEN})(?:\*\*|`)?" + r"(?:\s*[:–—]\s*|\s+-\s+|\s+)(?P\S.*)$" ) @@ -59,11 +62,15 @@ def explicit_requirements(text: str) -> list[tuple[str, str]]: for line in text.splitlines(): stripped = line.strip() if stripped.startswith("|"): - cells = [clean_text(cell) for cell in stripped.strip("|").split("|")] - if len(cells) >= 2 and STABLE_ID.fullmatch(cells[0]): - results.append((cells[0].upper(), cells[1])) + cells = [cell.strip() for cell in stripped.strip("|").split("|")] + label = re.sub(r"^(\*\*|__|\*|_|`)(.+)\1$", r"\2", cells[0]) + if len(cells) >= 2 and STABLE_ID.fullmatch(label): + results.append((label.upper(), clean_text(cells[1]))) continue - match = INLINE_ID.search(stripped.lstrip("#-+ 0123456789.")) + # Remove Markdown structure, not arbitrary prose before a requirement. + label_line = re.sub(r"^(?:#{1,6}\s+|(?:[-*+]|\d+[.)])\s+)", "", stripped) + label_line = re.sub(r"^\[[ xX]\]\s+", "", label_line) + match = INLINE_ID.search(label_line) if match: results.append( (match.group("id").upper(), clean_text(match.group("text"))) @@ -131,7 +138,7 @@ def derived_acceptance( requirement = clean_text(match.group(1)) if not requirement or set(requirement) <= {"-", ":"}: continue - explicit = INLINE_ID.search(requirement) + explicit = INLINE_ID.search(match.group(1).strip()) if explicit: requirement_id = explicit.group("id").upper() requirement = clean_text(explicit.group("text")) diff --git a/plugins/wigtn-plugins-with-codex/scripts/inspect-release-state.py b/plugins/wigtn-plugins-with-codex/scripts/inspect-release-state.py index b5d9785..676cb9b 100644 --- a/plugins/wigtn-plugins-with-codex/scripts/inspect-release-state.py +++ b/plugins/wigtn-plugins-with-codex/scripts/inspect-release-state.py @@ -9,6 +9,9 @@ import subprocess +DEFAULT_MAX_DIFF_BYTES = 64 * 1024 + + def git(root: Path, *args: str, check: bool = True) -> subprocess.CompletedProcess[str]: return subprocess.run( ["git", "-C", str(root), *args], @@ -23,10 +26,51 @@ def names(root: Path, *args: str) -> list[str]: return sorted(value for value in result.stdout.split("\0") if value) +def bounded_text(value: str, max_bytes: int) -> tuple[str, int, bool]: + encoded = value.encode("utf-8") + if len(encoded) <= max_bytes: + return value, len(encoded), False + return encoded[:max_bytes].decode("utf-8", errors="ignore"), len(encoded), True + + +def diff_section(root: Path, *args: str, max_bytes: int) -> dict[str, object]: + patch = git(root, "diff", *args).stdout + stat = git(root, "diff", *args, "--stat").stdout + bounded_patch, byte_count, truncated = bounded_text(patch, max_bytes) + return { + "bytes": byte_count, + "truncated": truncated, + "stat": stat, + "patch": bounded_patch, + } + + +def check_section(root: Path, *args: str) -> dict[str, object]: + result = git(root, "diff", *args, "--check", check=False) + return { + "exit_code": result.returncode, + "clean": result.returncode == 0, + "output": result.stdout + result.stderr, + } + + def main() -> int: parser = argparse.ArgumentParser() parser.add_argument("repository", nargs="?", type=Path, default=Path(".")) + parser.add_argument( + "--include-diffs", + action="store_true", + help="include bounded staged and unstaged patches plus whitespace checks", + ) + parser.add_argument( + "--max-diff-bytes", + type=int, + default=DEFAULT_MAX_DIFF_BYTES, + help="maximum UTF-8 bytes returned for each staged/unstaged patch", + ) args = parser.parse_args() + if args.max_diff_bytes <= 0: + parser.error("--max-diff-bytes must be positive") root = args.repository.resolve() probe = git(root, "rev-parse", "--is-inside-work-tree", check=False) if probe.returncode or probe.stdout.strip() != "true": @@ -83,6 +127,17 @@ def main() -> int: root, "diff", "--name-only", "--diff-filter=U", "-z" ), } + if args.include_diffs: + document["diffs"] = { + "staged": diff_section( + root, "--cached", max_bytes=args.max_diff_bytes + ), + "unstaged": diff_section(root, max_bytes=args.max_diff_bytes), + "check": { + "staged": check_section(root, "--cached"), + "unstaged": check_section(root), + }, + } print(json.dumps(document, indent=2, ensure_ascii=False)) return 0 diff --git a/plugins/wigtn-plugins-with-codex/scripts/validate-evidence.py b/plugins/wigtn-plugins-with-codex/scripts/validate-evidence.py index 34dab67..1a1ef77 100755 --- a/plugins/wigtn-plugins-with-codex/scripts/validate-evidence.py +++ b/plugins/wigtn-plugins-with-codex/scripts/validate-evidence.py @@ -13,7 +13,7 @@ PLUGIN_ROOT = Path(__file__).resolve().parents[1] DEFAULT_SCHEMA = PLUGIN_ROOT / "schemas" / "evidence-contract.schema.json" -REQUIREMENT_ID = re.compile(r"^[A-Z][A-Z0-9_-]*-[0-9]{2,}$") +REQUIREMENT_ID = re.compile(r"^[A-Z][A-Z0-9_]*(?:-[A-Z0-9_]+)+$") CHECK_ID = re.compile(r"^CHK-[0-9]{2,}$") ARTIFACT_TYPES = {"product-spec", "acceptance", "delivery", "release"} REQUIREMENT_STATUSES = { diff --git a/plugins/wigtn-plugins-with-codex/scripts/validate-screen-spec.py b/plugins/wigtn-plugins-with-codex/scripts/validate-screen-spec.py index f139dc6..5db1e6e 100644 --- a/plugins/wigtn-plugins-with-codex/scripts/validate-screen-spec.py +++ b/plugins/wigtn-plugins-with-codex/scripts/validate-screen-spec.py @@ -38,7 +38,7 @@ r'|(?:@import\s+(?:url\(\s*)?|url\(\s*)["\']?\s*(?:(?:https?:)?//)', re.I, ) -SCREEN_HEADING = re.compile(r"^## Screen:\s*(.+?)\s*$", re.M) +SCREEN_HEADING = re.compile(r"^## (?:Screen|화면):\s*(.+?)\s*$", re.M) def has_ia_page_map(text: str) -> bool: @@ -70,17 +70,20 @@ def main() -> int: parser.add_argument("directory", type=Path) parser.add_argument( "--artifacts", - default="all", - help="Comma-separated ia,flow,screen,wireframe,handoff or all", + nargs="+", + default=["all"], + help="Comma- or space-separated ia,flow,screen,wireframe,handoff or all", ) parser.add_argument("--json", action="store_true") args = parser.parse_args() errors: list[str] = [] - requested = ( - set(ARTIFACTS) - if args.artifacts == "all" - else {item.strip() for item in args.artifacts.split(",") if item.strip()} - ) + requested_tokens = { + item.strip() + for value in args.artifacts + for item in value.split(",") + if item.strip() + } + requested = set(ARTIFACTS) if requested_tokens == {"all"} else requested_tokens unknown = requested - set(ARTIFACTS) if not requested: errors.append("--artifacts must select at least one artifact") diff --git a/plugins/wigtn-plugins-with-codex/scripts/wigtn.py b/plugins/wigtn-plugins-with-codex/scripts/wigtn.py index 989713c..b7bd869 100755 --- a/plugins/wigtn-plugins-with-codex/scripts/wigtn.py +++ b/plugins/wigtn-plugins-with-codex/scripts/wigtn.py @@ -13,6 +13,7 @@ from workgraph_core import ( atomic_write_json, empty_graph, + graph_write_lock, inspect_drift, merge_import, plan_graph, @@ -153,7 +154,7 @@ def run_importer(root: Path, sources: list[Path]) -> dict[str, Any]: def command_import(args: argparse.Namespace) -> int: root = root_path(args) graph = require_graph(root) - sources = [path.resolve() for path in args.sources] + sources = [(root / path).resolve() for path in args.sources] for source in sources: repository_relative(source, root) if not source.is_file(): @@ -181,11 +182,21 @@ def command_import(args: argparse.Namespace) -> int: def command_plan(args: argparse.Namespace) -> int: root = root_path(args) graph = require_graph(root) - project = ( - read_json(project_path(root)) - if project_path(root).is_file() - else default_project() - ) + context = project_path(root) + if context.is_file(): + validation = subprocess.run( + [sys.executable, str(SCRIPT_DIR / "validate-project-context.py"), + str(context), "--json"], + text=True, capture_output=True, check=False, + ) + if validation.returncode: + raise ValueError( + "invalid project context: " + + (validation.stdout.strip() or validation.stderr.strip()) + ) + project = read_json(context) + else: + project = default_project() updated = plan_graph( graph, verification_commands=list(project.get("verification_commands", [])), @@ -278,6 +289,43 @@ def command_doctor(args: argparse.Namespace) -> int: return 1 if issues else 0 +def command_inspect(args: argparse.Namespace) -> int: + """Read-only validated, drift-adjusted view; never execute saved commands.""" + root = root_path(args) + graph = require_graph(root) + current, drift = inspect_drift(graph, root) + validations = {} + for label, path, script in ( + ("project", project_path(root), "validate-project-context.py"), + ("evidence", evidence_path(root), "validate-evidence.py"), + ): + result = subprocess.run( + [sys.executable, str(SCRIPT_DIR / script), str(path)], + text=True, capture_output=True, check=False, + ) + validations[label] = { + "valid": result.returncode == 0, + "exit_code": result.returncode, + "details": result.stdout + result.stderr, + } + valid = all(item["valid"] for item in validations.values()) + current_summary = summary(current) + if not valid: + current_summary["next_task_ids"] = [] + emit({ + "operation": "inspect", "read_only": True, + "valid_artifacts": valid, "fresh": not drift, + "validations": validations, "source_drift": drift, + "summary": current_summary, "sources": current["sources"], + "requirements": current["requirements"], + "tasks": current["tasks"], "checks": current["checks"], + "release_gates": current["release_gates"], + "next_tasks": ready_tasks(current) if valid else [], + "verification_boundary": "Saved evidence validity is structural, not proof of executed behavior. Commands are not executed. Drift is previewed, not persisted.", + }, args.json) + return 0 if valid and not drift else 1 + + def command_task_update(args: argparse.Namespace) -> int: root = root_path(args) graph = require_graph(root) @@ -346,7 +394,7 @@ def parser() -> argparse.ArgumentParser: init.add_argument("--apply", action="store_true") init.set_defaults(handler=command_init) import_command = commands.add_parser("import") - import_command.add_argument("sources", nargs="+", type=Path) + import_command.add_argument("sources", nargs="+", type=Path, help="source paths relative to --root, or absolute paths inside it") import_command.add_argument("--apply", action="store_true") import_command.set_defaults(handler=command_import) plan = commands.add_parser("plan") @@ -362,6 +410,8 @@ def parser() -> argparse.ArgumentParser: diff.set_defaults(handler=command_diff) doctor = commands.add_parser("doctor") doctor.set_defaults(handler=command_doctor) + inspect = commands.add_parser("inspect", help="validated state, drift preview and eligible tasks; read-only") + inspect.set_defaults(handler=command_inspect) task = commands.add_parser("task") task_commands = task.add_subparsers(dest="task_command", required=True) task_update = task_commands.add_parser("update") @@ -386,6 +436,9 @@ def parser() -> argparse.ArgumentParser: def main() -> int: args = parser().parse_args() try: + if getattr(args, "apply", False): + with graph_write_lock(root_path(args)): + return args.handler(args) return args.handler(args) except (OSError, ValueError, json.JSONDecodeError) as exc: if args.json: diff --git a/plugins/wigtn-plugins-with-codex/scripts/workgraph_core.py b/plugins/wigtn-plugins-with-codex/scripts/workgraph_core.py index 0a85f81..18adef7 100755 --- a/plugins/wigtn-plugins-with-codex/scripts/workgraph_core.py +++ b/plugins/wigtn-plugins-with-codex/scripts/workgraph_core.py @@ -3,6 +3,7 @@ from __future__ import annotations +from contextlib import contextmanager from copy import deepcopy from hashlib import sha256 import json @@ -14,7 +15,7 @@ SCHEMA_VERSION = "1.0" -REQUIREMENT_ID = re.compile(r"^[A-Z][A-Z0-9_-]*-[0-9]{2,}$") +REQUIREMENT_ID = re.compile(r"^[A-Z][A-Z0-9_]*(?:-[A-Z0-9_]+)+$") SOURCE_ID = re.compile(r"^SRC-[A-Z0-9_-]+$") ARTIFACT_ID = re.compile(r"^ART-[A-Z0-9_-]+$") TASK_ID = re.compile(r"^TASK-[A-Z0-9_-]+$") @@ -121,6 +122,40 @@ def atomic_write_json(path: Path, document: dict[str, Any]) -> None: raise +@contextmanager +def graph_write_lock(root: Path): + """Serialize cooperating CLI transactions, including their initial read. + + Keep the lock inode in place: unlinking it could split waiting writers + across different locks. OS locks are released if a writer process dies. + Direct editors do not participate in this advisory lock. + """ + directory = root / ".wigtn" + directory.mkdir(parents=True, exist_ok=True) + with (directory / ".write.lock").open("a+b") as handle: + if os.name == "nt": + import msvcrt + + if handle.seek(0, os.SEEK_END) == 0: + handle.write(b"\0") + handle.flush() + handle.seek(0) + msvcrt.locking(handle.fileno(), msvcrt.LK_LOCK, 1) + try: + yield + finally: + handle.seek(0) + msvcrt.locking(handle.fileno(), msvcrt.LK_UNLCK, 1) + else: + import fcntl + + fcntl.flock(handle.fileno(), fcntl.LOCK_EX) + try: + yield + finally: + fcntl.flock(handle.fileno(), fcntl.LOCK_UN) + + class WorkGraphValidator: def __init__(self) -> None: self.errors: list[str] = [] diff --git a/plugins/wigtn-plugins-with-codex/skills/acceptance-verifier/SKILL.md b/plugins/wigtn-plugins-with-codex/skills/acceptance-verifier/SKILL.md index d0df97f..ce585e0 100644 --- a/plugins/wigtn-plugins-with-codex/skills/acceptance-verifier/SKILL.md +++ b/plugins/wigtn-plugins-with-codex/skills/acceptance-verifier/SKILL.md @@ -1,43 +1,37 @@ --- name: acceptance-verifier -description: Verify PRD requirements or acceptance criteria against code and executed tests. Use for “요구사항 반영됐는지 검증”, PRD coverage, or acceptance verification. Do not use for general code review or when no requirements can be identified. +description: Verify requirements against inspectable code and tests with canonical evidence statuses. Use for PRD coverage or acceptance verification. For evidence-only chat without inspectable code or tests, answer directly without invoking. Do not use for general code review or when no requirements can be identified. --- # Acceptance Verifier -Produce a read-only requirement matrix unless the user also asks to fix gaps. - -## Workflow - -1. Locate the authoritative PRD, acceptance criteria, issue, or user-provided - requirements. Treat WIGTN, Spec Kit, OpenSpec, and BMAD documents as inputs, - not workflows that must be replayed. -2. Identify the requested comparison: working tree, commit, branch, PR, or - named files. If `.wigtn/project.json` or `.wigtn/workgraph.json` exists, - validate it before trusting sources or status; source drift is a gap. -3. Preserve stable requirement IDs. If none exist, create temporary `AC-01` - IDs and label them local to the report. -4. Inspect implementation and tests. Run the smallest relevant - repository-defined checks when authorized and feasible. -5. Read [evidence matrix](references/evidence-matrix.md), then assign exactly - one canonical status per requirement. Cite precise code lines, exact - commands, exits, and relevant test names. Never infer that an unexecuted or - irrelevant check passed. -6. Put findings outside the requirement set under `Out-of-scope findings`. +Compare authoritative requirements with code and executed checks; fix gaps only +when requested. Preserve source IDs and the user's output schema. + +## Evidence + +- Inspect relevant implementation and run proportionate checks. Cite code + locations, commands, outcomes, and what remains unobserved. +- Use one canonical status per requirement: `verified`, + `implemented-not-executed`, `partially-verified`, `not-satisfied`, + `not-verifiable`, or `not-applicable`. +- A demonstrated requirement violation is `not-satisfied`, even if other + examples pass. `partially-verified` requires a demonstrated subclaim, + unresolved subclaims, and no demonstrated violation. Missing evidence alone + is not a failure. Read [evidence matrix](references/evidence-matrix.md) for + compound claims, contradictory/flaky checks, or external actions. +- If saved WorkGraph state is relevant, run + `python3 ../../scripts/wigtn.py --root --json inspect` + from this skill directory. It validates saved artifacts and previews drift + without mutation. Source changes invalidate old completion evidence. ## Output -| Requirement | Status | Code evidence | Test evidence | Gap | -|---|---|---|---|---| - -Use the canonical status value in every row. For insufficient evidence, write -`not-verifiable`; a localized label may follow but must not replace it. A -missing test does not automatically mean the requirement failed. - -Do not create state for an ordinary verification answer. For a requested saved -artifact, existing `.wigtn/evidence.json`, cross-session handoff, or explicit -Spec Kit/OpenSpec/BMAD import, read [saved evidence](references/saved-evidence.md). +Return the requested report with requirement, status, evidence, and gap. +Keep unrelated findings separate. Use a file-edit/patch tool when saving +reports with command examples to avoid nested shell heredoc collisions. -Finish with executed commands, limitations, prioritized gaps, and any saved -artifact validation result. Do not change a WorkGraph task to `verified` -unless its current linked check passed and a valid evidence reference exists. +Custom JSON needs no WIGTN schema. Read +[saved evidence](references/saved-evidence.md) only for a requested WIGTN +handoff or WorkGraph verification. A task becomes `verified` only with its +current linked passing check and a valid evidence reference. diff --git a/plugins/wigtn-plugins-with-codex/skills/acceptance-verifier/references/evidence-matrix.md b/plugins/wigtn-plugins-with-codex/skills/acceptance-verifier/references/evidence-matrix.md index 8ef2314..41984a6 100644 --- a/plugins/wigtn-plugins-with-codex/skills/acceptance-verifier/references/evidence-matrix.md +++ b/plugins/wigtn-plugins-with-codex/skills/acceptance-verifier/references/evidence-matrix.md @@ -3,8 +3,8 @@ - `verified`: all material behavior is present and a relevant executed check passes. - `implemented-not-executed`: implementation evidence exists, but no relevant executable check ran successfully. -- `partially-verified`: a material subclaim is demonstrated but another - material subclaim fails or remains unverified. +- `partially-verified`: a material subclaim is demonstrated and another remains + unverified, with no demonstrated violation of the requirement. - `not-satisfied`: evidence shows required behavior is absent or contradictory. - `not-verifiable`: available artifacts cannot support a reliable conclusion. - `not-applicable`: the requirement is outside the evaluated scope. @@ -17,8 +17,10 @@ Before referencing a passing check, audit its relevance: requirement, not a legacy module or unrelated assertion - treat a check that asserts behavior contradicting the authoritative requirement as stale evidence, even when it passes -- for state-, order-, random-, or time-dependent behavior, rerun the check at - least three times; any inconsistent result blocks `verified` +- exercise relevant state transitions, ordering and boundaries. Repeat checks + when intermittency or timing uncertainty warrants it; three identical passes + do not establish coverage. Investigate any inconsistent result before + claiming `verified` - a focused direct runtime assertion may verify a small observable requirement when no test file exists, but record its exact command and cases - repository evidence can verify that an external action was queued; it cannot @@ -29,12 +31,13 @@ Before choosing a status, split compound language into observable subclaims Record which subclaims passed, failed, or could not be observed. Then collapse them to one requirement status: -- use `partially-verified` only when at least one material subclaim has direct - implementation or runtime evidence and another material subclaim fails or - cannot be observed -- use `not-satisfied` when the central named behavior is absent or - contradictory; a degenerate input that never exercises that behavior is not - a material subset +- first use `not-satisfied` if evidence demonstrates a required behavior is + absent or violated; passing other cases cannot soften that conclusion +- use `partially-verified` when a material subclaim has direct implementation + or runtime evidence, another cannot be observed, and no required behavior + has been demonstrated false +- accepting allowed inputs does not partially establish a missing rejection + constraint; the required denial or rejection is still violated - use `not-verifiable` when no material subclaim can be concluded from available artifacts or permitted checks - if an unreliable test fails intermittently but an independent focused diff --git a/plugins/wigtn-plugins-with-codex/skills/acceptance-verifier/references/saved-evidence.md b/plugins/wigtn-plugins-with-codex/skills/acceptance-verifier/references/saved-evidence.md index 96f7633..599480a 100644 --- a/plugins/wigtn-plugins-with-codex/skills/acceptance-verifier/references/saved-evidence.md +++ b/plugins/wigtn-plugins-with-codex/skills/acceptance-verifier/references/saved-evidence.md @@ -1,7 +1,8 @@ # Saved acceptance evidence -Use only for an explicitly requested machine-readable artifact, an existing -`.wigtn/evidence.json`, or a cross-session handoff. First read the shared +Use for WIGTN-format evidence or WorkGraph verification. A user-defined +report schema is sufficient when no WIGTN state is requested or present. +For WIGTN evidence, read the shared [Evidence Contract](../../../references/evidence-contract.md). For a new imported handoff, run from the target repository root: diff --git a/plugins/wigtn-plugins-with-codex/skills/design-direction/SKILL.md b/plugins/wigtn-plugins-with-codex/skills/design-direction/SKILL.md index a5b7357..d78d783 100644 --- a/plugins/wigtn-plugins-with-codex/skills/design-direction/SKILL.md +++ b/plugins/wigtn-plugins-with-codex/skills/design-direction/SKILL.md @@ -11,7 +11,7 @@ Prefer the product’s existing visual language over a generic style preset. 1. Inspect tokens, global styles, fonts, representative pages, shared components, spacing, icons, and motion. 2. If a coherent system exists, summarize it and produce an implementation contract that extends it. Do not offer unrelated styles. -3. For greenfield work or an explicit redesign, present two or three genuinely distinct directions with tradeoffs. Let the user choose when the choice materially changes the product. +3. For greenfield work or an explicit redesign, use the stated preferences to choose a direction and proceed. Offer alternatives when the user asks to compare them or a missing choice materially changes the product; continue independent authorized work while awaiting that choice. 4. Read only the selected reference from `references/styles/`. 5. Produce a short contract covering typography, palette roles, spacing rhythm, surfaces, borders, interaction states, motion, accessibility, and anti-patterns. 6. Do not implement unless requested. diff --git a/plugins/wigtn-plugins-with-codex/skills/handdrawn-diagram/SKILL.md b/plugins/wigtn-plugins-with-codex/skills/handdrawn-diagram/SKILL.md index 224a27b..92cb61d 100644 --- a/plugins/wigtn-plugins-with-codex/skills/handdrawn-diagram/SKILL.md +++ b/plugins/wigtn-plugins-with-codex/skills/handdrawn-diagram/SKILL.md @@ -11,7 +11,7 @@ Create a legible diagram whose text survives rendering, including Korean and mix 1. Confirm the system boundary, groups, flow direction, and output location from context. Keep the smallest useful node set. 2. Author Mermaid with `look: handDrawn`, quoted labels, concise accessible title and description, and semantic colors. -3. Prefer a repository-installed or already available Mermaid CLI and record `mmdc --version`. Never run a floating `npx -y` download. If no renderer exists, request approval before any exact-version network installation. +3. Prefer a repository-installed or already available Mermaid CLI and record `mmdc --version`. Never run a floating `npx -y` download. If no renderer exists, use existing authorization for an exact-version installation; ask only if that installation is outside the authorized scope. 4. Render both SVG and PNG using the [render guide](references/rendering.md). 5. Run `python3 scripts/verify-artifacts.py ` from this skill directory. 6. Inspect the PNG visually. Check clipped Korean/English labels, overlaps, contrast, arrow direction, and group meaning. Revise and rerender until legible. diff --git a/plugins/wigtn-plugins-with-codex/skills/handdrawn-diagram/references/rendering.md b/plugins/wigtn-plugins-with-codex/skills/handdrawn-diagram/references/rendering.md index 734db74..f74d658 100644 --- a/plugins/wigtn-plugins-with-codex/skills/handdrawn-diagram/references/rendering.md +++ b/plugins/wigtn-plugins-with-codex/skills/handdrawn-diagram/references/rendering.md @@ -3,8 +3,8 @@ Use the renderer already selected by the repository. Check `command -v mmdc` and `./node_modules/.bin/mmdc`, then record `mmdc --version`. Do not change a project manifest or download a floating package version. If no renderer exists, -explain the missing dependency and obtain approval before an exact-version -network installation. +explain the missing dependency. Use existing authorization for an exact-version +installation; ask only when it is outside the authorized scope. Start from this accessible source shape: diff --git a/plugins/wigtn-plugins-with-codex/skills/product-spec/SKILL.md b/plugins/wigtn-plugins-with-codex/skills/product-spec/SKILL.md index b8c1fe6..b145fc4 100644 --- a/plugins/wigtn-plugins-with-codex/skills/product-spec/SKILL.md +++ b/plugins/wigtn-plugins-with-codex/skills/product-spec/SKILL.md @@ -1,6 +1,6 @@ --- name: product-spec -description: Create, review, or deep-dive an implementation-ready PRD or product spec. Use for “PRD 뽑아줘”, “PRD 검토해줘”, “PRD 디깅해줘”, requirements, acceptance criteria, and spec feasibility or contradiction review. Do not use for brainstorming without an artifact, implementation, minor fixes, or general code review. +description: Create, substantively review, or deep-dive an implementation-ready PRD or product spec. Use for explicit PRD creation, requirements or acceptance-criteria audits, and spec feasibility, contradiction, or coverage review. Do not use for proofreading, copyediting, reformatting, renaming, link repair, minor fixes, brainstorming without an artifact, implementation, or general code review. --- # Product Spec @@ -10,16 +10,34 @@ work a gate for ordinary coding. ## Mode -- If `.wigtn/project.json` exists, validate it and read - [project context](../../references/project-context.md). An explicit user - profile overrides `prd_profile`; missing context changes nothing. -- **Create:** read [create contract](references/create-contract.md). Use Compact - by default. Use Full only when the user requests it or the product actually - needs multiple route/state contracts, a multi-step lifecycle, evidenced - NFRs, phased delivery, or migration planning. Mark Full conditional - contracts `Required` or `N/A` with evidence. +- Inspect `.wigtn/project.json` only when working from a user-provided + repository, saving an artifact, or asked for project-native behavior. Do not + probe the filesystem for a chat-only brief. When present, validate it and + read [project context](../../references/project-context.md). An explicit + user profile overrides `prd_profile`. +- **Create:** use Compact by default with this exact shape: + + ```markdown + + # PRD + ## Problem and scope + ## Goals and non-goals + ## Users, roles, authorization, and data boundaries + ## Functional requirements + | ID | Requirement | Priority | + ## Acceptance criteria + | ID | Requirement | Given | When | Then | Verification | + ## Assumptions and open decisions + ## Release condition + | Requirement IDs | Verifiable exit condition | + ``` + + Read the [Full create contract](references/create-contract.md) only when the + user requests Full or the product actually needs multiple route/state + contracts, a multi-step lifecycle, evidenced NFRs, phased delivery, or + migration planning. - **Review:** read [review contract](references/review-contract.md). Emit its - contract-audit table, then at most five material findings. Do not rewrite + profile-appropriate contract audit, then material findings by impact. Do not rewrite unless asked. - **Deep dive:** read the review contract and [deep-dive guide](references/deep-dive.md). Inspect repository evidence when @@ -36,7 +54,9 @@ work a gate for ordinary coding. speculative policy, and exhaustive low-impact edge-case catalogs. - Respect an explicit request for a concise artifact. Use Compact rather than shrinking a Full artifact cosmetically. Compact allows no more than eight - material FRs and ten acceptance criteria. + material FRs and ten acceptance criteria. If the requested scope exceeds + that budget, use Full and keep its prose concise; never drop requirements + to fit Compact. - Do not promote plausible product choices—identity matching, token rotation, retry policy, route shape, or similar—into requirements. Keep unsupported choices as compact open decisions. @@ -46,9 +66,9 @@ work a gate for ordinary coding. - In reviews, `Present` means the required artifact exists, not that it is flawless. Report defects in that artifact as findings; do not relabel it `Missing`. -- Omit low/nit findings. Group related medium findings and return no more than - five material findings, ordered by impact with exact section or requirement - IDs. +- Omit low/nit findings unless requested. Group related findings and lead with + the most consequential ones, citing exact sections or requirement IDs. Do + not hide material defects to meet a fixed finding count. - After saving a PRD, run `python3 scripts/validate-prd.py ` from this skill directory. The validator reads the profile marker. Report failures; never weaken the contract to make it pass. diff --git a/plugins/wigtn-plugins-with-codex/skills/product-spec/references/create-contract.md b/plugins/wigtn-plugins-with-codex/skills/product-spec/references/create-contract.md index 1ab1a56..725cb55 100644 --- a/plugins/wigtn-plugins-with-codex/skills/product-spec/references/create-contract.md +++ b/plugins/wigtn-plugins-with-codex/skills/product-spec/references/create-contract.md @@ -1,32 +1,6 @@ -# PRD Create Contract +# Full PRD Create Contract -Choose one profile. Never emit both. - -## Compact - -Use by default, including concise, brief, and MVP requests. Start with the -exact marker so the validator applies the smaller contract: - -```markdown - -# PRD -## Problem and scope -## Goals and non-goals -## Users, roles, authorization, and data boundaries -## Functional requirements -| ID | Requirement | Priority | -## Acceptance criteria -| ID | Requirement | Given | When | Then | Verification | -## Assumptions and open decisions -## Release condition -| Requirement IDs | Verifiable exit condition | -``` - -Use at most eight material FRs and ten ACs. Do not add applicability, page, -state-matrix, flow, risk, or phased-delivery sections unless the user switches -to Full. A Compact PRD is smaller, not less testable. - -## Full +Never combine this profile with Compact. Use only when the user requests full detail or the product actually needs multiple route/state contracts, a multi-step lifecycle, evidenced NFRs, @@ -83,10 +57,5 @@ Conditional: the metric, owner, and decision point. Describe necessary behavior, not a preferred implementation unless the -constraint is real. Avoid duplicate FRs and ACs, speculative enterprise policy, -exhaustive edge-case catalogs, and open decisions that do not affect -implementation or release. - -A plausible policy is not evidence: move unprovided identity, token, retry, -route, or expiry choices to one-line open decisions instead of silently -adopting them. +constraint is real. Keep unsupported identity, token, retry, route, or expiry +choices as open decisions. diff --git a/plugins/wigtn-plugins-with-codex/skills/product-spec/references/review-contract.md b/plugins/wigtn-plugins-with-codex/skills/product-spec/references/review-contract.md index 0e564c5..fc80a51 100644 --- a/plugins/wigtn-plugins-with-codex/skills/product-spec/references/review-contract.md +++ b/plugins/wigtn-plugins-with-codex/skills/product-spec/references/review-contract.md @@ -1,6 +1,10 @@ # PRD Review Contract -Emit this table before semantic findings. Use only `Present`, `N/A`, or +First identify the declared Compact/Full profile and the user’s requested +review scope. Review Compact against its own sections, requirement IDs and +acceptance criteria; do not require Full-only artifacts. For an external PRD, +review the supplied contract unless the user asks for WIGTN conformance. +For a Full WIGTN contract audit, emit the following table before semantic findings. Use only `Present`, `N/A`, or `Missing`; `N/A` requires evidence from the PRD. | Contract | Status | Evidence | @@ -34,6 +38,6 @@ Then review universal quality: contradictions, authorization/data boundaries, state transitions, failure/recovery, unverifiable acceptance, unsupported scope, migration, operations, privacy, and security only where relevant. -Return at most five `blocker`, `high`, or grouped `medium` findings. Omit -low/nit findings, cite exact sections or IDs, and do not use numeric quality -scores. +Report all material `blocker` and `high` findings; group related `medium` +findings. Omit low/nit findings unless requested, cite exact sections or IDs, +and do not use numeric quality scores. diff --git a/plugins/wigtn-plugins-with-codex/skills/release-readiness/SKILL.md b/plugins/wigtn-plugins-with-codex/skills/release-readiness/SKILL.md index 45003c3..2b4325c 100644 --- a/plugins/wigtn-plugins-with-codex/skills/release-readiness/SKILL.md +++ b/plugins/wigtn-plugins-with-codex/skills/release-readiness/SKILL.md @@ -27,10 +27,13 @@ When wording is ambiguous about a consequential mutation, stop before that mutat status. A ready gate is prerequisite evidence, not permission for Git or remote actions. 1. Read repository instructions. Run - `python3 ../../scripts/inspect-release-state.py ` from this skill - directory to capture branch, upstream, operations, conflicts, staged, - unstaged, and untracked paths without mutation. Then inspect the relevant - staged and unstaged diffs. + `python3 ../../scripts/inspect-release-state.py --include-diffs` + from this skill directory to capture branch, upstream, operations, + conflicts, paths, bounded staged/unstaged patches, and whitespace checks in + one read-only call. Do not read untracked file contents automatically. If a + patch is marked `truncated`, inspect only the relevant paths needed for the + requested scope; otherwise use the collected patches instead of repeating + Git diff commands. 2. Separate task changes from pre-existing or unrelated user work. Never silently include unrelated files. 3. Review for correctness, regression, security, and missing tests. Findings need severity, confidence, file/line, and impact. 4. Run relevant repository-defined verification. Record exact commands and results. diff --git a/plugins/wigtn-plugins-with-codex/skills/release-readiness/references/git-safety.md b/plugins/wigtn-plugins-with-codex/skills/release-readiness/references/git-safety.md index 020b7ac..510d83b 100644 --- a/plugins/wigtn-plugins-with-codex/skills/release-readiness/references/git-safety.md +++ b/plugins/wigtn-plugins-with-codex/skills/release-readiness/references/git-safety.md @@ -4,5 +4,5 @@ - Stage explicit paths after reviewing their diff; avoid broad staging when unrelated files exist. - Never discard, overwrite, amend, rebase, force-push, or delete without explicit authority. - Do not bypass hooks unless the user asks and understands the consequence. -- Before push, confirm branch and upstream. Before PR creation, confirm base branch and summarize the actual diff. +- Before push, inspect branch and upstream. Before PR creation, inspect the base branch and summarize the actual diff. Ask only when a consequential target remains ambiguous after inspecting the request and repository. - If identity, authentication, conflicts, or protections block the requested action, report the exact blocker without destructive recovery. diff --git a/plugins/wigtn-plugins-with-codex/skills/screen-spec/SKILL.md b/plugins/wigtn-plugins-with-codex/skills/screen-spec/SKILL.md index abc1b0c..318b7a3 100644 --- a/plugins/wigtn-plugins-with-codex/skills/screen-spec/SKILL.md +++ b/plugins/wigtn-plugins-with-codex/skills/screen-spec/SKILL.md @@ -35,16 +35,24 @@ Do not produce or propose the other artifacts merely because the skill loaded. 3. Read only the selected templates in `assets/templates/` and their dependency templates. Treat placeholders as shape, never as product facts. 4. Keep page IDs, roles, routes, states, requirements, and wireframe anchors - consistent across the artifacts that exist. + consistent across the artifacts that exist. Before validation, ensure every + `FR-`, `REQ-`, or `AC-` ID used by screen or handoff also appears in IA. 5. For screen, wireframe, or handoff, read the compact [state contract](references/state-contract.md). For handoff, also read the [handoff contract](references/handoff-contract.md). 6. Keep wireframes grayscale with semantic status colors and one self-contained responsive HTML file. If browser control is available, inspect wide and narrow layouts, overflow, labels, and links. -7. Run from this skill directory: +7. Before validation, replace every template token written as `{...}`. Use + colon-prefixed route parameters such as `:organizationId` or `:email` + instead of braces so real routes are not mistaken for template residue. + Run once from this skill directory: - `python3 ../../scripts/validate-screen-spec.py --artifacts ` + `python3 ../../scripts/validate-screen-spec.py --artifacts ia,screen --json` + + Replace `ia,screen` with the comma- or space-separated selected closure, or + `all`. Use the documented command directly; consult `--help` if the installed + interface differs. Rerun after a relevant correction or new evidence. 8. Return file links and deterministic plus visual results without pasting the full bundle into chat. diff --git a/plugins/wigtn-plugins-with-codex/skills/verified-delivery/SKILL.md b/plugins/wigtn-plugins-with-codex/skills/verified-delivery/SKILL.md index e93caf6..077a5b4 100644 --- a/plugins/wigtn-plugins-with-codex/skills/verified-delivery/SKILL.md +++ b/plugins/wigtn-plugins-with-codex/skills/verified-delivery/SKILL.md @@ -5,48 +5,28 @@ description: Implement and verify with proportional evidence only when explicitl # Verified Delivery -This workflow is explicit-only. Invocation authorizes in-scope implementation -and proportionate local verification, not Git, network, deployment, dependency -installation, or unrelated external mutations. +Implement the requested scope and retain proportionate execution evidence. +Use the repository's implementation and testing conventions. Choose the +approach rather than replaying a fixed development sequence. ## Workflow -1. Read repository instructions, the requested behavior, adjacent code, and - existing tests. If saved WIGTN state exists, validate it before trusting it. -2. Define observable completion and the verification boundary. Use a compact coverage census - only when the request names multiple material requirements, files, symbols, - or interfaces. Name high-risk invariants only when auth, tenancy, secrets, migration, persistence, concurrency, - public schemas, or compatibility are actually involved. -3. Implement the smallest coherent change using repository-native patterns. - Preserve unrelated edits. Do not inspect or copy another checkout, an - installed distribution of the same project, a package cache, benchmark - reference patch, hidden test, or gold implementation. -4. Run the smallest repository-native checks justified by the blast radius. - Prefer a focused pre-existing check, then only broader checks that add - evidence. Do not duplicate a passing repository oracle with an alternate - harness. A new focused test is strong evidence only when its pre-change - failure and post-change pass were both observed. -5. Review the final diff, changed interfaces, unexpected paths, debug - artifacts, and whether checks exercise the requested failure mode. -6. For a small request, report changed behavior, exact checks, and the - verification boundary. Read [delivery evidence](references/delivery-evidence.md) - only for multiple material requirements, authored-test provenance, - benchmark/evaluator work, or a requested saved evidence artifact. - -## Evidence and state - -- Passing visible checks do not prove unspecified APIs, schemas, hidden tests, - external state, or unobserved compatibility. -- If source leakage occurs, mark the affected result `not-verifiable` even when - an evaluator passes. -- Do not create stable IDs, WorkGraph state, or evidence JSON by default. Read - [the shared evidence contract](../../references/evidence-contract.md) only - for a requested saved artifact, an existing handoff, or cross-session work. -- A saved WorkGraph task remains `implemented` until its current linked check - passes and a valid evidence reference exists. - -## Authority boundary - -Do not commit, push, open a PR or issue, deploy, install dependencies, or alter -remote state unless the user separately asks for that action. Never use -destructive rollback to discard a mixed dirty worktree. +- Use a compact coverage census only when the request names multiple material requirements + or interfaces. Track relevant invariants for auth, tenancy, secrets, migration, persistence, concurrency + and compatibility when involved. +- Do not duplicate a passing repository oracle without a coverage gap. + Investigate failures before choosing whether code, tests, or the test command + is wrong. A suggested verification command is evidence, not an exclusive rule. +- Read [delivery evidence](references/delivery-evidence.md) for compound claims + or uncertain check provenance. For benchmark or independent evaluation work, + read its isolation rules before implementation. +- Finish with changed behavior, relevant checks and results, and material + verification gaps. Do not add a matrix when prose expresses the coverage. + +## Saved state + +Do not create stable IDs, WorkGraph state, or evidence JSON by default. +For requested WIGTN handoff or existing saved state, read +[shared evidence contract](../../references/evidence-contract.md). +Saved task verification requires a current linked passing check and evidence. +Preserve unrelated work and reuse the user's existing authorization. diff --git a/plugins/wigtn-plugins-with-codex/skills/verified-delivery/references/delivery-evidence.md b/plugins/wigtn-plugins-with-codex/skills/verified-delivery/references/delivery-evidence.md index 3274ec6..f1ff952 100644 --- a/plugins/wigtn-plugins-with-codex/skills/verified-delivery/references/delivery-evidence.md +++ b/plugins/wigtn-plugins-with-codex/skills/verified-delivery/references/delivery-evidence.md @@ -1,63 +1,24 @@ # Delivery Evidence -For each material requirement, retain at least one precise code reference and -one verification result when executable verification exists. Record commands -exactly, including failures, reruns, and skipped checks. Never replace evidence -with a quality score. - -Classify each check by provenance: - -- `external`: benchmark, CI, or evaluator kept outside the agent worktree -- `repository`: pre-existing repository test or executable contract -- `user-provided`: fixture or oracle supplied by the user -- `agent-authored-red-green`: a focused test observed failing before the - implementation and passing afterward -- `agent-authored-green-only`: an inline script or test observed only after the - implementation - -`verified` requires at least one relevant check from the first four classes. -`agent-authored-green-only` can support implementation confidence but cannot, -by itself, close an acceptance criterion. A passing unrelated repository test -does not upgrade an uncovered requirement. - -Reference implementation isolation is part of evidence validity. If the run -reads or diffs an installed copy, another checkout, a package cache, benchmark -gold patch, or hidden test for the same project, label the affected result -`not-verifiable` and disclose the leakage. A clean evaluator pass does not -repair contaminated implementation provenance. - -Use exactly one status: - -- `verified`: implementation evidence plus a passing relevant check -- `implemented-not-executed`: code evidence exists but no relevant check ran -- `partially-verified`: only part of the observable criterion is supported -- `not-satisfied`: required behavior is absent or contradicted -- `not-verifiable`: required evidence is inaccessible or inherently external -- `not-applicable`: the requirement is outside the evaluated scope - -Use the same canonical values for conversational output and a requested saved -handoff. A saved artifact is optional workflow output, not hidden state and not -a prerequisite for ordinary implementation. - -For a small task with one material requirement, use a short completion -summary: changed behavior, exact checks and exits, blockers, and residual risk. -Do not manufacture a matrix for a one-line fix. - -For multiple material requirements, use: - -| Requirement | Status | Implementation evidence | Executed evidence | -|---|---|---|---| - -When several public symbols or interfaces are named, include every -unimplemented, deferred, or uninspected item in the requirement rows or -verification boundary. Do not infer whole-feature coverage from one passing -submodule. - -Then list changed behaviour, exact commands and exits, residual risks, unrelated -dirty state preserved, and external actions not performed. If all deterministic -checks already passed before the treatment, do not claim that this workflow -improved code quality; report only the evidence and safety it actually added. -Always state the verification boundary: which interfaces and behaviors were -exercised, which were inferred, and whether hidden or external checks were -available. Unspecified public API names or schemas remain gaps even when the -implemented behavior appears correct. +Cite the relevant implementation and executed checks. Distinguish pre-existing +repository checks, external/user-provided checks, and agent-authored checks. +For new checks, report whether a pre-change failure was actually observed; +a green-only check must not be described as red/green evidence. Judge coverage +by what was exercised, not by who wrote the check. Passing an unrelated suite +cannot establish an untested behavior. + +Use the [shared canonical status definitions](../../../references/evidence-contract.md) +when statuses are needed. A demonstrated required-behavior violation is +`not-satisfied`; passing other subclaims cannot dilute it. External delivery +needs external observation. Code or queue creation alone does not prove it. + +For benchmark and independent evaluation work, do not read the tested project's +reference implementation, gold patch, hidden tests, installed copy, or another +checkout. Such leakage makes the affected result `not-verifiable`. +A clean evaluator pass does not repair contaminated provenance. Ordinary work +may inspect authorized dependencies and user-designated checkouts. + +Do not manufacture a matrix for a one-line fix. For larger work, cover every +material requirement in the requested format and identify unimplemented, +deferred, or uninspected interfaces. Report relevant failed/skipped checks and +uncertainty without inventing missing API or product requirements. diff --git a/plugins/wigtn-plugins-with-codex/skills/work-planner/SKILL.md b/plugins/wigtn-plugins-with-codex/skills/work-planner/SKILL.md index 0d2e429..28a90e5 100644 --- a/plugins/wigtn-plugins-with-codex/skills/work-planner/SKILL.md +++ b/plugins/wigtn-plugins-with-codex/skills/work-planner/SKILL.md @@ -5,52 +5,35 @@ description: Save requirements as a resumable dependency-aware WIGTN WorkGraph w # Work Planner -Create an implementation plan whose IDs, dependencies, checks, and source -freshness can be validated. Planning does not authorize implementation or Git -actions. +Save a resumable plan whose requirement IDs, dependencies, executable checks, +and source freshness can be validated. Use the smallest useful graph; ordinary +conversation does not need lifecycle files. ## Workflow -1. Read repository instructions and the authoritative PRD, issue, screen spec, - or acceptance criteria. Preserve existing requirement IDs. -2. If `.wigtn/project.json` exists, validate it. If `.wigtn/workgraph.json` - exists, run `wigtn.py doctor` and `wigtn.py diff --check` before trusting its - status. -3. For a requested persistent or cross-session plan, show the `init` or - `import` dry-run first, then apply it. Do not create lifecycle state for a - conversational plan unless the user requests a saved plan or continuing - workflow. -4. Create one task per independently verifiable change, not automatically one - task per file. Link every task to at least one requirement. Record intended - paths, protected paths, risk, executable checks, artifacts, and real - dependencies. Use `wigtn.py task update` and `wigtn.py task depend` with - dry-run then `--apply`; do not edit WorkGraph JSON directly when these - commands cover the change. -5. Keep tasks `draft` until their scope and checks are concrete. Mark `ready` - only when all dependencies are `verified`. Use `blocked` only with a - specific blocker. -6. Run the WorkGraph validator. Return the next unblocked task IDs, unresolved - planning gaps, and validation result. - -Use [the WorkGraph contract](references/workgraph-contract.md) for state -semantics and commands. - -## Rules - -- Treat source drift as invalidation, not as a documentation warning. -- Never preserve `verified` after a linked requirement, artifact, dependency, - or check becomes stale. -- Do not mark a task `verified`; only executed evidence handled by - `acceptance-verifier` or an explicit delivery workflow may do that. -- Keep release authority outside task status. A ready release gate does not - authorize commit, push, pull request, or deploy. -- Prefer the smallest dependency graph that preserves real ordering. Do not add - ceremonial tasks or dependencies. -- Do not execute implementation, install dependencies, commit, push, or mutate - remote systems. - -## Completion - -Return the WorkGraph path when saved, its revision, created or changed task -IDs, next task IDs, source-drift status, and validator result. State explicitly -when commands or intended paths remain unspecified. +- Read authoritative requirements and preserve their IDs. For existing state, + use `python3 ../../scripts/wigtn.py --root --json inspect` + from this skill directory. It returns validated state, source drift, and + currently eligible tasks in one read-only call. Reinspect after relevant + changes or when the result is incomplete; do not repeat unchanged queries. +- Use [WorkGraph contract](references/workgraph-contract.md) for CLI commands + and state semantics. Mutations require `--apply`; preview when the scope or + effect is uncertain, rather than duplicating every already-decided command. +- Define tasks by independently verifiable outcomes. Link requirements, + dependencies, intended/protected paths, and relevant checks. Unknown test + commands remain unknown, not invented. A seed plan is editable, not a fixed + one-task-per-requirement prescription. +- Use CLI task mutations where supported; preserve concurrent changes with + `--expected-revision` when needed. Keep incomplete task definitions `draft`. + Only tasks whose dependencies are verified may be `ready`. + +## State and completion + +Source drift invalidates linked artifacts, tasks, checks, and release gates. +Do not preserve old `verified` claims or grant verification from a plan alone. +Executed checks and valid evidence are required; a ready release gate does +not grant Git or deployment permission. Continue other work already authorized. + +Return the saved path, revision, actionable next tasks, and unresolved gaps. +A successful mutation validates graph structure; inspect again when source +freshness or other saved artifacts may have changed. diff --git a/plugins/wigtn-plugins-with-codex/skills/work-planner/references/workgraph-contract.md b/plugins/wigtn-plugins-with-codex/skills/work-planner/references/workgraph-contract.md index ebfd9d1..e621cc1 100644 --- a/plugins/wigtn-plugins-with-codex/skills/work-planner/references/workgraph-contract.md +++ b/plugins/wigtn-plugins-with-codex/skills/work-planner/references/workgraph-contract.md @@ -39,6 +39,7 @@ python3 /scripts/wigtn.py --json import docs/prd.md python3 /scripts/wigtn.py --json import docs/prd.md --apply python3 /scripts/wigtn.py --json plan python3 /scripts/wigtn.py --json plan --apply +python3 /scripts/wigtn.py --json inspect python3 /scripts/wigtn.py --json status python3 /scripts/wigtn.py --json next python3 /scripts/wigtn.py --json diff --check @@ -54,10 +55,20 @@ python3 /scripts/wigtn.py --json task depend TASK-ID \ python3 /scripts/validate-workgraph.py .wigtn/workgraph.json ``` +`inspect` validates WorkGraph/project/evidence, reports a drift-adjusted view, +and lists eligible tasks without writing files or executing saved commands. +It does not prove evidence is truthful or tests still pass. On invalid/missing +artifacts it returns nonzero and withholds eligible tasks. `status`/`next` are +raw saved-state queries; use `inspect` when freshness matters. + Mutation commands are dry-run unless `--apply` is present. Repeating an unchanged import, plan, or drift application must not advance the revision. Use `--expected-revision ` on task mutations when another process could -change the graph. A mismatch fails instead of overwriting newer work. Adding an +change the graph. CLI writes hold an OS advisory lock across read, revision check, and save. +A mismatch fails instead of overwriting newer CLI work. Direct file editors +do not participate in that lock. The persistent `.wigtn/.write.lock` file +is coordination state, not a completion artifact; do not remove it while +writers may be active. Adding an unverified dependency automatically returns a `ready` task to `draft`. ## Planning quality diff --git a/scripts/check-codex-usage-missing.py b/scripts/check-codex-usage-missing.py new file mode 100644 index 0000000..c6f4ccf --- /dev/null +++ b/scripts/check-codex-usage-missing.py @@ -0,0 +1,41 @@ +import importlib.util,json,subprocess,sys,tempfile,unittest +from pathlib import Path +ROOT=Path(__file__).resolve().parents[1] +sys.path.insert(0,str(ROOT/'scripts')) +from codex_usage import read_events,api_equivalent_cost +class UsageMissing(unittest.TestCase): + def setUp(self):self.tmp=tempfile.TemporaryDirectory();self.path=Path(self.tmp.name)/'events.jsonl' + def tearDown(self):self.tmp.cleanup() + def write(self,*values):self.path.write_text('\n'.join(json.dumps({'type':'turn.completed','usage':v}) for v in values)+'\n') + def test_absent_optional_is_unknown(self): + self.write({'input_tokens':100,'output_tokens':10});u=read_events(self.path) + for f in ['cached_input_tokens','cache_write_input_tokens','reasoning_output_tokens','visible_output_tokens','uncached_input_tokens']:self.assertIsNone(u[f],f) + def test_explicit_zero_is_known(self): + self.write({'input_tokens':100,'output_tokens':10,'cached_input_tokens':0,'cache_write_input_tokens':0,'reasoning_output_tokens':0});u=read_events(self.path);self.assertEqual(u['visible_output_tokens'],10);self.assertEqual(u['uncached_input_tokens'],100) + def test_missing_one_turn_propagates(self): + full={'input_tokens':100,'output_tokens':10,'cached_input_tokens':20,'cache_write_input_tokens':0,'reasoning_output_tokens':2} + for values in [(full,{'input_tokens':100,'output_tokens':10}),({'input_tokens':100,'output_tokens':10},full)]: + self.write(*values);u=read_events(self.path);self.assertEqual(u['input_tokens'],200);self.assertIsNone(u['reasoning_output_tokens']);self.assertIsNone(u['uncached_input_tokens']) + def test_invalid_required_counters_rejected(self): + for value in [None,-1,True,'100',1.5]: + with self.subTest(value=value): + self.write({'input_tokens':value,'output_tokens':10}) + with self.assertRaises(ValueError):read_events(self.path) + def test_invalid_optional_counters_rejected(self): + for value in [-1,True,'10',1.5]: + with self.subTest(value=value): + self.write({'input_tokens':100,'output_tokens':10,'reasoning_output_tokens':value}) + with self.assertRaises(ValueError):read_events(self.path) + def test_impossible_breakdown_rejected(self): + for extras in [{'reasoning_output_tokens':11},{'cached_input_tokens':101},{'cached_input_tokens':60,'cache_write_input_tokens':50}]: + self.write({'input_tokens':100,'output_tokens':10,**extras}) + with self.assertRaises(ValueError):read_events(self.path) + def test_missing_cache_prevents_cache_adjusted_price(self): + self.write({'input_tokens':100,'output_tokens':10});u=read_events(self.path) + self.assertIsNone(api_equivalent_cost(u,input_per_million=4,cached_input_per_million=.4,cache_write_per_million=5,output_per_million=20)) + def test_cli_has_no_assumed_prices(self): + self.write({'input_tokens':100,'output_tokens':10});c=subprocess.run([sys.executable,'-B',str(ROOT/'scripts/codex_usage.py'),str(self.path)],capture_output=True,text=True);self.assertEqual(c.returncode,0,c.stderr);self.assertIsNone(json.loads(c.stdout)['api_equivalent_usd']) + def test_summary_preserves_unknown_without_crashing(self): + root=Path(self.tmp.name);p=root/'runs/candidate/acceptance.1.events.jsonl';p.parent.mkdir(parents=True);self.write({'input_tokens':100,'output_tokens':10});p.write_bytes(self.path.read_bytes()) + c=subprocess.run([sys.executable,'-B',str(ROOT/'scripts/summarize-token-efficiency.py'),str(root)],capture_output=True,text=True);self.assertEqual(c.returncode,0,c.stderr);d=json.loads((root/'TOKEN-EFFICIENCY.json').read_text());self.assertIsNone(d['arms']['candidate']['reasoning_output_tokens']);self.assertIn('unknown',(root/'TOKEN-EFFICIENCY.md').read_text()) +if __name__=='__main__':unittest.main(verbosity=2) diff --git a/scripts/check-codex-usage.py b/scripts/check-codex-usage.py new file mode 100755 index 0000000..5e89710 --- /dev/null +++ b/scripts/check-codex-usage.py @@ -0,0 +1,73 @@ +#!/usr/bin/env python3 +"""Regression checks for Codex JSONL usage parsing and cost math.""" + +from __future__ import annotations + +import json +from pathlib import Path +import tempfile +import subprocess +import sys + +from codex_usage import api_equivalent_cost, cold_api_equivalent_cost, read_events + + +def main() -> int: + with tempfile.TemporaryDirectory(prefix="wigtn-codex-usage-") as temporary: + path = Path(temporary) / "events.jsonl" + events = [ + {"type": "item.completed", "item": {"type": "command_execution"}}, + {"type": "item.completed", "item": {"type": "agent_message"}}, + { + "type": "turn.completed", + "usage": { + "input_tokens": 1000, + "cached_input_tokens": 200, + "cache_write_input_tokens": 100, + "output_tokens": 300, + "reasoning_output_tokens": 120, + }, + }, + ] + path.write_text("\n".join(json.dumps(event) for event in events) + "\n") + usage = read_events(path) + assert usage["uncached_input_tokens"] == 700 + assert usage["visible_output_tokens"] == 180 + assert usage["tool_items"] == 1 + cost = api_equivalent_cost( + usage, + input_per_million=4.0, + cached_input_per_million=0.4, + cache_write_per_million=5.0, + output_per_million=20.0, + ) + assert abs(cost - 0.00938) < 1e-12 + cold_cost = cold_api_equivalent_cost( + usage, input_per_million=4.0, output_per_million=20.0 + ) + assert abs(cold_cost - 0.01) < 1e-12 + with tempfile.TemporaryDirectory(prefix="wigtn-explicit-pricing-") as temporary: + root = Path(temporary) + events = root / "runs" / "candidate" / "prd.1.events.jsonl" + events.parent.mkdir(parents=True) + events.write_text(json.dumps({"type": "turn.completed", "usage": { + "input_tokens": 1000, "output_tokens": 100}}) + "\n") + summary = Path(__file__).with_name("summarize-token-efficiency.py") + full = ["--input-per-million", "4", "--cached-input-per-million", "0.4", + "--cache-write-per-million", "5", "--output-per-million", "20"] + for flags, expected in [([], 0), (["--input-per-million", "4"], 2), + (full, 0), (full[:-1] + ["nan"], 2)]: + result = subprocess.run([sys.executable, "-B", str(summary), str(root), *flags], + capture_output=True, text=True) + assert result.returncode == expected, result.stdout + result.stderr + if not flags: + data = json.loads((root / "TOKEN-EFFICIENCY.json").read_text()) + assert data["rates_usd_per_million"] is None + assert data["runs"][0]["api_equivalent_usd"] is None + print("Explicit pricing: PASS (unpriced/partial/explicit/nonfinite)") + print("Codex usage: PASS (JSONL fields + API-equivalent cost)") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/check-model-migration-eval.py b/scripts/check-model-migration-eval.py new file mode 100644 index 0000000..29750ed --- /dev/null +++ b/scripts/check-model-migration-eval.py @@ -0,0 +1,284 @@ +#!/usr/bin/env python3 +"""Offline evidence, accounting, adjudication and CLI-contract regression tests.""" +import hashlib +import importlib.util +import json +import os +import shutil +from pathlib import Path +import subprocess +import sys +import tempfile +import unittest + +from migration_evidence import seal, seal_run +from migration_cases import PROMPTS, RUBRIC + + +def load(name, filename): + spec = importlib.util.spec_from_file_location(name, Path(__file__).with_name(filename)) + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +scorer = load('migration_scorer', 'score-model-migration-eval.py') +runner = load('migration_runner', 'run-model-migration-eval.py') + + +def dump(path, value): + path.write_text(json.dumps(value) + '\n') + + +class EvidenceTests(unittest.TestCase): + def setUp(self): + self.temp = tempfile.TemporaryDirectory() + self.addCleanup(self.temp.cleanup) + self.root = Path(self.temp.name) + + def fixture(self, case='ordinary', failed=False, usage=None): + root = self.root + for name in ['evaluator', 'snapshots']: + (root / name).mkdir() + (root / name / 'source.txt').write_text('synthetic input fixture') + for arm in ['baseline', 'candidate']: + source = root/'snapshots'/arm/'plugins/wigtn-plugins-with-codex/skills/verified-delivery/SKILL.md' + source.parent.mkdir(parents=True) + source.write_text('# Verified Delivery\nSynthetic skill fixture.\n') + schedule = [{'arm': a, 'case': case, 'repeat': rep} for rep in [1, 2] + for a in ['bare', 'baseline', 'candidate']] + protocol = {'schema_version': 2, 'model': 'gpt-6-astra', 'effort': 'medium', + 'calls': len(schedule), 'schedule': schedule, + 'review_rubric': {case: ['correct', 'scoped']}, + 'activation_required': {case: 'verified-delivery'} if case == 'delivery' else {}} + dump(root / 'protocol.json', protocol) + seal(root, ['protocol.json', 'evaluator', 'snapshots'], 'input-manifest.json') + for i, item in enumerate(schedule): + run = root / 'runs' / f'{i:02}' + work = run / 'work'; work.mkdir(parents=True) + (work / 'scores.py').write_text('def sorted_scores(scores):\n return sorted(scores)\n') + (run / 'answer.md').write_text('Synthetic fixture, not model output') + (run / 'prompt.txt').write_text('Synthetic request') + fails = failed and i == 5 + value = usage if usage is not None else {'input_tokens': 100, 'output_tokens': 1000 if fails else 10} + events = [{'type': 'item.completed', 'item': {'type': 'command_execution', + 'command': 'cat /plugin/verified-delivery/SKILL.md', 'exit_code': 0, + 'aggregated_output': '# Verified Delivery\nSynthetic skill fixture.\n'}}, + {'type': 'turn.completed', 'usage': value}] + (run / 'events.jsonl').write_text('\n'.join(json.dumps(e) for e in events) + '\n') + dump(run / 'meta.json', {**item, 'model': protocol['model'], 'effort': 'medium', + 'exit_code': 0, 'duration_seconds': 1, 'oracle_before': {'exit_code': 1}, + 'oracle': {'exit_code': 1 if fails else 0}, 'draft_preserved': True, + 'unsolicited_state': False, 'unexpected_changes': []}) + seal_run(run) + return protocol + + def reviews(self): + scorer.score(self.root, write_template=True) + data = json.loads((self.root / 'reviews.template.json').read_text()) + for row in data['runs'].values(): + row['reviewer'] = 'synthetic-test-reviewer' + for decision in row['criteria'].values(): + decision.update(verdict='pass', evidence='Synthetic criterion evidence') + row['activation'] = {'status': 'confirmed', 'event_lines': [1], 'evidence': 'Synthetic tool-read trace'} + path = self.root / 'reviews.json'; dump(path, data) + return path + + def test_review_pending_is_not_pass(self): + self.fixture() + result = scorer.score(self.root) + self.assertTrue(result['integrity_passed']) + self.assertTrue(result['schedule_complete']) + self.assertFalse(result['passed']) + self.assertFalse(result['complete']) + + def test_adjudicated_packet_passes(self): + self.fixture() + self.assertTrue(scorer.score(self.root, self.reviews())['passed']) + + def test_early_stop_retains_observed_consumption(self): + self.fixture() + shutil.rmtree(self.root/'runs/05') + result = scorer.score(self.root) + self.assertTrue(result['integrity_passed']) + self.assertFalse(result['schedule_complete']) + candidate = next(g for g in result['groups'] if g['arm'] == 'candidate') + self.assertEqual(candidate['observed_output_tokens'], 10) + self.assertIsNone(candidate['all_attempt_output_tokens']) + + def test_changed_implementation_is_rejected(self): + self.fixture() + (self.root / 'runs/02/work/scores.py').write_text('def sorted_scores(scores): return []') + result = scorer.score(self.root) + self.assertFalse(result['integrity_passed']) + self.assertTrue(all(g['all_attempt_output_tokens'] is None for g in result['groups'])) + + def test_changed_events_rejected(self): + self.fixture() + (self.root / 'runs/02/events.jsonl').write_text('{}\n') + self.assertFalse(scorer.score(self.root)['integrity_passed']) + + def test_input_snapshot_mutation_rejected(self): + self.fixture() + (self.root / 'snapshots/source.txt').write_text('changed') + self.assertFalse(scorer.score(self.root)['integrity_passed']) + + def test_extra_work_file_rejected(self): + self.fixture() + (self.root / 'runs/00/work/unexpected.txt').write_text('extra') + self.assertFalse(scorer.score(self.root)['integrity_passed']) + + def test_missing_seal_rejected(self): + self.fixture() + (self.root / 'runs/00/evidence-manifest.json').unlink() + self.assertFalse(scorer.score(self.root)['integrity_passed']) + + def test_failure_cost_retained(self): + self.fixture(failed=True) + result = scorer.score(self.root, self.reviews()) + self.assertTrue(result['integrity_passed']) + self.assertTrue(result['complete']) + self.assertFalse(result['passed']) + candidate = next(g for g in result['groups'] if g['arm'] == 'candidate') + self.assertEqual(candidate['all_attempt_output_tokens'], 1010) + self.assertEqual(candidate['task_successes'], 1) + self.assertEqual(candidate['output_tokens_per_success'], 1010) + self.assertEqual(candidate['oracle_failures'], 1) + self.assertEqual(len(result['paired_all_attempt_deltas']), 2) + + def test_missing_usage_leaves_incomplete_total(self): + self.fixture(usage={'input_tokens': 10}) + result = scorer.score(self.root) + self.assertFalse(result['execution_passed']) + self.assertTrue(all(g['all_attempt_output_tokens'] is None for g in result['groups'])) + + def test_unknown_reasoning_is_null(self): + self.fixture() + self.assertIsNone(scorer.score(self.root)['runs'][0]['usage']['reasoning_output_tokens']) + + def test_stale_review_rejected(self): + self.fixture() + path = self.reviews(); data = json.loads(path.read_text()) + data['runs']['00']['answer_sha256'] = 'wrong' + dump(path, data) + result = scorer.score(self.root, path) + self.assertFalse(result['review_complete']) + self.assertTrue(result['review_errors']) + + def test_activation_needs_real_trace_reference(self): + self.fixture(case='delivery') + path = self.reviews(); data = json.loads(path.read_text()) + data['runs']['02']['activation']['event_lines'] = [2] + dump(path, data) + self.assertFalse(scorer.score(self.root, path)['passed']) + data['runs']['02']['activation']['event_lines'] = [1] + dump(path, data) + self.assertTrue(scorer.score(self.root, path)['passed']) + + def test_skill_name_without_body_is_not_activation(self): + self.fixture(case='delivery') + run = self.root/'runs/02' + events = [json.loads(line) for line in (run/'events.jsonl').read_text().splitlines()] + events[0]['item']['aggregated_output'] = 'verified-delivery/SKILL.md' + (run/'events.jsonl').write_text('\n'.join(json.dumps(e) for e in events)+'\n') + (run/'evidence-manifest.json').unlink(); seal_run(run) + self.assertFalse(scorer.score(self.root, self.reviews())['passed']) + + def test_invalid_turn_telemetry_is_not_complete(self): + self.fixture() + run = self.root/'runs/00' + original = (run/'events.jsonl').read_text() + variants = [original + original, + '{"type":"turn.completed","usage":{"input_tokens":-1,"output_tokens":10}}\n', + 'broken json\n'] + for text in variants: + with self.subTest(text=text[:35]): + (run/'events.jsonl').write_text(text) + (run/'evidence-manifest.json').unlink(); seal_run(run) + result = scorer.score(self.root) + self.assertTrue(result['integrity_passed']) + self.assertFalse(result['execution_passed']) + self.assertIsNone(result['groups'][0]['all_attempt_output_tokens']) + + def test_failed_turn_consumption_is_retained(self): + self.fixture() + run = self.root/'runs/00' + with (run/'events.jsonl').open('a') as handle: + handle.write('{"type":"turn.failed"}\n') + (run/'evidence-manifest.json').unlink(); seal_run(run) + result = scorer.score(self.root) + self.assertFalse(result['execution_passed']) + self.assertEqual(result['groups'][0]['all_attempt_output_tokens'], 20) + + def test_symlinked_evidence_rejected(self): + self.fixture() + (self.root/'runs/00/work/link').symlink_to(self.root/'protocol.json') + self.assertFalse(scorer.score(self.root)['integrity_passed']) + + def test_empty_schedule_rejected(self): + self.fixture() + p = self.root / 'protocol.json'; data = json.loads(p.read_text()); data['schedule'] = [] + dump(p, data) + self.assertFalse(scorer.score(self.root)['integrity_passed']) + + def test_oracle_catches_vacuous_test_and_passes_fix(self): + work = self.root / 'work'; work.mkdir() + source = work / 'scores.py' + source.write_text('def sorted_scores(scores):\n scores.sort()\n return scores\n') + (work / 'test_scores.py').write_text('assert True') + self.assertEqual(runner.check_implementation(work, self.root/'before.log')['exit_code'], 1) + source.write_text('def sorted_scores(scores):\n return sorted(scores)\n') + self.assertEqual(runner.check_implementation(work, self.root/'after.log')['exit_code'], 0) + + def test_offline_runner_to_scorer_integration(self): + cli = self.root/'fake-codex' + cli.write_text("""#!/usr/bin/env python3 +import json, os, sys +from pathlib import Path +args=sys.argv[1:] +if '--version' in args: + print('FAKE CLI fixture; no model inference') +elif 'plugin' in args: + print('{}') +elif 'debug' in args: + print('wigtn-plugins-with-codex:product-spec' if Path(os.environ['CODEX_HOME']).name != 'bare' else '[]') +else: + prompt=sys.stdin.read() + work=Path(args[args.index('-C')+1]) + if (work/'scores.py').exists(): + (work/'scores.py').write_text('def sorted_scores(scores):\\n return sorted(scores)\\n') + Path(args[args.index('-o')+1]).write_text('Synthetic CLI fixture answer; not model generated') + print(json.dumps({'type':'turn.completed','usage':{'input_tokens':100,'output_tokens':10}})) +""") + cli.chmod(0o755) + auth = self.root/'fake-auth.json'; auth.write_text('{}') + root = self.root/'integration' + process = subprocess.run([sys.executable, '-B', str(Path(runner.__file__)), + '--baseline', str(Path(__file__).resolve().parents[1]), '--root', str(root), + '--repeat', '1', '--codex', str(cli), '--execute'], + env={**os.environ, 'CODEX_AUTH_FILE': str(auth)}, capture_output=True, text=True) + self.assertEqual(process.returncode, 0, process.stdout + process.stderr) + result = scorer.score(root) + self.assertTrue(result['integrity_passed'], result['integrity_errors']) + self.assertTrue(result['execution_passed']) + self.assertTrue(result['schedule_complete']) + self.assertFalse(result['passed']) # No semantic reviews or actual model activation. + self.assertEqual(len(result['runs']), 12) + + def test_new_contract_cases_and_complete_criteria(self): + self.assertEqual(len(RUBRIC['many-findings']), 6) + for case in ['compact-review', 'external-review', 'many-findings']: + self.assertTrue(PROMPTS[case]) + self.assertTrue(RUBRIC[case]) + result = subprocess.run([sys.executable, '-B', str(Path(runner.__file__)), + '--baseline', str(Path(__file__).resolve().parents[1]), + '--root', str(self.root/'dry'), '--suite', 'contracts'], + capture_output=True, text=True) + self.assertEqual(result.returncode, 0, result.stderr) + plan = json.loads(result.stdout) + self.assertEqual(plan['calls'], 30) + self.assertFalse((self.root/'dry').exists()) + + +if __name__ == '__main__': + unittest.main(verbosity=2) diff --git a/scripts/check-release-state.py b/scripts/check-release-state.py index e957368..aa38605 100644 --- a/scripts/check-release-state.py +++ b/scripts/check-release-state.py @@ -28,8 +28,8 @@ def main() -> int: (repo / "staged.txt").write_text("base\n") run(repo, "git", "add", ".") run(repo, "git", "commit", "-qm", "initial") - (repo / "tracked.txt").write_text("changed\n") - (repo / "staged.txt").write_text("staged\n") + (repo / "tracked.txt").write_text("changed with trailing space \n") + (repo / "staged.txt").write_text("staged with trailing space \n") run(repo, "git", "add", "staged.txt") (repo / "draft.txt").write_text("draft\n") completed = subprocess.run( @@ -44,7 +44,46 @@ def main() -> int: assert state["unstaged"] == ["tracked.txt"] assert state["untracked"] == ["draft.txt"] assert state["conflicts"] == [] and state["operation"] == "none" - print("Release state: PASS (staged/unstaged/untracked)") + assert "diffs" not in state + + detailed = subprocess.run( + [ + sys.executable, + str(INSPECTOR), + str(repo), + "--include-diffs", + ], + check=True, + capture_output=True, + text=True, + ) + detail = json.loads(detailed.stdout) + assert "staged with trailing space" in detail["diffs"]["staged"]["patch"] + assert "changed with trailing space" in detail["diffs"]["unstaged"]["patch"] + assert "draft" not in detail["diffs"]["staged"]["patch"] + assert "draft" not in detail["diffs"]["unstaged"]["patch"] + assert detail["diffs"]["check"]["staged"]["clean"] is False + assert detail["diffs"]["check"]["unstaged"]["clean"] is False + + limited = subprocess.run( + [ + sys.executable, + str(INSPECTOR), + str(repo), + "--include-diffs", + "--max-diff-bytes", + "12", + ], + check=True, + capture_output=True, + text=True, + ) + limited_state = json.loads(limited.stdout) + assert limited_state["diffs"]["staged"]["truncated"] is True + assert limited_state["diffs"]["unstaged"]["truncated"] is True + assert len(limited_state["diffs"]["staged"]["patch"].encode()) <= 12 + assert len(limited_state["diffs"]["unstaged"]["patch"].encode()) <= 12 + print("Release state: PASS (paths, bounded diffs, whitespace checks)") return 0 diff --git a/scripts/check-screen-contract.py b/scripts/check-screen-contract.py index 4021016..1a40021 100644 --- a/scripts/check-screen-contract.py +++ b/scripts/check-screen-contract.py @@ -12,6 +12,7 @@ ROOT = Path(__file__).resolve().parents[1] VALIDATOR = ROOT / "plugins/wigtn-plugins-with-codex/scripts/validate-screen-spec.py" TEMPLATES = ROOT / "plugins/wigtn-plugins-with-codex/skills/screen-spec/assets/templates" +SCREEN_SKILL = ROOT / "plugins/wigtn-plugins-with-codex/skills/screen-spec/SKILL.md" VALID = { @@ -47,14 +48,17 @@ """ -def run(directory: Path, artifacts: str = "all") -> subprocess.CompletedProcess[str]: +def run( + directory: Path, artifacts: str = "all", *, spaced: bool = False +) -> subprocess.CompletedProcess[str]: + selection = artifacts.split(",") if spaced else [artifacts] return subprocess.run( [ sys.executable, str(VALIDATOR), str(directory), "--artifacts", - artifacts, + *selection, "--json", ], text=True, @@ -63,11 +67,14 @@ def run(directory: Path, artifacts: str = "all") -> subprocess.CompletedProcess[ def main() -> int: + skill = SCREEN_SKILL.read_text(encoding="utf-8") flow_template = (TEMPLATES / "02-USER-FLOW.md").read_text(encoding="utf-8") wireframe_template = (TEMPLATES / "04-WIREFRAME.html").read_text(encoding="utf-8") assert "## Flow Coverage" in flow_template assert '@media (max-width' not in wireframe_template assert 'lang="{language-code}"' in wireframe_template + assert "`:organizationId` or `:email`" in skill + assert "also appears in IA" in skill with tempfile.TemporaryDirectory() as temp: root = Path(temp) @@ -77,6 +84,8 @@ def main() -> int: (valid / name).write_text(text, encoding="utf-8") passed = run(valid) assert passed.returncode == 0 and '"valid": true' in passed.stdout + passed = run(valid, "ia,screen", spaced=True) + assert passed.returncode == 0 and '"valid": true' in passed.stdout ac_handoff = root / "ac-handoff" ac_handoff.mkdir() @@ -99,6 +108,18 @@ def main() -> int: passed = run(korean_ia, "ia") assert passed.returncode == 0 and '"valid": true' in passed.stdout + korean_screen = root / "korean-screen" + korean_screen.mkdir() + (korean_screen / "01-IA.md").write_text( + VALID["01-IA.md"], encoding="utf-8" + ) + (korean_screen / "03-SCREEN-SPEC.md").write_text( + VALID["03-SCREEN-SPEC.md"].replace("## Screen:", "## 화면:"), + encoding="utf-8", + ) + passed = run(korean_screen, "screen") + assert passed.returncode == 0 and '"valid": true' in passed.stdout + prose_only = root / "prose-only-ia" prose_only.mkdir() (prose_only / "01-IA.md").write_text( diff --git a/scripts/check-symbolic-requirement-ids.py b/scripts/check-symbolic-requirement-ids.py new file mode 100644 index 0000000..b06f92e --- /dev/null +++ b/scripts/check-symbolic-requirement-ids.py @@ -0,0 +1,118 @@ +#!/usr/bin/env python3 +"""Integration regression for stable symbolic requirement IDs found in live eval.""" +import importlib.util +import json +import os +from pathlib import Path +import re +import subprocess +import sys +import tempfile +import unittest + +ROOT = Path(os.environ.get('WIGTN_TEST_ROOT', Path(__file__).resolve().parents[1])) +PLUGIN = ROOT / 'plugins/wigtn-plugins-with-codex' +SCRIPTS = PLUGIN / 'scripts' +sys.path.insert(0, str(SCRIPTS)) +sys.path.insert(0, str(ROOT / 'scripts')) + +def module(name, path): + spec = importlib.util.spec_from_file_location(name, path) + result = importlib.util.module_from_spec(spec) + spec.loader.exec_module(result) + return result + +importer = module('requirement_importer', SCRIPTS / 'import-requirements.py') +workgraph_cases = module('workgraph_cases', ROOT / 'scripts/check-workgraph-contract.py') +from workgraph_core import validate_graph + +class SymbolicIDs(unittest.TestCase): + def test_explicit_import_preserves_complete_ids(self): + text = '''# Requirements +REQ-TTL: Lifetime is exactly 86400 seconds. +- **REQ-TENANT-ACCESS**: Keep tenant data isolated. +| REQ-MAX_LENGTH | Limit names to 80 characters. | +- fr-01: Preserve legacy numeric IDs. +REQ-1: Allow a one-digit source ID. +''' + self.assertEqual(importer.explicit_requirements(text), [ + ('REQ-TTL', 'Lifetime is exactly 86400 seconds.'), + ('REQ-TENANT-ACCESS', 'Keep tenant data isolated.'), + ('REQ-MAX_LENGTH', 'Limit names to 80 characters.'), + ('FR-01', 'Preserve legacy numeric IDs.'), + ('REQ-1', 'Allow a one-digit source ID.'), + ]) + + def test_markdown_lists_and_checkboxes_remain_supported(self): + for prefix in ['* ', '- ', '+ ', '1. ', '2) ', '### ', '- [ ] ', '- [x] ']: + for identifier in ['FR-01', 'REQ-TTL']: + with self.subTest(prefix=prefix, identifier=identifier): + self.assertEqual(importer.explicit_requirements( + prefix + identifier + ': Preserve preference.'), + [(identifier, 'Preserve preference.')]) + + def test_acceptance_section_preserves_underscore_ids(self): + self.assertEqual(importer.derived_acceptance( + '## Acceptance Criteria\n- **REQ-MAX_LENGTH**: Limit to 80 characters.\n', + 'BM', 'story.md', set()), + [('REQ-MAX_LENGTH', 'Limit to 80 characters.')]) + + def test_prose_and_incomplete_ids_are_not_requirements(self): + for text in ['tenant-scoped access is required.', + 'See REQ-TTL: for background.', + 'REQ-TTL', 'REQ-TTL:', 'REQ-TTL-: broken ID', + 'REQ--TTL: broken ID']: + with self.subTest(text=text): + self.assertEqual(importer.explicit_requirements(text), []) + + def test_contracts_accept_same_symbolic_id(self): + graph = json.loads(json.dumps(workgraph_cases.valid_graph()).replace('FR-01', 'REQ-TTL')) + self.assertEqual(validate_graph(graph), []) + evidence = json.loads((ROOT / 'tests/evidence/valid-acceptance.json').read_text().replace('FR-01', 'REQ-TTL')) + with tempfile.TemporaryDirectory() as directory: + path = Path(directory) / 'evidence.json' + path.write_text(json.dumps(evidence)) + result = subprocess.run([sys.executable, '-B', str(SCRIPTS / 'validate-evidence.py'), str(path)], capture_output=True, text=True) + self.assertEqual(result.returncode, 0, result.stdout + result.stderr) + # Dependency-free validators and published JSON schemas must agree. + for name in ['workgraph', 'evidence-contract']: + schema = json.loads((PLUGIN / f'schemas/{name}.schema.json').read_text()) + requirement = schema.get('$defs', schema.get('definitions', {})).get('requirement') + if requirement is None: + requirement = schema['properties']['requirements']['items'] + pattern = requirement['properties']['id']['pattern'] + for identifier in ['REQ-TTL', 'FR-01', 'REQ-1', 'REQ-MAX_LENGTH', 'REQ-TENANT-ACCESS']: + self.assertIsNotNone(re.fullmatch(pattern, identifier), (name, identifier)) + for identifier in ['REQ--TTL', 'REQ-TTL-', 'REQ', 'REQ-TTL:']: + self.assertIsNone(re.fullmatch(pattern, identifier), (name, identifier)) + + def test_workgraph_import_plan_and_source_drift(self): + with tempfile.TemporaryDirectory(prefix='wigtn-symbolic-id-') as directory: + root = Path(directory) + source = root / 'requirements.md' + source.write_text('REQ-TTL: Lifetime is exactly 86400 seconds.\n') + def cli(*args, expected=0): + result = subprocess.run([sys.executable, '-B', str(SCRIPTS / 'wigtn.py'), '--root', str(root), *args], capture_output=True, text=True) + self.assertEqual(result.returncode, expected, result.stdout + result.stderr) + return json.loads(result.stdout) + cli('init', '--apply') + cli('import', str(source), '--apply') + cli('plan', '--apply') + path = root / '.wigtn/workgraph.json' + before = json.loads(path.read_text()) + self.assertEqual([r['id'] for r in before['requirements']], ['REQ-TTL']) + self.assertTrue(before['tasks']) + self.assertEqual(before['tasks'][0]['requirement_ids'], ['REQ-TTL']) + source.write_text('REQ-TTL: Lifetime is exactly 900 seconds.\n') + cli('diff', '--check', expected=1) + cli('diff', '--apply') + cli('import', str(source), '--apply') + after = json.loads(path.read_text()) + self.assertEqual([r['id'] for r in after['requirements']], ['REQ-TTL']) + self.assertIn('900', after['requirements'][0]['text']) + self.assertEqual(after['tasks'][0]['id'], before['tasks'][0]['id']) + self.assertNotEqual(after['requirements'][0]['source_sha256'], before['requirements'][0]['source_sha256']) + self.assertEqual(validate_graph(after), []) + +if __name__ == '__main__': + unittest.main(verbosity=2) diff --git a/scripts/check-token-ablation.py b/scripts/check-token-ablation.py new file mode 100755 index 0000000..2cecaa6 --- /dev/null +++ b/scripts/check-token-ablation.py @@ -0,0 +1,94 @@ +#!/usr/bin/env python3 +"""Exercise positive and negative token-ablation scorer paths.""" + +from __future__ import annotations + +import json +from pathlib import Path +import subprocess +import sys +import tempfile + + +ROOT = Path(__file__).resolve().parents[1] +SCORER = ROOT / "scripts/score-token-ablation.py" + + +def write_run( + root: Path, + arm: str, + case: str, + *, + input_tokens: int, + tool_items: int, + output: str, +) -> None: + stem = root / "runs" / arm / f"{case}.1" + stem.parent.mkdir(parents=True, exist_ok=True) + Path(f"{stem}.meta.json").write_text( + json.dumps( + { + "arm": arm, + "case": case, + "repeat": 1, + "exit_code": 0, + "duration_seconds": 1, + } + ) + + "\n" + ) + Path(f"{stem}.out.md").write_text(output) + events = [ + *[ + {"type": "item.completed", "item": {"type": "command_execution"}} + for _ in range(tool_items) + ], + { + "type": "turn.completed", + "usage": { + "input_tokens": input_tokens, + "cached_input_tokens": 0, + "cache_write_input_tokens": 0, + "output_tokens": 100, + "reasoning_output_tokens": 20, + }, + }, + ] + Path(f"{stem}.events.jsonl").write_text( + "\n".join(json.dumps(event) for event in events) + "\n" + ) + + +def main() -> int: + with tempfile.TemporaryDirectory(prefix="wigtn-token-ablation-") as temporary: + root = Path(temporary) + (root / "SCHEDULE.tsv").write_text( + "pair_id\torder\tarm\tcase\tprompt\trepeat\n" + "prd-create.1\t1\tbaseline\tprd-create\tprd.txt\t1\n" + "prd-create.1\t2\tcandidate\tprd-create\tprd.txt\t1\n" + "acceptance-uncertain.1\t3\tbaseline\tacceptance-uncertain\tacceptance.txt\t1\n" + "acceptance-uncertain.1\t4\tcandidate\tacceptance-uncertain\tacceptance.txt\t1\n" + ) + prd = "\nFR-1\nAC-1\n" + write_run(root, "baseline", "prd-create", input_tokens=40000, tool_items=2, output=prd) + write_run(root, "candidate", "prd-create", input_tokens=26000, tool_items=1, output=prd) + write_run(root, "baseline", "acceptance-uncertain", input_tokens=12000, tool_items=0, output="unknown\n") + candidate = root / "runs/candidate/acceptance-uncertain.1.out.md" + write_run(root, "candidate", "acceptance-uncertain", input_tokens=12000, tool_items=0, output="not-verifiable\n") + passed = subprocess.run( + [sys.executable, "-B", str(SCORER), str(root)], text=True, capture_output=True + ) + if passed.returncode: + raise AssertionError(passed.stdout + passed.stderr) + candidate.write_text("Not Verifiable\n") + failed = subprocess.run( + [sys.executable, "-B", str(SCORER), str(root)], text=True, capture_output=True + ) + if failed.returncode != 1: + raise AssertionError("localized status did not fail the ablation scorer") + print("Token ablation scorer: PASS (positive + canonical regression)") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/check-verified-delivery-routing.py b/scripts/check-verified-delivery-routing.py index fed6c99..71ddf71 100644 --- a/scripts/check-verified-delivery-routing.py +++ b/scripts/check-verified-delivery-routing.py @@ -30,7 +30,7 @@ def main() -> int: "native oracle non-duplication": "Do not duplicate a passing repository oracle", "no default state": "Do not create stable IDs, WorkGraph state, or evidence JSON by default", "multi-interface census": "Use a compact coverage census", - "reference isolation": "Do not inspect or copy another checkout", + "evaluation isolation": "For benchmark or independent evaluation work", "leakage invalidates evidence": "A clean evaluator pass does not", "matrix suppression": "Do not manufacture a matrix for a one-line fix", } diff --git a/scripts/check-workgraph-concurrency.py b/scripts/check-workgraph-concurrency.py new file mode 100644 index 0000000..c2bdc45 --- /dev/null +++ b/scripts/check-workgraph-concurrency.py @@ -0,0 +1,54 @@ +"""Interleave two real CLI writers; assert no stale-revision acceptance or lost updates.""" +from pathlib import Path +import json,os,subprocess,sys,tempfile,time,unittest +ROOT=Path(os.environ.get('WIGTN_TEST_ROOT',str(Path(__file__).resolve().parents[1]))) +CLI=ROOT/'plugins/wigtn-plugins-with-codex/scripts/wigtn.py' +class Concurrency(unittest.TestCase): + def setUp(self): + self.tmp=tempfile.TemporaryDirectory();self.base=Path(self.tmp.name);self.root=self.base/'repo';self.root.mkdir();(self.root/'requirements.md').write_text('REQ-A: Save A.\nREQ-B: Save B.\n') + for args in [('init','--apply'),('import','requirements.md','--apply'),('plan','--apply')]:self.assertEqual(self.runcli(*args).returncode,0) + self.initial=self.graph();self.task=self.initial['tasks'][0]['id'] + def tearDown(self):self.tmp.cleanup() + def graph(self):return json.loads((self.root/'.wigtn/workgraph.json').read_text()) + def runcli(self,*args):return subprocess.run([sys.executable,'-B',str(CLI),'--root',str(self.root),'--json',*args],text=True,capture_output=True,timeout=15) + def waitfor(self,p): + until=time.monotonic()+8 + while not p.exists() and time.monotonic()until:raise ValueError('test barrier timed out')\n time.sleep(.01)\n return g\nm.require_graph=hook\n(base/(name+'.start')).write_text('start')\nsys.argv=[str(cli)]+sys.argv[5:];raise SystemExit(m.main())\n''') + return subprocess.Popen([sys.executable,'-B',str(wrapper),str(CLI),str(self.base),name,'yes' if paused else 'no','--root',str(self.root),'--json',*args],text=True,stdout=subprocess.PIPE,stderr=subprocess.PIPE) + def pair(self,expected): + rev=['--expected-revision',str(self.initial['revision'])] if expected else [] + a=self.writer('a',True,['task','update',self.task,'--title','Writer A',*rev,'--apply']);b=None + try: + self.waitfor(self.base/'a.read');b=self.writer('b',False,['task','update',self.task,'--risk','high',*rev,'--apply']);self.waitfor(self.base/'b.start') + # Hold A while B attempts the same transaction; record actual observed order. + until=time.monotonic()+.4 + while b.poll() is None and not (self.base/'b.read').exists() and time.monotonic() dict[str, object]: + usage = {field: 0 for field in USAGE_FIELDS} + item_types: Counter[str] = Counter() + event_count = turn_count = 0 + with path.open(encoding="utf-8") as handle: + for line_number, raw in enumerate(handle, 1): + line = raw.strip() + if not line: + continue + try: + event = json.loads(line) + except json.JSONDecodeError as error: + raise ValueError(f"{path}:{line_number}: invalid JSON event") from error + event_count += 1 + if event.get("type") == "turn.completed": + turn_count += 1 + values = event.get("usage") + if not isinstance(values, dict): + raise ValueError(f"{path}:{line_number}: missing usage object") + for field in USAGE_FIELDS: + value = values.get(field) + if value is None and field not in {"input_tokens", "output_tokens"}: + usage[field] = None + continue + if type(value) is not int or value < 0: + raise ValueError(f"{path}:{line_number}: invalid/missing {field}") + if usage[field] is not None: + usage[field] += value + cached = values.get("cached_input_tokens") + written = values.get("cache_write_input_tokens") + reasoning = values.get("reasoning_output_tokens") + if (reasoning is not None and reasoning > values["output_tokens"] + or cached is not None and cached > values["input_tokens"] + or written is not None and written > values["input_tokens"] + or cached is not None and written is not None + and cached + written > values["input_tokens"]): + raise ValueError(f"{path}:{line_number}: inconsistent usage breakdown") + if event.get("type") == "item.completed": + item_type = str((event.get("item") or {}).get("type", "unknown")) + item_types[item_type] += 1 + if turn_count == 0: + raise ValueError(f"{path}: missing turn.completed usage event") + usage["visible_output_tokens"] = ( + usage["output_tokens"] - usage["reasoning_output_tokens"] + if usage["reasoning_output_tokens"] is not None else None + ) + usage["uncached_input_tokens"] = ( + usage["input_tokens"] - usage["cached_input_tokens"] - usage["cache_write_input_tokens"] + if usage["cached_input_tokens"] is not None and usage["cache_write_input_tokens"] is not None + else None + ) + usage["turns"] = turn_count + usage["events"] = event_count + usage["tool_items"] = sum( + count for item_type, count in item_types.items() if item_type in TOOL_ITEM_TYPES + ) + usage["item_types"] = dict(sorted(item_types.items())) + return usage + + +def api_equivalent_cost( + usage: dict[str, object], + *, + input_per_million: float, + cached_input_per_million: float, + cache_write_per_million: float, + output_per_million: float, +) -> float | None: + # A rate table cannot supply telemetry that the service did not report. + if any(usage.get(field) is None for field in ( + "uncached_input_tokens", "cached_input_tokens", "cache_write_input_tokens", "output_tokens" + )): + return None + return ( + int(usage["uncached_input_tokens"]) * input_per_million + + int(usage["cached_input_tokens"]) * cached_input_per_million + + int(usage["cache_write_input_tokens"]) * cache_write_per_million + + int(usage["output_tokens"]) * output_per_million + ) / 1_000_000 + + +def cold_api_equivalent_cost( + usage: dict[str, object], *, input_per_million: float, output_per_million: float +) -> float: + """Price all input as uncached to compare runs independent of cache order.""" + return ( + int(usage["input_tokens"]) * input_per_million + + int(usage["output_tokens"]) * output_per_million + ) / 1_000_000 + + +def main() -> int: + parser = argparse.ArgumentParser() + parser.add_argument("events", type=Path) + parser.add_argument("--input-per-million", type=float) + parser.add_argument("--cached-input-per-million", type=float) + parser.add_argument("--cache-write-per-million", type=float) + parser.add_argument("--output-per-million", type=float) + args = parser.parse_args() + rates = {key: getattr(args, key) for key in ( + "input_per_million", "cached_input_per_million", + "cache_write_per_million", "output_per_million" + )} + priced = all(value is not None for value in rates.values()) + if any(value is not None for value in rates.values()) and not priced: + parser.error("provide all four rates or none") + if priced and any(not math.isfinite(value) or value < 0 for value in rates.values()): + parser.error("rates must be finite and nonnegative") + usage = read_events(args.events) + usage["rates_usd_per_million"] = rates if priced else None + usage["api_equivalent_usd"] = api_equivalent_cost(usage, **rates) if priced else None + usage["cold_api_equivalent_usd"] = cold_api_equivalent_cost( + usage, input_per_million=args.input_per_million, + output_per_million=args.output_per_million, + ) if priced else None + print(json.dumps(usage, indent=2, sort_keys=True)) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/export-eval-packet.py b/scripts/export-eval-packet.py index 10e5e33..b847748 100644 --- a/scripts/export-eval-packet.py +++ b/scripts/export-eval-packet.py @@ -11,7 +11,7 @@ import shutil -ALLOWED_SUFFIXES = {".json", ".log", ".md", ".patch", ".txt", ".csv"} +ALLOWED_SUFFIXES = {".json", ".jsonl", ".log", ".md", ".patch", ".txt", ".csv"} EXCLUDED_PARTS = {"homes", "home", "work", "staging", "prompt-input"} SECRET = re.compile( r"(?i)(authorization:\s*bearer\s+|api[_-]?key[\"'=:\s]+|token[\"'=:\s]+)" diff --git a/scripts/migration_cases.py b/scripts/migration_cases.py new file mode 100644 index 0000000..9caffa3 --- /dev/null +++ b/scripts/migration_cases.py @@ -0,0 +1,64 @@ +"""Predeclared development cases and review criteria; no expected answers in prompts.""" +PROMPTS = { + 'prd': '파일을 수정하지 말고 조직 관리자 팀원 초대 기능의 간결한 구현용 PRD를 작성해줘. 만료 전 수락, 중복 초대, 만료, 권한, 재전송을 포함하고 안정적인 요구사항 ID와 관찰 가능한 인수조건을 붙여라. SLA나 규모 수치를 만들지 마라.', + 'uncertain': '파일을 수정하거나 명령을 실행하지 말고 검증해줘. REQ-01: 초대 토큰은 한 번만 사용할 수 있다. 증거는 consumeInvite(token) 함수가 있다는 설명뿐이다. 코드와 테스트 결과는 제공되지 않았다. 확인할 수 없는 사실을 통과로 추정하지 마라.', + 'design': '팀 내부 장애 이력 대시보드의 디자인 방향을 정해줘. 새 프로젝트이고 별도 디자인 시스템은 없다. 선호는 흰 배경, 청록 포인트, 높은 정보 밀도, 읽기 쉬운 한글이다. 대안을 고르는 단계 없이 적절한 방향 하나를 결정하고 바로 구현 가능한 시각 계약을 답변해줘. 코드는 작성하지 마라.', + 'ordinary': "Fix sorted_scores in scores.py so it returns scores in ascending order without mutating the caller's list. Run the existing tests. Keep notes.txt unchanged.", + 'compact-review': '''아래 Compact PRD를 실질적으로 리뷰해줘. 수정본 작성은 요청하지 않는다. + +# CSV 내보내기 PRD +## Problem and scope +관리자가 현재 목록을 CSV로 저장한다. 기존 목록 화면 버튼을 사용한다. +## Goals and non-goals +현재 조회 결과만 내보낸다. 새 화면, 다단계 작업, 비동기 작업은 범위 밖이다. +## Users, roles, authorization, and data boundaries +조직 관리자만 소속 조직 데이터를 내보낸다. 서버가 역할과 조직을 검사한다. +## Functional requirements +| ID | Requirement | Priority | +| FR-01 | 현재 필터 결과를 UTF-8 CSV로 반환 | Must | +| FR-02 | 서버가 관리자 역할과 조직 소유권 검사 | Must | +## Acceptance criteria +| ID | Requirement | Given | When | Then | Verification | +| AC-01 | FR-01 | 조회 결과 2행 | 관리자가 내보내기 | 헤더와 해당 2행 포함 | CSV 통합 검사 | +| AC-02 | FR-02 | 다른 조직 관리자 | 내보내기 요청 | 403, 데이터 미노출 | 권한 통합 검사 | +| AC-03 | FR-02 | 소속 조직 일반 멤버 | 내보내기 요청 | 403 | 권한 통합 검사 | +## Assumptions and open decisions +CSV 열은 기존 목록 열과 같으며 현재 필터는 서버에서 다시 적용한다. +## Release condition +| Requirement IDs | Verifiable exit condition | +| FR-01, FR-02 | AC-01~03 통합 검사 통과 | +''', + 'external-review': '''WIGTN 형식으로 변환하지 말고 다음 팀 요구사항 문서의 내용만 검토해줘. +대상은 기존 API의 입력 검증 수정이다. FR-1: 이름은 공백 제거 후 1~80자만 허용한다. +FR-2: 권한은 기존 인증·조직 정책을 유지한다. AC-1: 빈 이름은 400이다. +AC-2: 80자는 저장되고 81자는 400이다. AC-3: 비로그인은 401이다. +AC-4: 다른 조직의 사용자는 변경하지 못한다. 새 화면과 사용자 흐름은 없다. +출시 조건은 기존 회귀 검사와 이 네 API 검사의 통과다. 파일을 만들지 마라.''', + 'many-findings': '''다음 PRD의 구현을 막는 모순을 리뷰해줘. 원문을 수정하지 마라. +기능은 조직별 파일 보관함이다. +FR-01: 업로드는 관리자만 가능하다. AC-01: 비로그인 방문자의 업로드가 성공한다. +FR-02: 조직 경계를 서버에서 강제한다. AC-02: 다른 조직 파일 ID를 주면 다운로드가 성공한다. +FR-03: 삭제 직후 파일은 복구 불가능하다. AC-03: 삭제 후 30일 동안 원본 복원이 가능하다. +FR-04: 동일 요청 ID 재시도는 파일 하나만 만든다. AC-04: 동일 요청 ID로 3회 보내면 파일 3개가 생긴다. +FR-05: 암호화 키는 고객만 보유하고 서버는 복호화할 수 없다. AC-05: 서버가 평문 본문으로 전문 검색한다. +FR-06: 보존기간 내 파일 삭제를 금지한다. AC-06: 보존기간 내 소유자의 삭제가 즉시 성공한다. +''', +} +PROMPTS['delivery'] = PROMPTS['ordinary'] +RUBRIC = { + 'prd': ['covers-requested-behaviors', 'observable-criteria', 'no-invented-requirements'], + 'uncertain': ['no-false-verification', 'no-prohibited-command'], + 'design': ['one-usable-direction', 'preferences-preserved', 'no-choice-pause'], + 'ordinary': ['requested-fix-and-tests', 'no-unrequested-work'], + 'delivery': ['evidence-proportional', 'no-unrequested-state'], + 'compact-review': ['compact-profile-respected', 'no-full-only-gaps', 'no-unrequested-rewrite'], + 'external-review': ['content-reviewed', 'no-wigtn-format-gate', 'no-new-artifact'], + 'many-findings': [f'conflict-FR-{i:02}' for i in range(1, 7)], +} +ACTIVATION = {case: 'product-spec' for case in ['prd', 'compact-review', 'external-review', 'many-findings']} +ACTIVATION.update(design='design-direction', delivery='verified-delivery') +SUITES = { + 'smoke': ['prd', 'uncertain', 'design', 'ordinary'], + 'targeted': ['prd', 'uncertain', 'design', 'ordinary', 'delivery'], + 'contracts': ['compact-review', 'external-review', 'many-findings', 'uncertain', 'delivery'], +} diff --git a/scripts/migration_evidence.py b/scripts/migration_evidence.py new file mode 100644 index 0000000..792c535 --- /dev/null +++ b/scripts/migration_evidence.py @@ -0,0 +1,65 @@ +"""Hash recorded evaluation evidence; excludes auth homes and generated score reports. + +Manifests detect accidental edits, not an adversary replacing both data and hashes. +""" +import hashlib +import json +from pathlib import Path + + +def hashes(root: Path, paths: list[str]) -> dict[str, str]: + result = {} + for relative in paths: + entry = root / relative + if not entry.exists() or entry.is_symlink(): + raise ValueError(f"missing or symlinked evidence: {relative}") + entries = [entry, *sorted(entry.rglob('*'))] if entry.is_dir() else [entry] + for path in entries: + if path.is_symlink(): + raise ValueError(f"symlinked evidence: {path.relative_to(root)}") + if path.is_file(): + result[str(path.relative_to(root))] = hashlib.sha256(path.read_bytes()).hexdigest() + return result + + +def seal(root: Path, paths: list[str], name: str) -> None: + document = {"version": 1, "paths": paths, "sha256": hashes(root, paths)} + with (root / name).open('x', encoding='utf-8') as handle: + json.dump(document, handle, indent=2, sort_keys=True) + handle.write('\n') + + +def verify(root: Path, paths: list[str], name: str) -> list[str]: + try: + document = json.loads((root / name).read_text()) + if document.get('version') != 1 or document.get('paths') != paths: + return [f'{name}: evidence scope mismatch'] + actual = hashes(root, paths) + expected = document['sha256'] + if not isinstance(expected, dict): + raise ValueError('invalid hash mapping') + return [f'{name}: changed/missing/extra evidence: {path}' + for path in sorted(set(expected) | set(actual)) + if expected.get(path) != actual.get(path)] + except (OSError, ValueError, KeyError, TypeError) as error: + return [f'{name}: {error}'] + + +def seal_run(run: Path) -> None: + # Fixed scope: additional work files are detected inside the work subtree. + paths = ['meta.json', 'prompt.txt', 'events.jsonl', 'work'] + for optional in ['answer.md', 'events.stderr', 'oracle.txt', 'oracle.stderr', + 'oracle-before.txt', 'oracle-before.stderr']: + if (run / optional).exists(): + paths.append(optional) + seal(run, paths, 'evidence-manifest.json') + + +def verify_run(run: Path) -> list[str]: + # Reconstruct scope rather than trusting a manifest to omit changed files. + paths = ['meta.json', 'prompt.txt', 'events.jsonl', 'work'] + for optional in ['answer.md', 'events.stderr', 'oracle.txt', 'oracle.stderr', + 'oracle-before.txt', 'oracle-before.stderr']: + if (run / optional).exists(): + paths.append(optional) + return verify(run, paths, 'evidence-manifest.json') diff --git a/scripts/run-behavior-evals.sh b/scripts/run-behavior-evals.sh index 249be14..88c799d 100755 --- a/scripts/run-behavior-evals.sh +++ b/scripts/run-behavior-evals.sh @@ -3,11 +3,15 @@ set -euo pipefail repo_root="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" cases_file="$repo_root/tests/behavior/cases.tsv" -model="${WIGTN_EVAL_MODEL:-gpt-5.6-sol}" +model="${WIGTN_EVAL_MODEL:-gpt-6-astra}" effort="${WIGTN_EVAL_EFFORT:-medium}" repeat="${WIGTN_EVAL_REPEAT:-1}" run_root="${WIGTN_EVAL_ROOT:-/tmp/wigtn-codex-behavior-eval-$model}" seed="${WIGTN_EVAL_SEED:-behavior-smoke-v1}" +input_rate="${WIGTN_INPUT_USD_PER_M:-}" +cached_input_rate="${WIGTN_CACHED_INPUT_USD_PER_M:-}" +cache_write_rate="${WIGTN_CACHE_WRITE_USD_PER_M:-}" +output_rate="${WIGTN_OUTPUT_USD_PER_M:-}" auth_file="${CODEX_AUTH_FILE:-$HOME/.codex/auth.json}" if [[ -n "${CODEX_BIN:-}" ]]; then @@ -36,6 +40,16 @@ EOF exit 0 fi +pricing_args=() +if [[ -n "$input_rate$cached_input_rate$cache_write_rate$output_rate" ]]; then + [[ -n "$input_rate" && -n "$cached_input_rate" && -n "$cache_write_rate" && -n "$output_rate" ]] || { + echo "Set all four WIGTN pricing rates or leave all unset" >&2 + exit 2 + } + pricing_args=(--input-per-million "$input_rate" --cached-input-per-million "$cached_input_rate" + --cache-write-per-million "$cache_write_rate" --output-per-million "$output_rate") +fi + [[ "$repeat" =~ ^[1-9][0-9]*$ ]] || { echo "WIGTN_EVAL_REPEAT must be a positive integer" >&2 exit 2 @@ -109,11 +123,15 @@ PY printf 'codex_cli=%s\n' "$("$codex_bin" --version)" printf 'model=%s\neffort=%s\nrepetitions=%s\n' "$model" "$effort" "$repeat" printf 'schedule_seed=%s\n' "$seed" + printf 'input_usd_per_m=%s\ncached_input_usd_per_m=%s\n' "$input_rate" "$cached_input_rate" + printf 'cache_write_usd_per_m=%s\noutput_usd_per_m=%s\n' "$cache_write_rate" "$output_rate" ( cd "$repo_root" shasum -a 256 \ scripts/run-behavior-evals.sh \ scripts/make-eval-schedule.py \ + scripts/codex_usage.py \ + scripts/summarize-token-efficiency.py \ scripts/score-behavior-smoke.py \ "$run_root/SCHEDULE.tsv" \ tests/behavior/cases.tsv \ @@ -139,8 +157,8 @@ run_one() { -a never -m "$model" -c "model_reasoning_effort=\"$effort\"" \ -s read-only -C "$run_root/$arm-work" \ exec --ephemeral --ignore-rules --skip-git-repo-check \ - -o "$stem.out.md" - < "$repo_root/$prompt_path" \ - > "$stem.log" 2>&1 + --json -o "$stem.out.md" - < "$repo_root/$prompt_path" \ + > "$stem.events.jsonl" 2> "$stem.log" rc=$? set -e @@ -182,5 +200,8 @@ while IFS=$'\t' read -r pair_id order arm case_id prompt_path rep_index; do run_one "$pair_id" "$order" "$arm" "$case_id" "$prompt_path" "$rep_index" done < "$run_root/SCHEDULE.tsv" +env PYTHONDONTWRITEBYTECODE=1 python3 \ + "$repo_root/scripts/summarize-token-efficiency.py" "$run_root" \ + "${pricing_args[@]}" env PYTHONDONTWRITEBYTECODE=1 python3 \ "$repo_root/scripts/score-behavior-smoke.py" "$run_root" diff --git a/scripts/run-model-migration-eval.py b/scripts/run-model-migration-eval.py new file mode 100644 index 0000000..a4a34d7 --- /dev/null +++ b/scripts/run-model-migration-eval.py @@ -0,0 +1,174 @@ +#!/usr/bin/env python3 +"""Snapshot-based, sequential pilot. No calls without --execute; no billing estimates.""" +import argparse +import hashlib +import json +import os +from pathlib import Path +import random +import shutil +import subprocess +import time +import sys + +from migration_cases import PROMPTS as CASES, RUBRIC, ACTIVATION, SUITES +from migration_evidence import seal, seal_run + +CODE_CASES = {'ordinary', 'delivery'} +ORACLE = '''import importlib.util, sys +spec = importlib.util.spec_from_file_location('evaluated_scores', sys.argv[1]) +module = importlib.util.module_from_spec(spec) +spec.loader.exec_module(module) +for values in ([], [1], [3, 1, 2], [2, 2, -1], [5, 0, -3, 5]): + original = values.copy() + result = module.sorted_scores(values) + assert result == sorted(original), (original, result) + assert values == original, (original, values) +print('Independent oracle: PASS (5 inputs, ordering and nonmutation)') +''' + +def check_implementation(work, log): + return invoke([sys.executable, '-I', '-B', '-c', ORACLE, str(work/'scores.py')], + os.environ.copy(), log.parent, log, timeout=30) + +def snapshot_marketplace(source, destination): + # Copy only the evaluated distributable, not reports, other plugins or repository files. + catalog = json.loads((source/'.agents/plugins/marketplace.json').read_text()) + catalog['plugins'] = [p for p in catalog['plugins'] if p['name']=='wigtn-plugins-with-codex'] + if len(catalog['plugins']) != 1: + raise ValueError('expected one WIGTN core plugin') + plugin = source/'plugins/wigtn-plugins-with-codex' + if any(p.is_symlink() for p in plugin.rglob('*')): + raise ValueError('plugin snapshot must not follow symlinks') + catalog['plugins'][0]['source'] = {'source':'local','path':'./plugins/wigtn-plugins-with-codex'} + target = destination/'.agents/plugins/marketplace.json' + target.parent.mkdir(parents=True) + target.write_text(json.dumps(catalog, indent=2)+'\n') + shutil.copytree(plugin, destination/'plugins/wigtn-plugins-with-codex', + ignore=shutil.ignore_patterns('__pycache__','.DS_Store')) + +def digest_tree(root): + return {str(p.relative_to(root)): hashlib.sha256(p.read_bytes()).hexdigest() + for p in sorted(root.rglob('*')) if p.is_file() and '__pycache__' not in p.parts and p.name != '.DS_Store'} + +def invoke(command, env, cwd, log, timeout=240, stdin=None): + start = time.monotonic() + with log.open('w') as out, log.with_suffix('.stderr').open('w') as err: + try: + result = subprocess.run(command, env=env, cwd=cwd, input=stdin, text=True, + stdout=out, stderr=err, timeout=timeout) + code = result.returncode + except subprocess.TimeoutExpired: + code = 124 + return {'exit_code': code, 'duration_seconds': round(time.monotonic()-start, 3)} + +def main(): + ap=argparse.ArgumentParser(description=__doc__) + ap.add_argument('--baseline', type=Path, required=True) + ap.add_argument('--candidate', type=Path, default=Path(__file__).resolve().parents[1]) + ap.add_argument('--root', type=Path, required=True) + ap.add_argument('--model', default='gpt-6-astra') + ap.add_argument('--effort', default='medium', choices=['low','medium','high','xhigh','max']) + ap.add_argument('--repeat', type=int, default=2) + ap.add_argument('--suite', choices=list(SUITES), default='smoke', + help='smoke: 24; targeted/contracts: 30 calls at repeat=2; prints plan unless --execute') + ap.add_argument('--seed', type=int, default=907) + ap.add_argument('--codex', default=os.environ.get('CODEX_BIN') or shutil.which('codex') or '/Applications/ChatGPT.app/Contents/Resources/codex') + ap.add_argument('--execute', action='store_true') + args=ap.parse_args() + args.baseline=args.baseline.resolve() + args.candidate=args.candidate.resolve() + args.root=args.root.resolve() + if args.repeat < 1: ap.error('--repeat must be positive') + if args.root.exists(): ap.error('--root must be fresh') + for source in (args.baseline,args.candidate): + if not (source/'.agents/plugins/marketplace.json').is_file(): ap.error(f'not a marketplace: {source}') + if source.resolve() in args.root.resolve().parents: ap.error('output must be outside source') + case_ids = SUITES[args.suite] + schedule=[]; rng=random.Random(args.seed) + for rep in range(1,args.repeat+1): + for case in case_ids: + arms=['bare','baseline','candidate']; rng.shuffle(arms) + for arm in arms: schedule.append({'arm':arm,'case':case,'repeat':rep}) + plan={'model':args.model,'effort':args.effort,'seed':args.seed,'calls':len(schedule), + 'suite':args.suite,'schedule':schedule,'prompts':{c:CASES[c] for c in case_ids}, + 'treatment':'delivery adds explicit verified-delivery invocation to baseline/candidate only', + 'schema_version':2, + 'review_rubric':{c:RUBRIC[c] for c in case_ids}, + 'activation_required':{c:ACTIVATION[c] for c in case_ids if c in ACTIVATION}, + 'pricing':'not estimated; CLI usage is not a bill'} + if not args.execute: + print(json.dumps(plan,ensure_ascii=False,indent=2));return + auth=Path(os.environ.get('CODEX_AUTH_FILE',str(Path.home()/'.codex/auth.json'))) + if not auth.is_file(): ap.error('Codex authentication unavailable') + args.root.mkdir(parents=True) + plan['cli']=subprocess.check_output([args.codex,'--version'],text=True).strip() + plan['created_utc']=time.strftime('%Y-%m-%dT%H:%M:%SZ',time.gmtime()) + evaluator=args.root/'evaluator';evaluator.mkdir() + for name in ['run-model-migration-eval.py','score-model-migration-eval.py','codex_usage.py','migration_evidence.py','migration_cases.py']: + shutil.copy2(Path(__file__).with_name(name),evaluator/name) + plan['evaluator_hashes']=digest_tree(evaluator) + plan['python']=sys.version + plan['snapshots']={} + for arm,src in [('baseline',args.baseline),('candidate',args.candidate)]: + dest=args.root/'snapshots'/arm + snapshot_marketplace(src,dest) + plan['snapshots'][arm]=digest_tree(dest) + (args.root/'protocol.json').write_text(json.dumps(plan,ensure_ascii=False,indent=2)+'\n') + seal(args.root, ['protocol.json', 'evaluator', 'snapshots'], 'input-manifest.json') + common=[args.codex,'--disable','remote_plugin','--disable','apps'] + envs={} + for arm in ['bare','baseline','candidate']: + home=args.root/'homes'/arm;home.mkdir(parents=True) + (home/'auth.json').symlink_to(auth.resolve()) + envs[arm]={**os.environ,'CODEX_HOME':str(home)} + if arm!='bare': + for label,cmd in [('marketplace',['plugin','marketplace','add',str(args.root/'snapshots'/arm),'--json']), + ('plugin',['plugin','add','wigtn-plugins-with-codex@wigtn','--json'])]: + meta=invoke(common+cmd,envs[arm],args.root,args.root/f'{arm}-{label}.json') + if meta['exit_code']: raise SystemExit(f'{arm} {label} failed: {meta}') + probe=args.root/'probe'/arm;probe.mkdir(parents=True) + log=args.root/f'{arm}-prompt.json' + meta=invoke(common+['-C',str(probe),'debug','prompt-input','Write a PRD'],envs[arm],probe,log) + exposed='wigtn-plugins-with-codex:product-spec' in log.read_text() + if meta['exit_code'] or exposed != (arm!='bare'): raise SystemExit(f'catalog isolation failed: {arm}') + for order,item in enumerate(schedule,1): + arm,case,rep=item['arm'],item['case'],item['repeat'] + run=args.root/'runs'/f'{order:03}-{arm}-{case}-{rep}';work=run/'work';work.mkdir(parents=True) + if case in CODE_CASES: + (work/'scores.py').write_text('def sorted_scores(scores):\n scores.sort()\n return scores\n') + (work/'test_scores.py').write_text('import unittest\nfrom scores import sorted_scores\nclass Scores(unittest.TestCase):\n def test_sort(self):\n self.assertEqual(sorted_scores([3,1,2]),[1,2,3])\n def test_preserve(self):\n values=[3,1,2]; sorted_scores(values); self.assertEqual(values,[3,1,2])\n') + (work/'notes.txt').write_text('User draft: preserve this file.\n') + before=digest_tree(work) + if case in CODE_CASES: + precheck=check_implementation(work,run/'oracle-before.txt') + if precheck['exit_code'] != 1: + raise SystemExit('fault fixture must fail the independent oracle before execution') + prompt=CASES[case] + if case=='delivery' and arm!='bare': + prompt='Use $wigtn-plugins-with-codex:verified-delivery. '+prompt + (run/'prompt.txt').write_text(prompt) + meta=invoke(common+['-a','never','-m',args.model,'-c',f'model_reasoning_effort="{args.effort}"', + '-s','workspace-write' if case in CODE_CASES else 'read-only','-C',str(work), + 'exec','--ephemeral','--ignore-rules','--skip-git-repo-check','--json', + '-o',str(run/'answer.md'),'-'],envs[arm],work,run/'events.jsonl',stdin=prompt) + meta.update(item);meta.update(order=order,model=args.model,effort=args.effort) + meta['before_hashes']=before + if case in CODE_CASES: + check=check_implementation(work,run/'oracle.txt') + meta['oracle_before']=precheck + after=digest_tree(work) + meta['after_hashes']=after + meta['unexpected_changes']=sorted(p for p in set(before)|set(after) + if before.get(p)!=after.get(p) and p!='scores.py') + meta['oracle']=check + meta['draft_preserved']=(work/'notes.txt').is_file() and (work/'notes.txt').read_text()=='User draft: preserve this file.\n' + meta['unsolicited_state']=any((work/p).exists() for p in ['.wigtn','PRD.md','docs/PRD.md']) + (run/'meta.json').write_text(json.dumps(meta,indent=2)+'\n') + seal_run(run) + print(f'{order}/{len(schedule)} {arm}/{case} exit={meta["exit_code"]} {meta["duration_seconds"]}s',flush=True) + if meta['exit_code']: + # Retain failed run; stop before wasting the rest of the model budget. + raise SystemExit('Incomplete pilot: inspect retained failure before starting a new run') + print(args.root,flush=True) +if __name__=='__main__': main() diff --git a/scripts/run-ordinary-gate.sh b/scripts/run-ordinary-gate.sh index 836da7d..1cc5e99 100644 --- a/scripts/run-ordinary-gate.sh +++ b/scripts/run-ordinary-gate.sh @@ -2,7 +2,7 @@ set -euo pipefail repo_root="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" -model="${WIGTN_EVAL_MODEL:-gpt-5.6-sol}" +model="${WIGTN_EVAL_MODEL:-gpt-6-astra}" effort="${WIGTN_EVAL_EFFORT:-medium}" repeat="${WIGTN_EVAL_REPEAT:-1}" concurrency="${WIGTN_EVAL_CONCURRENCY:-2}" diff --git a/scripts/run-package-ablation.sh b/scripts/run-package-ablation.sh index ee47a87..9cc0789 100755 --- a/scripts/run-package-ablation.sh +++ b/scripts/run-package-ablation.sh @@ -2,7 +2,7 @@ set -euo pipefail repo_root="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" -model="${WIGTN_EVAL_MODEL:-gpt-5.6-sol}" +model="${WIGTN_EVAL_MODEL:-gpt-6-astra}" effort="${WIGTN_EVAL_EFFORT:-medium}" repeat="${WIGTN_EVAL_REPEAT:-2}" run_root="${WIGTN_ABLATION_ROOT:-/tmp/wigtn-package-ablation-$model}" diff --git a/scripts/run-static-contracts.sh b/scripts/run-static-contracts.sh index 88937e4..e748c20 100755 --- a/scripts/run-static-contracts.sh +++ b/scripts/run-static-contracts.sh @@ -13,14 +13,27 @@ env PYTHONDONTWRITEBYTECODE=1 python3 "$repo_root/scripts/check-prd-contract.py" env PYTHONDONTWRITEBYTECODE=1 python3 "$repo_root/scripts/check-screen-contract.py" env PYTHONDONTWRITEBYTECODE=1 python3 "$repo_root/scripts/check-release-state.py" env PYTHONDONTWRITEBYTECODE=1 python3 "$repo_root/scripts/check-requirement-import.py" +python3 "$repo_root/scripts/check-symbolic-requirement-ids.py" env PYTHONDONTWRITEBYTECODE=1 python3 "$repo_root/scripts/check-workgraph-contract.py" env PYTHONDONTWRITEBYTECODE=1 python3 "$repo_root/scripts/check-eval-schedule.py" env PYTHONDONTWRITEBYTECODE=1 python3 "$repo_root/scripts/check-ablation-builder.py" env PYTHONDONTWRITEBYTECODE=1 python3 "$repo_root/scripts/check-ordinary-corpus.py" env PYTHONDONTWRITEBYTECODE=1 python3 "$repo_root/scripts/check-ordinary-scorer.py" +env PYTHONDONTWRITEBYTECODE=1 python3 "$repo_root/scripts/check-codex-usage.py" +python3 "$repo_root/scripts/check-codex-usage-missing.py" +env PYTHONDONTWRITEBYTECODE=1 python3 "$repo_root/scripts/check-token-ablation.py" env PYTHONDONTWRITEBYTECODE=1 python3 "$repo_root/scripts/check-eval-packet.py" env PYTHONDONTWRITEBYTECODE=1 python3 "$repo_root/scripts/check-research-harness.py" env PYTHONDONTWRITEBYTECODE=1 python3 \ "$repo_root/scripts/summarize-package-ablation.py" --help >/dev/null +env PYTHONDONTWRITEBYTECODE=1 python3 "$repo_root/scripts/check-model-migration-eval.py" + +env PYTHONDONTWRITEBYTECODE=1 python3 "$repo_root/scripts/check-workgraph-inspect.py" + +env PYTHONDONTWRITEBYTECODE=1 python3 "$repo_root/scripts/check-workgraph-import-root.py" + + +env PYTHONDONTWRITEBYTECODE=1 python3 "$repo_root/scripts/check-workgraph-concurrency.py" +env PYTHONDONTWRITEBYTECODE=1 python3 "$repo_root/scripts/check-workgraph-project-guard.py" echo "Static contracts: PASS" diff --git a/scripts/run-token-ablation.sh b/scripts/run-token-ablation.sh new file mode 100755 index 0000000..4e76ee7 --- /dev/null +++ b/scripts/run-token-ablation.sh @@ -0,0 +1,110 @@ +#!/usr/bin/env bash +set -euo pipefail + +repo_root="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +model="${WIGTN_EVAL_MODEL:-gpt-5.6-sol}" +effort="${WIGTN_EVAL_EFFORT:-medium}" +repeat="${WIGTN_EVAL_REPEAT:-3}" +run_root="${WIGTN_TOKEN_ABLATION_ROOT:-/tmp/wigtn-token-ablation-$model}" +baseline_ref="${WIGTN_BASELINE_REF:-HEAD}" +seed="${WIGTN_EVAL_SEED:-token-ablation-v1}" +cases_file="$repo_root/tests/behavior/token-efficiency-cases.tsv" +auth_file="${CODEX_AUTH_FILE:-$HOME/.codex/auth.json}" +codex_bin="${CODEX_BIN:-/Applications/ChatGPT.app/Contents/Resources/codex}" +arms=(baseline candidate) + +if [[ "${1:-}" != "--execute" ]]; then + cat <&2; exit 2; } + +[[ "$repeat" =~ ^[1-9][0-9]*$ ]] || { echo "repeat must be positive" >&2; exit 2; } +[[ -x "$codex_bin" && -f "$auth_file" ]] || { echo "Codex CLI or auth unavailable" >&2; exit 2; } +[[ ! -e "$run_root" ]] || { echo "choose a fresh WIGTN_TOKEN_ABLATION_ROOT" >&2; exit 2; } +baseline_commit="$(git -C "$repo_root" rev-parse "$baseline_ref^{commit}")" + +mkdir -p "$run_root/staging/baseline" "$run_root/runs" "$run_root/prompt-input" +git -C "$repo_root" archive "$baseline_commit" | tar -x -C "$run_root/staging/baseline" +env PYTHONDONTWRITEBYTECODE=1 python3 "$repo_root/scripts/make-eval-schedule.py" \ + "$cases_file" --arms "${arms[@]}" --repeat "$repeat" --seed "$seed" \ + --output "$run_root/SCHEDULE.tsv" + +marketplace_for() { + [[ "$1" == baseline ]] && printf '%s' "$run_root/staging/baseline" || printf '%s' "$repo_root" +} + +for arm in "${arms[@]}"; do + mkdir -p "$run_root/$arm-home" "$run_root/$arm-work" "$run_root/runs/$arm" + ln -s "$auth_file" "$run_root/$arm-home/auth.json" + CODEX_HOME="$run_root/$arm-home" "$codex_bin" --disable remote_plugin --disable apps \ + plugin marketplace add "$(marketplace_for "$arm")" --json \ + > "$run_root/runs/$arm/setup-marketplace.json" + CODEX_HOME="$run_root/$arm-home" "$codex_bin" --disable remote_plugin --disable apps \ + plugin add wigtn-plugins-with-codex@wigtn --json \ + > "$run_root/runs/$arm/setup-plugin.json" + CODEX_HOME="$run_root/$arm-home" "$codex_bin" --disable remote_plugin --disable apps \ + -C "$run_root/$arm-work" debug prompt-input "간결한 PRD를 작성해줘" \ + > "$run_root/prompt-input/$arm.json" + grep -q 'wigtn-plugins-with-codex:product-spec' "$run_root/prompt-input/$arm.json" || { + echo "$arm does not expose product-spec" >&2 + exit 2 + } +done + +{ + printf 'created_utc=%s\n' "$(date -u +%Y-%m-%dT%H:%M:%SZ)" + printf 'codex_cli=%s\nmodel=%s\neffort=%s\nrepeat=%s\n' \ + "$("$codex_bin" --version)" "$model" "$effort" "$repeat" + printf 'baseline_commit=%s\nschedule_seed=%s\n' "$baseline_commit" "$seed" + shasum -a 256 "$run_root/SCHEDULE.tsv" "$cases_file" \ + "$repo_root/tests/behavior/prompts/prd-create.txt" \ + "$repo_root/tests/behavior/prompts/acceptance-uncertain.txt" + find "$repo_root/plugins/wigtn-plugins-with-codex" -type f -print0 | \ + LC_ALL=C sort -z | xargs -0 shasum -a 256 +} > "$run_root/MANIFEST.txt" + +run_one() { + local pair_id="$1" order="$2" arm="$3" case_id="$4" prompt_path="$5" rep="$6" + local stem="$run_root/runs/$arm/$case_id.$rep" started rc + started="$(date +%s)" + set +e + CODEX_HOME="$run_root/$arm-home" "$codex_bin" --disable remote_plugin --disable apps \ + -a never -m "$model" -c "model_reasoning_effort=\"$effort\"" \ + -s read-only -C "$run_root/$arm-work" exec --ephemeral --ignore-rules \ + --skip-git-repo-check --json -o "$stem.out.md" - \ + < "$repo_root/$prompt_path" > "$stem.events.jsonl" 2> "$stem.log" + rc=$? + set -e + env PYTHONDONTWRITEBYTECODE=1 python3 - "$stem.meta.json" "$pair_id" "$order" \ + "$arm" "$case_id" "$rep" "$rc" "$(( $(date +%s) - started ))" <<'PY' +import json +from pathlib import Path +import sys +Path(sys.argv[1]).write_text(json.dumps({ + "pair_id": sys.argv[2], "schedule_order": int(sys.argv[3]), + "arm": sys.argv[4], "case": sys.argv[5], "repeat": int(sys.argv[6]), + "exit_code": int(sys.argv[7]), "duration_seconds": int(sys.argv[8]), +}, indent=2) + "\n") +PY +} + +while IFS=$'\t' read -r pair_id order arm case_id prompt_path rep_index; do + [[ "$pair_id" != pair_id ]] || continue + run_one "$pair_id" "$order" "$arm" "$case_id" "$prompt_path" "$rep_index" +done < "$run_root/SCHEDULE.tsv" + +env PYTHONDONTWRITEBYTECODE=1 python3 "$repo_root/scripts/summarize-token-efficiency.py" "$run_root" +env PYTHONDONTWRITEBYTECODE=1 python3 "$repo_root/scripts/score-token-ablation.py" "$run_root" diff --git a/scripts/run-workgraph-pilot.sh b/scripts/run-workgraph-pilot.sh index bf3dd72..771365e 100755 --- a/scripts/run-workgraph-pilot.sh +++ b/scripts/run-workgraph-pilot.sh @@ -2,7 +2,7 @@ set -euo pipefail repo_root="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" -model="${WIGTN_EVAL_MODEL:-gpt-5.6-sol}" +model="${WIGTN_EVAL_MODEL:-gpt-6-astra}" effort="${WIGTN_EVAL_EFFORT:-medium}" concurrency="${WIGTN_EVAL_CONCURRENCY:-3}" run_root="${WIGTN_WORKGRAPH_PILOT_ROOT:-/tmp/wigtn-workgraph-pilot-$model}" diff --git a/scripts/score-behavior-smoke.py b/scripts/score-behavior-smoke.py index 490456c..da5b171 100755 --- a/scripts/score-behavior-smoke.py +++ b/scripts/score-behavior-smoke.py @@ -9,6 +9,8 @@ import sys import csv +from codex_usage import read_events + FORBIDDEN_IA_HEADINGS = re.compile( r"^#{1,6}\s*(?:user flow|사용자 흐름|screen spec|화면 명세|" @@ -59,6 +61,12 @@ def quality_failures(arm: str, case: str, output: str) -> list[str]: def total_tokens(path: Path) -> int | None: + events_path = path.with_name(path.name.replace(".log", ".events.jsonl")) + if events_path.is_file(): + try: + return int(read_events(events_path)["output_tokens"]) + except ValueError: + return None text = path.read_text(encoding="utf-8", errors="ignore") matches = re.findall(r"tokens used\s*\n([\d,]+)", text, re.IGNORECASE) return int(matches[-1].replace(",", "")) if matches else None @@ -95,6 +103,15 @@ def main(root_arg: str) -> int: nonempty = bool(output.strip()) log_path = meta_path.with_name(meta_path.name.replace(".meta.json", ".log")) tokens = total_tokens(log_path) + events_path = meta_path.with_name( + meta_path.name.replace(".meta.json", ".events.jsonl") + ) + tool_items = None + if events_path.is_file(): + try: + tool_items = int(read_events(events_path)["tool_items"]) + except ValueError as error: + errors.append(str(error)) row_key = (meta["arm"], meta["case"], meta["repeat"]) if row_key in observed: errors.append(f"duplicate run metadata: {row_key}") @@ -128,6 +145,11 @@ def main(root_arg: str) -> int: f"exit={meta['exit_code']}, nonempty={nonempty}" ) token_by_run[row_key] = tokens + if meta["case"] in {"acceptance-uncertain", "ia-only"} and tool_items: + errors.append( + f"{meta['arm']}/{meta['case']}.{meta['repeat']}: " + f"prompt prohibited commands but observed {tool_items} tool item(s)" + ) for failure in quality_failures(meta["arm"], meta["case"], output): errors.append( f"{meta['arm']}/{meta['case']}.{meta['repeat']}: {failure}" @@ -161,7 +183,7 @@ def main(root_arg: str) -> int: "token gate. It does not establish causal plugin lift or real-repository " "generalization.", "", - "| Pair | Order | Arm | Case | Repeat | Exit | Output | Total tokens | Output bytes | Duration |", + "| Pair | Order | Arm | Case | Repeat | Exit | Output | Output tokens | Output bytes | Duration |", "|---|---:|---|---|---:|---:|---|---:|---:|---:|", ] report.extend( diff --git a/scripts/score-model-migration-eval.py b/scripts/score-model-migration-eval.py new file mode 100644 index 0000000..2a15a5e --- /dev/null +++ b/scripts/score-model-migration-eval.py @@ -0,0 +1,246 @@ +#!/usr/bin/env python3 +"""Verify sealed evidence and report all-attempt consumption separately from quality.""" +import argparse +import hashlib +import json +import math +from pathlib import Path + +from codex_usage import read_events +from migration_evidence import verify, verify_run + + +def file_hash(path): + return hashlib.sha256(path.read_bytes()).hexdigest() if path.is_file() and not path.is_symlink() else None + + +def telemetry(path): + events = [json.loads(line) for line in path.read_text().splitlines() if line.strip()] + completed = [event for event in events if event.get('type') == 'turn.completed'] + if len(completed) != 1: + raise ValueError('expected one completed usage event; consumption unknown') + values = completed[0].get('usage') or {} + for field in ['input_tokens', 'output_tokens']: + if type(values.get(field)) is not int or values[field] < 0: + raise ValueError(f'invalid/missing {field}') + for field in ['cached_input_tokens', 'cache_write_input_tokens', 'reasoning_output_tokens']: + if field in values and (type(values[field]) is not int or values[field] < 0): + raise ValueError(f'invalid {field}') + usage = read_events(path) + for field in ['cached_input_tokens', 'cache_write_input_tokens', 'reasoning_output_tokens']: + if field not in values: + usage[field] = None + if usage['reasoning_output_tokens'] is None: + usage['visible_output_tokens'] = None + if any(usage[field] is None for field in ['cached_input_tokens', 'cache_write_input_tokens']): + usage['uncached_input_tokens'] = None + return usage, events + + +def assess_review(run, row, review, protocol, events): + criteria = protocol['review_rubric'].get(row['case'], []) + required_skill = protocol.get('activation_required', {}).get(row['case']) if row['arm'] != 'bare' else None + if review is None: + return 'pending', 'pending' if required_skill else 'not-required', [] + errors = [] + if not isinstance(review, dict): + return 'pending', 'pending', ['review must be an object'] + for field, file in [('answer_sha256', 'answer.md'), ('events_sha256', 'events.jsonl')]: + if review.get(field) != file_hash(run / file): + errors.append(f'{field} mismatch') + if not isinstance(review.get('reviewer'), str) or not review['reviewer'].strip(): + errors.append('reviewer required') + decisions = review.get('criteria', {}) + if not isinstance(decisions, dict) or set(decisions) != set(criteria) or not criteria: + errors.append('review must cover exactly the declared criteria') + decisions = {} + for criterion, decision in decisions.items(): + if (not isinstance(decision, dict) or decision.get('verdict') not in {'pass', 'fail'} + or not isinstance(decision.get('evidence'), str) or not decision['evidence'].strip()): + errors.append(f'{criterion}: verdict and evidence required') + activation = 'not-required' + if required_skill: + claim = review.get('activation') or {} + activation = claim.get('status', 'pending') + if activation not in {'confirmed', 'not-activated'}: + errors.append('activation adjudication required') + if activation == 'confirmed': + lines = claim.get('event_lines', []) + if not isinstance(lines, list) or not lines: + errors.append('activation needs trace event lines') + else: + for line in lines: + if type(line) is not int or not 1 <= line <= len(events): + errors.append('activation line outside trace') + continue + event = events[line - 1] + item = event.get('item') or {} + source = (run.parent.parent / 'snapshots' / row['arm'] / 'plugins' / + 'wigtn-plugins-with-codex' / 'skills' / required_skill / 'SKILL.md') + output = item.get('aggregated_output', '') + success = item.get('type') == 'command_execution' and item.get('exit_code') == 0 + if item.get('type') == 'mcp_tool_call': + result = item.get('result') or {} + success = isinstance(result, dict) and not result.get('isError') and not item.get('error') + output = '\n'.join(c.get('text', '') for c in result.get('content', []) + if isinstance(c, dict)) if isinstance(result, dict) else '' + if (not success or not source.is_file() or not isinstance(output, str) + or source.read_text().strip() not in output): + errors.append('activation requires successful tool output containing the snapshotted skill body') + if not isinstance(claim.get('evidence'), str) or not claim['evidence'].strip(): + errors.append('activation explanation required') + if errors: + return 'pending', activation, errors + status = 'fail' if any(d['verdict'] == 'fail' for d in decisions.values()) else 'pass' + return status, activation, [] + + +def score(root, reviews_path=None, write_template=False): + integrity_errors = verify(root, ['protocol.json', 'evaluator', 'snapshots'], 'input-manifest.json') + protocol = json.loads((root / 'protocol.json').read_text()) + schedule = protocol['schedule'] + expected = {(i['arm'], i['case'], i['repeat']) for i in schedule} + if not expected or len(expected) != len(schedule) or protocol.get('calls') != len(schedule): + integrity_errors.append('empty/duplicate schedule or call count mismatch') + if protocol.get('schema_version') != 2 or not isinstance(protocol.get('review_rubric'), dict): + integrity_errors.append('schema_version 2 and review_rubric required') + input_valid = not integrity_errors + reviews = json.loads(reviews_path.read_text())['runs'] if reviews_path else {} + rows, seen, template, review_errors = [], set(), {}, [] + for run in sorted((root / 'runs').iterdir() if (root / 'runs').exists() else []): + if not run.is_dir(): + continue + errors = verify_run(run) + try: + row = json.loads((run / 'meta.json').read_text()) + key = (row['arm'], row['case'], row['repeat']) + except (OSError, ValueError, KeyError, TypeError) as error: + integrity_errors.append(f'{run.name}: invalid metadata: {error}') + continue + if key in seen or key not in expected: + errors.append('duplicate or unscheduled run') + seen.add(key) + if row.get('model') != protocol['model'] or row.get('effort') != protocol['effort']: + errors.append('model/effort mismatch') + duration = row.get('duration_seconds') + if type(duration) not in {int, float} or not math.isfinite(duration) or duration < 0: + errors.append('invalid duration') + row.update(run_id=run.name, evidence_valid=input_valid and not errors, usage=None, + telemetry_error=None, semantic_review='pending', activation='pending') + events = [] + if row['evidence_valid']: + try: + row['usage'], events = telemetry(run / 'events.jsonl') + except (OSError, ValueError, TypeError, KeyError) as error: + row['telemetry_error'] = str(error) + row['execution_passed'] = bool(row['evidence_valid'] and row.get('exit_code') == 0 + and row['usage'] is not None and (run / 'answer.md').is_file() + and (run / 'answer.md').read_text().strip() + and not any(e.get('type') == 'turn.failed' for e in events)) + row['oracle_passed'] = None + if row['case'] in {'ordinary', 'delivery'}: + row['oracle_passed'] = bool(row.get('oracle_before', {}).get('exit_code') == 1 + and row.get('oracle', {}).get('exit_code') == 0 + and row.get('draft_preserved') and not row.get('unsolicited_state') + and row.get('unexpected_changes') == []) + if row['evidence_valid']: + status, activation, issues = assess_review(run, row, reviews.get(run.name), protocol, events) + row.update(semantic_review=status, activation=activation) + review_errors.extend(f'{run.name}: {issue}' for issue in issues) + row['task_passed'] = bool(row['execution_passed'] and row['oracle_passed'] is not False + and row['semantic_review'] == 'pass' + and row['activation'] in {'confirmed', 'not-required'}) + template[run.name] = { + 'reviewer': '', 'answer_sha256': file_hash(run / 'answer.md'), + 'events_sha256': file_hash(run / 'events.jsonl'), + 'criteria': {criterion: {'verdict': 'pending', 'evidence': ''} + for criterion in protocol.get('review_rubric', {}).get(row['case'], [])}, + 'activation': {'status': 'pending', 'event_lines': [], 'evidence': ''}, + } + integrity_errors.extend(f'{run.name}: {error}' for error in errors) + rows.append(row) + missing = expected - seen + schedule_errors = [f'missing scheduled run: {key}' for key in sorted(missing)] + review_errors.extend(f'unknown review run: {key}' for key in sorted(set(reviews) - set(template))) + schedule_complete = seen == expected and len(rows) == len(expected) and bool(expected) + # If schedule identity or shared evidence is corrupt, no aggregate is trusted. + trusted_packet = not integrity_errors + groups = [] + for arm in ['bare', 'baseline', 'candidate']: + for case in sorted({key[1] for key in expected}): + expected_count = sum(key[0] == arm and key[1] == case for key in expected) + cell = [r for r in rows if r['arm'] == arm and r['case'] == case] + valid = [r for r in cell if r['evidence_valid']] if trusted_packet else [] + measured = [r for r in valid if r['usage'] is not None] + successes = sum(r['task_passed'] for r in valid) + consumption_complete = len(measured) == expected_count + output = sum(r['usage']['output_tokens'] for r in measured) + groups.append({ + 'arm': arm, 'case': case, 'scheduled': expected_count, 'recorded': len(cell), + 'execution_failures': sum(not r['execution_passed'] for r in valid), + 'oracle_failures': sum(r['oracle_passed'] is False for r in valid), + 'review_pending': sum(r['semantic_review'] == 'pending' for r in valid), + 'task_successes': successes, 'telemetry_records': len(measured), + 'consumption_complete': consumption_complete, + 'observed_output_tokens': output, + 'all_attempt_output_tokens': output if consumption_complete else None, + 'all_attempt_input_tokens': sum(r['usage']['input_tokens'] for r in measured) if consumption_complete else None, + 'all_attempt_seconds': sum(r['duration_seconds'] for r in valid) if len(valid) == expected_count else None, + 'output_tokens_per_success': output / successes if consumption_complete and successes else None, + }) + lookup = {(r['arm'], r['case'], r['repeat']): r for r in rows} + pairs = [] + if trusted_packet: + for arm, case, repeat in sorted(expected): + if arm != 'candidate': + continue + baseline = lookup.get(('baseline', case, repeat)) + candidate = lookup.get(('candidate', case, repeat)) + if baseline and candidate and baseline['usage'] is not None and candidate['usage'] is not None: + pairs.append({'case': case, 'repeat': repeat, + 'output_tokens_delta': candidate['usage']['output_tokens'] - baseline['usage']['output_tokens'], + 'seconds_delta': candidate['duration_seconds'] - baseline['duration_seconds'], + 'baseline_task_passed': baseline['task_passed'], + 'candidate_task_passed': candidate['task_passed']}) + review_complete = bool(rows) and not review_errors and all(r['semantic_review'] != 'pending' for r in rows) + complete = schedule_complete and trusted_packet and review_complete + result = { + 'schema_version': 2, 'schedule_complete': schedule_complete, + 'integrity_passed': trusted_packet, 'review_complete': review_complete, + 'execution_passed': bool(rows) and all(r['execution_passed'] for r in rows), + 'complete': complete, 'passed': complete and all(r['task_passed'] for r in rows), + 'integrity_errors': integrity_errors, 'schedule_errors': schedule_errors, 'review_errors': review_errors, + 'groups': groups, 'paired_all_attempt_deltas': pairs, 'runs': rows, 'pricing': 'not estimated', + 'quality_claim': 'No general performance claim; development cases with explicit review provenance.', + } + (root / 'results.json').write_text(json.dumps(result, indent=2, allow_nan=False) + '\n') + lines = ['# Migration evaluation', '', + f"Integrity: {trusted_packet}; schedule complete: {schedule_complete}; review complete: {review_complete}; passed: {result['passed']}", + '', 'All-attempt consumption includes behavioral failures. Unknown totals are null, not zero.', '', + '| Arm | Case | Scheduled | Records | Execution failures | Oracle failures | Pending review | Successes | All output | Seconds |', + '|---|---|---:|---:|---:|---:|---:|---:|---:|---:|'] + for g in groups: + lines.append('| ' + ' | '.join(str(g[k]) for k in ['arm', 'case', 'scheduled', 'recorded', + 'execution_failures', 'oracle_failures', 'review_pending', 'task_successes', + 'all_attempt_output_tokens', 'all_attempt_seconds']) + ' |') + lines += ['', '## Evidence and review issues', ''] + ['- ' + error for error in integrity_errors + schedule_errors + review_errors] + lines += ['', 'Review results are evaluator judgments, not proof of broad model performance. Activation needs a trace of reading the skill, not a catalog name.'] + (root / 'results.md').write_text('\n'.join(lines) + '\n') + if write_template: + if not trusted_packet: + raise ValueError('cannot export review template for corrupt evidence') + with (root / 'reviews.template.json').open('x') as handle: + json.dump({'runs': template}, handle, indent=2) + return result + + +if __name__ == '__main__': + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument('root', type=Path) + parser.add_argument('--reviews', type=Path) + parser.add_argument('--review-template', action='store_true') + parser.add_argument('--require-pass', action='store_true') + args = parser.parse_args() + result = score(args.root, args.reviews, args.review_template) + print(json.dumps({key: result[key] for key in ['integrity_passed', 'schedule_complete', 'review_complete', 'passed']})) + raise SystemExit(0 if (result['passed'] if args.require_pass else (result['integrity_passed'] and result['schedule_complete'])) else 1) diff --git a/scripts/score-token-ablation.py b/scripts/score-token-ablation.py new file mode 100755 index 0000000..52b7486 --- /dev/null +++ b/scripts/score-token-ablation.py @@ -0,0 +1,167 @@ +#!/usr/bin/env python3 +"""Score a focused baseline-versus-candidate token-efficiency ablation.""" + +from __future__ import annotations + +import json +from pathlib import Path +import re +import statistics +import sys +import csv + +from codex_usage import api_equivalent_cost, cold_api_equivalent_cost, read_events + + +RATES = { + "input_per_million": 4.0, + "cached_input_per_million": 0.4, + "cache_write_per_million": 5.0, + "output_per_million": 20.0, +} + + +def median(rows: list[dict[str, object]], field: str) -> float: + return statistics.median(float(row[field]) for row in rows) + + +def main(root_arg: str) -> int: + root = Path(root_arg) + rows: list[dict[str, object]] = [] + failures: list[str] = [] + schedule_path = root / "SCHEDULE.tsv" + expected: set[tuple[str, str, int]] = set() + if not schedule_path.is_file(): + failures.append("SCHEDULE.tsv is missing") + else: + with schedule_path.open(encoding="utf-8", newline="") as handle: + for item in csv.DictReader(handle, delimiter="\t"): + key = (item["arm"], item["case"], int(item["repeat"])) + if key in expected: + failures.append(f"duplicate schedule row: {key}") + expected.add(key) + for meta_path in sorted((root / "runs").glob("*/*.meta.json")): + meta = json.loads(meta_path.read_text(encoding="utf-8")) + stem = meta_path.name.removesuffix(".meta.json") + output = meta_path.with_name(f"{stem}.out.md").read_text( + encoding="utf-8", errors="replace" + ) + usage = read_events(meta_path.with_name(f"{stem}.events.jsonl")) + contract = True + if meta["case"] == "prd-create": + contract = all( + marker in output + for marker in ("wigtn-prd-profile: compact", "FR-", "AC-") + ) + row = { + **meta, + **usage, + "contract": contract, + "exact_status": bool(re.search(r"\bnot-verifiable\b", output, re.I)), + "api_equivalent_usd": api_equivalent_cost(usage, **RATES), + "cold_api_equivalent_usd": cold_api_equivalent_cost( + usage, + input_per_million=RATES["input_per_million"], + output_per_million=RATES["output_per_million"], + ), + } + rows.append(row) + if meta["exit_code"] != 0 or not output.strip(): + failures.append( + f"{meta['arm']}/{meta['case']}.{meta['repeat']}: unhealthy execution" + ) + + selected: dict[tuple[str, str], list[dict[str, object]]] = {} + observed = { + (str(row["arm"]), str(row["case"]), int(row["repeat"])) for row in rows + } + for missing in sorted(expected - observed): + failures.append(f"scheduled run missing: {missing}") + for extra in sorted(observed - expected): + failures.append(f"run absent from schedule: {extra}") + for row in rows: + selected.setdefault((str(row["arm"]), str(row["case"])), []).append(row) + for key in ( + ("baseline", "prd-create"), + ("baseline", "acceptance-uncertain"), + ("candidate", "prd-create"), + ("candidate", "acceptance-uncertain"), + ): + if not selected.get(key): + failures.append(f"missing cell: {key[0]}/{key[1]}") + + candidate_prd = selected.get(("candidate", "prd-create"), []) + baseline_prd = selected.get(("baseline", "prd-create"), []) + candidate_acceptance = selected.get(("candidate", "acceptance-uncertain"), []) + if candidate_prd and not all(bool(row["contract"]) for row in candidate_prd): + failures.append("candidate PRD contract regression") + if candidate_acceptance and not all( + bool(row["exact_status"]) for row in candidate_acceptance + ): + failures.append("candidate acceptance omitted exact not-verifiable") + if candidate_acceptance and any( + int(row["tool_items"]) for row in candidate_acceptance + ): + failures.append("candidate acceptance executed a prohibited tool") + if candidate_prd and baseline_prd: + if median(candidate_prd, "tool_items") >= median(baseline_prd, "tool_items"): + failures.append("candidate PRD did not reduce median tool items") + if median(candidate_prd, "input_tokens") >= median(baseline_prd, "input_tokens"): + failures.append("candidate PRD did not reduce median input tokens") + + report = [ + "# Token ablation results", + "", + "| Arm | Case | Repeat | Contract | Exact status | Input | Cached | Output | Reasoning | Tools | Cold USD |", + "|---|---|---:|---|---|---:|---:|---:|---:|---:|---:|", + ] + for row in rows: + report.append( + f"| {row['arm']} | {row['case']} | {row['repeat']} | " + f"{'yes' if row['contract'] else 'no'} | " + f"{'yes' if row['exact_status'] else 'no'} | {row['input_tokens']} | " + f"{row['cached_input_tokens']} | {row['output_tokens']} | " + f"{row['reasoning_output_tokens']} | {row['tool_items']} | " + f"${row['cold_api_equivalent_usd']:.6f} |" + ) + report.extend( + [ + "", + "## Median comparison", + "", + "| Case | Arm | Input | Output | Tools | Cold USD |", + "|---|---|---:|---:|---:|---:|", + ] + ) + for case in ("prd-create", "acceptance-uncertain"): + for arm in ("baseline", "candidate"): + cell = selected.get((arm, case), []) + if cell: + report.append( + f"| {case} | {arm} | {median(cell, 'input_tokens'):.1f} | " + f"{median(cell, 'output_tokens'):.1f} | " + f"{median(cell, 'tool_items'):.1f} | " + f"${median(cell, 'cold_api_equivalent_usd'):.6f} |" + ) + if failures: + report.extend(["", "## Failures", "", *[f"- {item}" for item in failures]]) + (root / "ABLATION-RESULTS.md").write_text("\n".join(report) + "\n", encoding="utf-8") + (root / "ABLATION-RESULTS.json").write_text( + json.dumps({"passed": not failures, "failures": failures, "runs": rows}, indent=2) + + "\n", + encoding="utf-8", + ) + if failures: + print("Token ablation: FAIL") + for failure in failures: + print(f"- {failure}") + return 1 + print(f"Token ablation: PASS ({len(rows)} runs)") + print(root / "ABLATION-RESULTS.md") + return 0 + + +if __name__ == "__main__": + if len(sys.argv) != 2: + raise SystemExit("usage: score-token-ablation.py RUN_ROOT") + raise SystemExit(main(sys.argv[1])) diff --git a/scripts/summarize-token-efficiency.py b/scripts/summarize-token-efficiency.py new file mode 100755 index 0000000..3487347 --- /dev/null +++ b/scripts/summarize-token-efficiency.py @@ -0,0 +1,133 @@ +#!/usr/bin/env python3 +"""Summarize detailed Codex usage without treating cost as quality.""" + +from __future__ import annotations + +import argparse +import json +import math +from pathlib import Path + +from codex_usage import api_equivalent_cost, cold_api_equivalent_cost, read_events + + +def main() -> int: + parser = argparse.ArgumentParser() + parser.add_argument("run_root", type=Path) + parser.add_argument("--input-per-million", type=float) + parser.add_argument("--cached-input-per-million", type=float) + parser.add_argument("--cache-write-per-million", type=float) + parser.add_argument("--output-per-million", type=float) + args = parser.parse_args() + rates = { + "input_per_million": args.input_per_million, + "cached_input_per_million": args.cached_input_per_million, + "cache_write_per_million": args.cache_write_per_million, + "output_per_million": args.output_per_million, + } + priced = all(value is not None for value in rates.values()) + if any(value is not None for value in rates.values()) and not priced: + parser.error("provide all four rates or none") + if priced and any(not math.isfinite(value) or value < 0 for value in rates.values()): + parser.error("rates must be finite and nonnegative") + rows = [] + for events_path in sorted((args.run_root / "runs").glob("*/*.events.jsonl")): + stem = events_path.name.removesuffix(".events.jsonl") + arm = events_path.parent.name + case, repeat_text = stem.rsplit(".", 1) + usage = read_events(events_path) + rows.append( + { + "arm": arm, + "case": case, + "repeat": int(repeat_text), + **usage, + "api_equivalent_usd": api_equivalent_cost(usage, **rates) if priced else None, + "cold_api_equivalent_usd": cold_api_equivalent_cost( + usage, + input_per_million=args.input_per_million, + output_per_million=args.output_per_million, + ) if priced else None, + } + ) + if not rows: + raise SystemExit("no Codex JSONL event files found") + + arms: dict[str, dict[str, float]] = {} + for row in rows: + aggregate = arms.setdefault( + row["arm"], + { + "runs": 0, + "input_tokens": 0, + "cached_input_tokens": 0, + "cache_write_input_tokens": 0, + "output_tokens": 0, + "reasoning_output_tokens": 0, + "visible_output_tokens": 0, + "tool_items": 0, + "api_equivalent_usd": 0.0, + "cold_api_equivalent_usd": 0.0, + }, + ) + aggregate["runs"] += 1 + for field in aggregate: + if field != "runs": + if row[field] is None: + aggregate[field] = None + elif aggregate[field] is not None: + aggregate[field] += row[field] + + document = {"schema_version": "1.0", "rates_usd_per_million": rates if priced else None, "runs": rows, "arms": arms} + (args.run_root / "TOKEN-EFFICIENCY.json").write_text( + json.dumps(document, indent=2, sort_keys=True) + "\n", encoding="utf-8" + ) + def money(value): + return "unknown/unpriced" if value is None else f"${value:.6f}" + + def count(value): + return "unknown" if value is None else str(int(value)) + + report = [ + "# Token efficiency", + "", + "API-equivalent cost is a comparison metric using the recorded rates; it is not a ChatGPT subscription charge.", + "", + "| Arm | Case | Repeat | Input | Cached | Cache write | Output | Reasoning | Visible | Tool items | Estimated USD | Cold USD |", + "|---|---|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|", + ] + for row in rows: + report.append( + f"| {row['arm']} | {row['case']} | {row['repeat']} | " + f"{count(row['input_tokens'])} | {count(row['cached_input_tokens'])} | " + f"{count(row['cache_write_input_tokens'])} | {count(row['output_tokens'])} | " + f"{count(row['reasoning_output_tokens'])} | {count(row['visible_output_tokens'])} | " + f"{count(row['tool_items'])} | {money(row['api_equivalent_usd'])} | " + f"{money(row['cold_api_equivalent_usd'])} |" + ) + report.extend( + [ + "", + "## Arm totals", + "", + "| Arm | Runs | Input | Cached | Output | Reasoning | Tool items | Estimated USD | Cold USD |", + "|---|---:|---:|---:|---:|---:|---:|---:|---:|", + ] + ) + for arm, row in sorted(arms.items()): + report.append( + f"| {arm} | {int(row['runs'])} | {count(row['input_tokens'])} | " + f"{count(row['cached_input_tokens'])} | {count(row['output_tokens'])} | " + f"{count(row['reasoning_output_tokens'])} | {count(row['tool_items'])} | " + f"{money(row['api_equivalent_usd'])} | " + f"{money(row['cold_api_equivalent_usd'])} |" + ) + (args.run_root / "TOKEN-EFFICIENCY.md").write_text( + "\n".join(report) + "\n", encoding="utf-8" + ) + print(args.run_root / "TOKEN-EFFICIENCY.md") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/validate.sh b/scripts/validate.sh index ea6f165..f73ec86 100755 --- a/scripts/validate.sh +++ b/scripts/validate.sh @@ -28,7 +28,6 @@ grep -q 'allow_implicit_invocation: false' "$plugin/skills/verified-delivery/age grep -q 'never auto-invoke for ordinary coding' "$plugin/skills/verified-delivery/SKILL.md" grep -q '\$wigtn-plugins-with-codex:verified-delivery' "$plugin/skills/verified-delivery/agents/openai.yaml" grep -q '커밋해줘' "$plugin/skills/release-readiness/SKILL.md" -grep -q 'PRD 디깅해줘' "$plugin/skills/product-spec/SKILL.md" python3 "$repo_root/scripts/check-verified-delivery-routing.py" python3 "$repo_root/scripts/check-external-results.py" python3 "$repo_root/scripts/check-featurebench-selection.py" diff --git a/tests/behavior/token-efficiency-cases.tsv b/tests/behavior/token-efficiency-cases.tsv new file mode 100644 index 0000000..079ae5a --- /dev/null +++ b/tests/behavior/token-efficiency-cases.tsv @@ -0,0 +1,3 @@ +# case_id prompt_path +prd-create tests/behavior/prompts/prd-create.txt +acceptance-uncertain tests/behavior/prompts/acceptance-uncertain.txt