From afab32615c72ef8eeaaa18fac26fb77f47eb5e5a Mon Sep 17 00:00:00 2001 From: dacharyc Date: Sat, 19 Sep 2026 20:55:18 -0400 Subject: [PATCH 1/2] Add description-length-unit check and multibyte fixtures The spec caps description at "1024 characters" without defining a character, and enforcing implementations disagree: the skills-ref reference validator and Codex count code points, Cline counts UTF-16 code units, and Crush counts UTF-8 bytes (skill-validator did too, until issue #94). The existing ASCII oversize fixture cannot tell these apart because its byte, code unit, and code point counts are equal. Add two fixtures that stay under 1024 code points but exceed the limit in other units: probe-multibyte-description (Japanese prose, over in bytes only) and probe-astral-description (emoji, over in UTF-16 units and bytes). Read with probe-long-description, the three outcomes separate no enforcement, code points, UTF-16 units, and bytes. Wire the check into the runner with a three-session spec, report labels, and a spec-alignment judge; bump the check list to 0.3. Co-Authored-By: Claude Fable 5.1 --- README.md | 4 +- benchmark-runner/checks/checks.go | 5 +- benchmark-runner/checks/discovery.go | 122 ++++++++++++++++++ benchmark-runner/report.go | 8 +- benchmark-runner/specalign.go | 13 ++ benchmark-skills/README.md | 11 ++ .../probe-astral-description/SKILL.md | 41 ++++++ .../probe-multibyte-description/SKILL.md | 41 ++++++ checks.md | 14 +- platform-findings/template.md | 9 ++ site/content/_index.md | 8 +- site/copy_content_and_build | 2 +- 12 files changed, 266 insertions(+), 12 deletions(-) create mode 100644 benchmark-skills/probe-astral-description/SKILL.md create mode 100644 benchmark-skills/probe-multibyte-description/SKILL.md diff --git a/README.md b/README.md index 97e1689..aab231c 100644 --- a/README.md +++ b/README.md @@ -30,12 +30,12 @@ through empirical testing rather than assumptions. ## What's here -- **[checks.md](checks.md)**: 40 checks across 10 categories +- **[checks.md](checks.md)**: 41 checks across 10 categories of platform behavior (loading, validation, execution, and access control) that need empirical testing. Each check describes what it evaluates and why it matters for skill authors. -- **[benchmark-skills/](benchmark-skills/)**: 33 spec-compliant skills designed +- **[benchmark-skills/](benchmark-skills/)**: 35 spec-compliant skills designed to exercise those checks. Each contains unique canary phrases that reveal what a platform loaded and when, without relying on model self-reporting. See the [benchmark skills README](benchmark-skills/README.md) for the full inventory, diff --git a/benchmark-runner/checks/checks.go b/benchmark-runner/checks/checks.go index f8072a7..2502c5e 100644 --- a/benchmark-runner/checks/checks.go +++ b/benchmark-runner/checks/checks.go @@ -1,6 +1,6 @@ // Package checks defines the automated platform-behavior checks and their // verdict logic. Check IDs match ../checks.md (check list -// version 0.2) and canary phrases match ../benchmark-skills/README.md's +// version 0.3) and canary phrases match ../benchmark-skills/README.md's // canary index. The invocation and observation machinery lives in // skillxp; this package owns only what makes these observations a // benchmark: which skills, which prompts, and how facts map to verdicts. @@ -23,7 +23,7 @@ import ( // ChecklistVersion is the checks.md check list version these // specs implement; reports stamp it so readers know which checks existed // when a platform was tested. -const ChecklistVersion = "0.2" +const ChecklistVersion = "0.3" // Statuses, matching the results template vocabulary. const ( @@ -186,6 +186,7 @@ func Registry() []Spec { nameDirectoryMismatch(), metadataValueEdgeCases(), oversizeDescriptionHandling(), + descriptionLengthUnit(), oversizeCompatibilityHandling(), } } diff --git a/benchmark-runner/checks/discovery.go b/benchmark-runner/checks/discovery.go index c606863..5593e33 100644 --- a/benchmark-runner/checks/discovery.go +++ b/benchmark-runner/checks/discovery.go @@ -361,3 +361,125 @@ func oversizeCompatibilityHandling() Spec { } }) } + +// Fixtures and markers for the description-length-unit check. Read +// alongside probe-long-description (ASCII, over 1024 in every unit), the +// two fixtures below separate what an enforcing platform counts: the +// multibyte one is under 1024 code points and UTF-16 units but over in +// UTF-8 bytes; the astral one is under 1024 code points but over in both +// UTF-16 units and bytes. +const ( + multibyteDescBodyCanary = "PUFFIN-BASALT-4471" + multibyteDescHeadMarker = "GANNET-PYRITE-1130" + multibyteDescTailMarker = "SHRIKE-TALC-2210" + astralDescBodyCanary = "ORIOLE-GRANITE-5583" + astralDescHeadMarker = "MAGPIE-OBSIDIAN-1240" + astralDescTailMarker = "LINNET-MALACHITE-2420" +) + +// descriptionFate is what became of one fixture's description value on +// its way to the model. +type descriptionFate string + +const ( + descIntact descriptionFate = "intact" // tail marker reached the model + descTruncated descriptionFate = "truncated" // head marker reached the model, tail did not + descRejected descriptionFate = "rejected" // skill absent from the catalog + descUnobserved descriptionFate = "unobserved" // listed, but marker delivery is not recorded +) + +// descriptionFateOf grades one fixture session. Marker delivery in +// harness-injected content is the direct signal; where the harness does +// not record what it injected, the model's own quoting of the markers in +// its listing answer stands in. +func descriptionFateOf(so *observe.SessionObservation, sessionNum int, skillName, bodyCanary, head, tail string) (descriptionFate, []Evidence) { + obs := so.Final() + listed, _, ev := catalogPresence(so, skillName) + for i := range ev { + ev[i].Session = sessionNum + } + if inj, pull := loadsOf(so, bodyCanary); len(inj)+len(pull) > 0 { + ev = append(ev, evSession(so, sessionNum, loadsInOrder(inj, pull)[0].EventIndex, "body canary loaded on activation")) + } + headInj := trace.At(trace.Phrase(obs.Session, head, obs.Profile.EchoSubtypes), trace.LocHarnessInjected) + tailInj := trace.At(trace.Phrase(obs.Session, tail, obs.Profile.EchoSubtypes), trace.LocHarnessInjected) + switch { + case len(tailInj) > 0: + return descIntact, append(ev, evSession(so, sessionNum, tailInj[0].EventIndex, "description tail marker in injected content")) + case len(headInj) > 0: + return descTruncated, append(ev, evSession(so, sessionNum, headInj[0].EventIndex, "description head marker in injected content; tail absent")) + case !listed: + return descRejected, ev + } + if m := assistantMentions(so, tail); len(m) > 0 { + return descIntact, append(ev, evSession(so, sessionNum, m[0].EventIndex, "model quoted the tail marker from its catalog entry (inferred)")) + } + if m := assistantMentions(so, head); len(m) > 0 { + return descTruncated, append(ev, evSession(so, sessionNum, m[0].EventIndex, "model quoted the head marker but never the tail (inferred)")) + } + return descUnobserved, ev +} + +func descriptionLengthUnit() Spec { + fixtures := []struct{ label, dir, body, head, tail string }{ + {"ascii", "probe-long-description", longDescBodyCanary, longDescHeadMarker, longDescTailMarker}, + {"multibyte", "probe-multibyte-description", multibyteDescBodyCanary, multibyteDescHeadMarker, multibyteDescTailMarker}, + {"astral", "probe-astral-description", astralDescBodyCanary, astralDescHeadMarker, astralDescTailMarker}, + } + sessions := make([]Session, 0, len(fixtures)) + for _, fx := range fixtures { + dir := fx.dir + sessions = append(sessions, Session{ + Skills: []string{dir}, + Turns: []Turn{ + {Prompt: listingPrompt}, + {Prompt: func(p profile.Profile) string { return p.ActivationPrompt(dir) }, Activation: true}, + }, + }) + } + enforced := func(d descriptionFate) bool { return d == descTruncated || d == descRejected } + return Spec{ + ID: "description-length-unit", + Description: "When a platform enforces the 1024-character description limit, does it count Unicode code points, UTF-16 code units, or UTF-8 bytes?", + Sessions: sessions, + Evaluate: func(sos []*observe.SessionObservation) Finding { + f := Finding{CheckID: "description-length-unit", Status: StatusObserved, Confidence: ConfidenceDirect} + fates := make([]descriptionFate, len(fixtures)) + summary := make([]string, 0, len(fixtures)) + for i, fx := range fixtures { + fate, ev := descriptionFateOf(sos[i], i+1, fx.dir, fx.body, fx.head, fx.tail) + fates[i] = fate + f.Evidence = append(f.Evidence, ev...) + summary = append(summary, fx.label+":"+string(fate)) + if !sos[i].Final().Profile.RecordsInjectedContext { + f.Confidence = ConfidenceInferred + } + f.Notes = append(f.Notes, fx.label+" final answer: "+finalAnswer(sos[i])) + } + f.Notes = append([]string{"description fates: " + strings.Join(summary, ", ")}, f.Notes...) + ascii, multibyte, astral := fates[0], fates[1], fates[2] + switch { + case ascii == descUnobserved || multibyte == descUnobserved || astral == descUnobserved: + f.Status = StatusInconclusive + f.Verdict = "length-unit-unobservable" + f.Notes = append(f.Notes, "at least one fixture's description delivery could not be observed, so the counting unit cannot be inferred") + case ascii == descIntact && multibyte == descIntact && astral == descIntact: + f.Verdict = "no-length-enforcement" + f.Notes = append(f.Notes, "all three descriptions reached the model intact, including the ASCII one that exceeds 1024 in every unit: the platform does not enforce the limit, so its counting unit is moot") + case enforced(ascii) && multibyte == descIntact && astral == descIntact: + f.Verdict = "counts-code-points" + f.Notes = append(f.Notes, "the ASCII overrun was enforced while both fixtures under 1024 code points survived intact: the platform counts code points, the reference validator's unit") + case enforced(ascii) && multibyte == descIntact && enforced(astral): + f.Verdict = "counts-utf16-units" + f.Notes = append(f.Notes, "the ASCII and astral overruns were enforced while the multibyte fixture survived: the platform counts UTF-16 code units (JavaScript's .length)") + case enforced(ascii) && enforced(multibyte) && enforced(astral): + f.Verdict = "counts-bytes" + f.Notes = append(f.Notes, "all three fixtures were enforced, including the two under 1024 code points: the platform counts UTF-8 bytes") + default: + f.Verdict = "inconsistent-length-unit" + f.Notes = append(f.Notes, "the three outcomes fit no single counting unit; see the per-fixture fates") + } + return f + }, + } +} diff --git a/benchmark-runner/report.go b/benchmark-runner/report.go index 29c78ee..47c0dca 100644 --- a/benchmark-runner/report.go +++ b/benchmark-runner/report.go @@ -52,7 +52,7 @@ var reportCategories = []struct { {"Skill-to-Skill Invocation", []string{"cross-skill-invocation", "invocation-depth-limit", "circular-invocation-handling", "invocation-language-sensitivity"}}, {"Skill Dependencies", []string{"informal-dependency-resolution", "missing-dependency-behavior", "nonstandard-dependency-fields", "cross-scope-dependency"}}, {"Discovery Scope", []string{"cross-client-directory-interop", "recursive-root-discovery", "nested-skill-discovery", "name-collision-precedence"}}, - {"Validation Strictness", []string{"malformed-yaml-tolerance", "missing-description-handling", "invalid-name-tolerance", "name-directory-mismatch", "metadata-value-edge-cases", "oversize-description-handling", "oversize-compatibility-handling"}}, + {"Validation Strictness", []string{"malformed-yaml-tolerance", "missing-description-handling", "invalid-name-tolerance", "name-directory-mismatch", "metadata-value-edge-cases", "oversize-description-handling", "description-length-unit", "oversize-compatibility-handling"}}, } // manualChecks require interactive sessions the runner cannot drive. @@ -222,6 +222,12 @@ var verdictPhrases = map[string]string{ "all-invalid-names-rejected": "All three invalid names rejected", "loaded-despite-oversize-description": "Loaded anyway", "skipped-oversize-description": "Skipped", + "no-length-enforcement": "No enforcement (unit is moot)", + "counts-code-points": "Counts Unicode code points", + "counts-utf16-units": "Counts UTF-16 code units", + "counts-bytes": "Counts UTF-8 bytes", + "inconsistent-length-unit": "Fits no single unit", + "length-unit-unobservable": "Unit not observable", "loaded-despite-oversize-compatibility": "Loaded anyway", "skipped-oversize-compatibility": "Skipped", } diff --git a/benchmark-runner/specalign.go b/benchmark-runner/specalign.go index d2a84be..7882989 100644 --- a/benchmark-runner/specalign.go +++ b/benchmark-runner/specalign.go @@ -234,6 +234,19 @@ var specJudges = []struct { } return unclassifiedSpec(f) }}, + {"description-length-unit", func(f checks.Finding) specJudgement { + switch f.Verdict { + case "no-length-enforcement": + return specJudgement{specConsistent, "The spec caps description at 1024 characters without defining the unit. This platform does not enforce the limit at all, so every fixture under 1024 code points loaded intact and the counting unit is moot."} + case "counts-code-points": + return specJudgement{specConsistent, "The spec caps description at 1024 characters without defining the unit. This platform counts Unicode code points, the same unit as the spec's skills-ref reference validator, so descriptions the reference validator accepts load here too."} + case "counts-utf16-units": + return specJudgement{specContradicts, "The spec caps description at 1024 characters without defining the unit; its skills-ref reference validator counts code points. This platform counts UTF-16 code units, so a description with emoji or other supplementary-plane characters that the reference validator accepts is rejected or truncated here."} + case "counts-bytes": + return specJudgement{specContradicts, "The spec caps description at 1024 characters without defining the unit; its skills-ref reference validator counts code points. This platform counts UTF-8 bytes, so a description in a CJK script that the reference validator accepts is rejected or truncated here."} + } + return unclassifiedSpec(f) + }}, {"oversize-compatibility-handling", func(f checks.Finding) specJudgement { switch f.Verdict { case "loaded-despite-oversize-compatibility": diff --git a/benchmark-skills/README.md b/benchmark-skills/README.md index 2e536c6..3dd6895 100644 --- a/benchmark-skills/README.md +++ b/benchmark-skills/README.md @@ -41,6 +41,8 @@ determine what content the platform loaded and when. | `probe--double-hyphen` | Invalid name: consecutive hyphens (name matches directory) | SKILL.md only | | `probe-overlong-name-…-limit` | Invalid name: 72 characters, past the 64-char limit (name matches directory) | SKILL.md only | | `probe-long-description` | Oversize description (1116 chars, head + tail markers) | SKILL.md only | +| `probe-multibyte-description` | Description under 1024 code points but over 1024 UTF-8 bytes (Japanese prose; 848 chars, 1822 bytes; head + tail markers) | SKILL.md only | +| `probe-astral-description` | Description under 1024 code points but over 1024 UTF-16 units and bytes (emoji; 869 chars, 1319 UTF-16 units, 2219 bytes; head + tail markers) | SKILL.md only | | `probe-long-compatibility` | Oversize compatibility value (570 chars, tail marker) | SKILL.md only | ## Check-to-Skill Mapping @@ -137,6 +139,7 @@ skills** provide additional signal or are needed as part of the test setup. | `name-directory-mismatch` | `probe-mismatch-dir` | Install the directory as-is. Check the available skills list: does the skill appear as `probe-name-mismatch` (frontmatter), `probe-mismatch-dir` (directory), or not at all? Then activate it by whichever name appeared and look for SWAN-BERYL-3324. | | `metadata-value-edge-cases` | `probe-metadata-values` | Activate the skill. If it loads successfully, the platform didn't reject the edge-case metadata. Check step 2-3 to see which values the model received and whether any keys were dropped. Look for canary phrase THRUSH-FLINT-8294 to confirm the body loaded. | | `oversize-description-handling` | `probe-long-description` | Install normally. Is the skill listed despite the 1116-char description? Does the listing show the head marker SANDERLING-GNEISS-1010 but not the tail marker WHIMBREL-DOLOMITE-2020 (truncation)? Activate and look for BITTERN-HALITE-2264. | +| `description-length-unit` | `probe-long-description` + `probe-multibyte-description` + `probe-astral-description` | Install all three. For each, is it listed, and does the listing show its head marker, its tail marker, both, or neither? ASCII fixture rejected or truncated while both others survive intact = code points; multibyte survives but astral does not = UTF-16 units; none survive = bytes; all three survive = no enforcement. Markers: GANNET-PYRITE-1130 / SHRIKE-TALC-2210 (multibyte), MAGPIE-OBSIDIAN-1240 / LINNET-MALACHITE-2420 (astral). Body canaries: PUFFIN-BASALT-4471 (multibyte), ORIOLE-GRANITE-5583 (astral). | | `oversize-compatibility-handling` | `probe-long-compatibility` | Install normally. Is the skill listed despite the 570-char compatibility value? Activate and look for KESTREL-BAUXITE-6690; note whether the tail marker TURNSTONE-ARAGONITE-3030 surfaces anywhere. | ## Structural Validation @@ -162,6 +165,8 @@ nonstandard structures to test platform loading behavior: | `probe--double-hyphen` | **Error**: consecutive hyphens in name | Tests invalid-name tolerance — the hyphen violation is the fixture | | `probe-overlong-name-…-limit` | **Error**: name exceeds 64 characters | Tests invalid-name tolerance — the length violation is the fixture | | `probe-long-description` | **Error**: description exceeds 1024 characters | Tests oversize-field handling — the overrun is the fixture | +| `probe-multibyte-description` | **Error** on skill-validator 1.6.1 and earlier only: description exceeds 1024 characters | Those versions counted UTF-8 bytes ([issue #94](https://github.com/agent-ecosystem/skill-validator/issues/94)); the description is 848 characters and passes once the validator counts characters | +| `probe-astral-description` | **Error** on skill-validator 1.6.1 and earlier only: description exceeds 1024 characters | Same byte-counting bug; the description is 869 characters | | `probe-long-compatibility` | **Error**: compatibility exceeds 500 characters | Tests oversize-field handling — the overrun is the fixture | | `probe-allowed-tools` | Experimental `allowed-tools` field | Tests whether the field pre-approves tools — pair with its control twin | @@ -225,5 +230,11 @@ it reveals what the platform loaded automatically. | BITTERN-HALITE-2264 | SKILL.md body | probe-long-description | | SANDERLING-GNEISS-1010 | SKILL.md description ONLY (head marker; the body never spells it out) | probe-long-description | | WHIMBREL-DOLOMITE-2020 | SKILL.md description ONLY (tail marker; the body never spells it out) | probe-long-description | +| PUFFIN-BASALT-4471 | SKILL.md body | probe-multibyte-description | +| GANNET-PYRITE-1130 | SKILL.md description ONLY (head marker; the body never spells it out) | probe-multibyte-description | +| SHRIKE-TALC-2210 | SKILL.md description ONLY (tail marker; the body never spells it out) | probe-multibyte-description | +| ORIOLE-GRANITE-5583 | SKILL.md body | probe-astral-description | +| MAGPIE-OBSIDIAN-1240 | SKILL.md description ONLY (head marker; the body never spells it out) | probe-astral-description | +| LINNET-MALACHITE-2420 | SKILL.md description ONLY (tail marker; the body never spells it out) | probe-astral-description | | KESTREL-BAUXITE-6690 | SKILL.md body | probe-long-compatibility | | TURNSTONE-ARAGONITE-3030 | SKILL.md compatibility ONLY (tail marker; the body never spells it out) | probe-long-compatibility | diff --git a/benchmark-skills/probe-astral-description/SKILL.md b/benchmark-skills/probe-astral-description/SKILL.md new file mode 100644 index 0000000..6c2823d --- /dev/null +++ b/benchmark-skills/probe-astral-description/SKILL.md @@ -0,0 +1,41 @@ +--- +name: probe-astral-description +description: Benchmark skill whose description stays under the spec's 1024-character limit when counted in Unicode code points but runs past it when counted in UTF-16 code units or UTF-8 bytes. Use when asked to probe description length units. The head marker MAGPIE-OBSIDIAN-1240 sits near the start and a tail marker sits at the very end; between them is deliberate emoji padding. 🐦🪨🧪📏🔤🧭🌐📐📦🧱🔎🧮📚🪶🧵🪵🔬🧩🗺🪙🐦🪨🧪📏🔤🧭🌐📐📦🧱🔎🧮📚🪶🧵🪵🔬🧩🗺🪙🐦🪨🧪📏🔤🧭🌐📐📦🧱 🔎🧮📚🪶🧵🪵🔬🧩🗺🪙🐦🪨🧪📏🔤🧭🌐📐📦🧱🔎🧮📚🪶🧵🪵🔬🧩🗺🪙🐦🪨🧪📏🔤🧭🌐📐📦🧱🔎🧮📚🪶🧵🪵🔬🧩🗺🪙 🐦🪨🧪📏🔤🧭🌐📐📦🧱🔎🧮📚🪶🧵🪵🔬🧩🗺🪙🐦🪨🧪📏🔤🧭🌐📐📦🧱🔎🧮📚🪶🧵🪵🔬🧩🗺🪙🐦🪨🧪📏🔤🧭🌐📐📦🧱 🔎🧮📚🪶🧵🪵🔬🧩🗺🪙🐦🪨🧪📏🔤🧭🌐📐📦🧱🔎🧮📚🪶🧵🪵🔬🧩🗺🪙🐦🪨🧪📏🔤🧭🌐📐📦🧱🔎🧮📚🪶🧵🪵🔬🧩🗺🪙 🐦🪨🧪📏🔤🧭🌐📐📦🧱🔎🧮📚🪶🧵🪵🔬🧩🗺🪙🐦🪨🧪📏🔤🧭🌐📐📦🧱🔎🧮📚🪶🧵🪵🔬🧩🗺🪙🐦🪨🧪📏🔤🧭🌐📐📦🧱 🔎🧮📚🪶🧵🪵🔬🧩🗺🪙🐦🪨🧪📏🔤🧭🌐📐📦🧱🔎🧮📚🪶🧵🪵🔬🧩🗺🪙🐦🪨🧪📏🔤🧭🌐📐📦🧱🔎🧮📚🪶🧵🪵🔬🧩🗺🪙 🐦🪨🧪📏🔤🧭🌐📐📦🧱🔎🧮📚🪶🧵🪵🔬🧩🗺🪙🐦🪨🧪📏🔤🧭🌐📐📦🧱🔎🧮📚🪶🧵🪵🔬🧩🗺🪙🐦🪨🧪📏🔤🧭🌐📐📦🧱 🔎🧮📚🪶🧵🪵🔬🧩🗺🪙🐦🪨🧪📏🔤🧭🌐📐📦🧱🔎🧮📚🪶🧵🪵🔬🧩🗺🪙🐦🪨🧪📏🔤🧭🌐📐📦🧱🔎🧮📚🪶🧵🪵🔬🧩🗺🪙 🐦🪨🧪📏🔤🧭🌐📐📦🧱🔎🧮📚🪶🧵🪵🔬🧩🗺🪙🐦🪨🧪📏🔤🧭🌐📐📦🧱🔎🧮📚🪶🧵🪵🔬🧩🗺🪙🐦🪨🧪📏🔤🧭🌐📐📦🧱 The tail marker is LINNET-MALACHITE-2420 +--- + +# Astral Description Probe + +The spec's `description` field "must be 1-1024 characters." It does not +say what a character is. This skill's description is 869 Unicode +code points, but 1319 UTF-16 code units and 2219 UTF-8 bytes, +because most of it is emoji from the supplementary planes: each one is a +single code point, two UTF-16 units, and four bytes. A platform that +counts code points sees a compliant description; one that counts UTF-16 +units (JavaScript's `.length`) or bytes sees an oversize one. + +The description carries an ASCII head marker near its start and an +ASCII tail marker as its final characters (bird-mineral phrases; the +head marker's bird is MAGPIE and the tail marker's bird is LINNET). A +listing that shows the head but not the tail reveals truncation. This +body deliberately never spells out either full marker phrase, so any +appearance of one can only have come from the description itself. + +Pair this skill with `probe-long-description` (ASCII, over the limit in +every unit) and `probe-multibyte-description` (under the limit in code +points and UTF-16 units, over it in bytes) to tell the counting units +apart. + +## Canary Phrase + +The canary phrase for this skill's body is: **ORIOLE-GRANITE-5583** + +## Instructions + +When activated, report: + +1. "probe-astral-description activated. Canary: **ORIOLE-GRANITE-5583**" + +2. **Description visibility**: Looking only at your catalog entry for + this skill (not this file), can you see the MAGPIE head marker, the + LINNET tail marker, both, or neither? Quote exactly what the catalog + shows. diff --git a/benchmark-skills/probe-multibyte-description/SKILL.md b/benchmark-skills/probe-multibyte-description/SKILL.md new file mode 100644 index 0000000..b53c7fd --- /dev/null +++ b/benchmark-skills/probe-multibyte-description/SKILL.md @@ -0,0 +1,41 @@ +--- +name: probe-multibyte-description +description: Benchmark skill whose description stays under the spec's 1024-character limit when counted in characters but runs past it when counted in UTF-8 bytes. Use when asked to probe description length units. The head marker GANNET-PYRITE-1130 sits near the start of this description and a tail marker sits at the very end. このスキルの説明文は、仕様が定める1024文字の上限を文字数では下回りますが、UTF-8のバイト数では上回ります。日本語の文字は一文字あたり三バイトを占めるため、文字数とバイト数の差が大きく開きます。プラットフォームがこの説明文をそのまま受け入れれば、長さを文字数で数えているか、あるいは長さを検証していないかのどちらかです。逆にこのスキルを拒否したり、説明文の末尾を切り捨てたりするなら、長さをバイト数で数えている可能性が高いと言えます。この段落は意図的に日本語で書かれた詰め物であり、スキルの機能を追加するものではありません。検証の対象は説明文の長さの数え方だけであり、それ以外の点ではこのスキルは通常の有効なスキルです。説明文の冒頭と末尾には英語の目印が置かれているため、末尾の目印が見えるかどうかで切り捨ての有無を判断できます。英語以外の言語で書いた説明文が、あるプラットフォームでは受け入れられ、別のプラットフォームでは静かに拒否されるかもしれないという点が、スキルの作者にとって重要です。この違いはエラーとして表面化しないため、数え方の違いを実際に観測して記録する必要があります。 The tail marker is SHRIKE-TALC-2210 +--- + +# Multibyte Description Probe + +The spec's `description` field "must be 1-1024 characters." It does not +say what a character is. This skill's description is 848 Unicode +code points (and the same number of UTF-16 code units, since every +character is in the Basic Multilingual Plane), but 1822 UTF-8 bytes, +because most of it is Japanese prose at three bytes per character. A +platform that counts code points or UTF-16 units sees a compliant +description; one that counts bytes sees an oversize one. + +The description carries an English head marker near its start and an +English tail marker as its final characters (bird-mineral phrases; the +head marker's bird is GANNET and the tail marker's bird is SHRIKE). A +listing that shows the head but not the tail reveals truncation. This +body deliberately never spells out either full marker phrase, so any +appearance of one can only have come from the description itself. + +Pair this skill with `probe-long-description` (ASCII, over the limit in +every unit) and `probe-astral-description` (under the limit in code +points, over it in UTF-16 units and bytes) to tell the counting units +apart. + +## Canary Phrase + +The canary phrase for this skill's body is: **PUFFIN-BASALT-4471** + +## Instructions + +When activated, report: + +1. "probe-multibyte-description activated. Canary: **PUFFIN-BASALT-4471**" + +2. **Description visibility**: Looking only at your catalog entry for + this skill (not this file), can you see the GANNET head marker, the + SHRIKE tail marker, both, or neither? Quote exactly what the catalog + shows. diff --git a/checks.md b/checks.md index 2bc32ec..cf3dd91 100644 --- a/checks.md +++ b/checks.md @@ -1,6 +1,6 @@ # Platform Behavior Checks -**Check list version: 0.2** ([changelog](#changelog)) +**Check list version: 0.3** ([changelog](#changelog)) How do agent platforms actually load skill content? The [Agent Skills specification](https://agentskills.io/specification) defines a file format and recommends a three-tier "progressive disclosure" model, but gives platforms wide latitude in implementation. The [client implementation guide](https://agentskills.io/client-implementation/adding-skills-support) provides more detailed guidance, but claims it was derived from analysis of 7 of 25+ adopting platforms and published months after most platforms had already shipped their implementations. @@ -331,6 +331,12 @@ These checks feed platforms skills that break the spec's format rules (invalid Y - **What it checks**: Whether a skill whose description exceeds the spec's 1024-character cap (this fixture's runs to 1116) is discovered, and whether the value survives intact or truncated. The description carries a head marker near its start and a tail marker as its final characters, and the body never repeats either, so what surfaces reveals exactly how much of the value survived. - **Why it matters**: Every installed skill's description occupies context at startup, which gives platforms a real incentive to cap or truncate. Rejection makes the skill vanish on strict platforms; silent truncation quietly deletes the end of the description, which is often where the "use when" activation cues live. Either way, an author with a long description gets different discovery behavior per platform without any error. +### `description-length-unit` + +- **Category**: Validation Strictness +- **What it checks**: When a platform enforces the description limit, which unit it counts: Unicode code points, UTF-16 code units, or UTF-8 bytes. Two fixtures stay under 1024 code points but pass the limit in other units. `probe-multibyte-description` is mostly Japanese prose (848 code points, 848 UTF-16 units, 1822 bytes), so only a byte counter sees it as oversize. `probe-astral-description` is mostly emoji (869 code points, 1319 UTF-16 units, 2219 bytes), so a UTF-16 counter and a byte counter both see it as oversize. Each carries ASCII head and tail markers, like `probe-long-description`. Read together with that ASCII fixture (over the limit in every unit), the three outcomes separate four postures: no enforcement, code points, UTF-16 units, or bytes. +- **Why it matters**: The spec says the description "must be 1-1024 characters" and never defines a character. Implementations that enforce the limit disagree on the unit. Reading the public loaders as of 2026-09-19: the spec's `skills-ref` reference validator counts code points (Python `len`), Codex CLI truncates its catalog entry at 1024 code points, Cline rejects by JavaScript `.length` (UTF-16 code units), and Crush rejects by Go `len` (UTF-8 bytes). skill-validator itself counted bytes until [issue #94](https://github.com/agent-ecosystem/skill-validator/issues/94), where a 984-character Japanese description (2952 bytes) was flagged as over the limit. Such a description is compliant to the reference validator and loads on a code-point platform, yet vanishes or loses its tail on a byte-counting one, with no error. Authors writing descriptions in CJK scripts, or with emoji, get different discovery behavior per platform without changing a thing. + ### `oversize-compatibility-handling` - **Category**: Validation Strictness @@ -340,7 +346,7 @@ These checks feed platforms skills that break the spec's format rules (invalid Y --- ## Benchmark Skills -The [`benchmark-skills/`](https://github.com/agent-ecosystem/agent-skill-implementation/tree/main/benchmark-skills) directory contains 33 benchmark fixtures (spec-compliant skills plus deliberate structural and validation edge cases) designed to exercise these checks. Each skill contains unique **canary phrases** (e.g., CARDINAL-ZEBRA-7742) embedded in specific files. By asking the model whether it knows a canary phrase, testers can determine exactly what the platform loaded and when, without relying on the model's self-reporting about its own context. +The [`benchmark-skills/`](https://github.com/agent-ecosystem/agent-skill-implementation/tree/main/benchmark-skills) directory contains 35 benchmark fixtures (spec-compliant skills plus deliberate structural and validation edge cases) designed to exercise these checks. Each skill contains unique **canary phrases** (e.g., CARDINAL-ZEBRA-7742) embedded in specific files. By asking the model whether it knows a canary phrase, testers can determine exactly what the platform loaded and when, without relying on the model's self-reporting about its own context. See [`benchmark-skills/README.md`](https://github.com/agent-ecosystem/agent-skill-implementation/blob/main/benchmark-skills/README.md) for: @@ -358,6 +364,10 @@ Even partial data is valuable. A single platform tested thoroughly is more usefu Findings submissions record the check list version they were tested against (see the template's "Check list version" field), so readers can tell which checks existed when a platform was tested. +### 0.3 (2026-09-19) + +- Added `description-length-unit` (Category 10: Validation Strictness) with two fixtures, `probe-multibyte-description` and `probe-astral-description`, prompted by a skill-validator bug report ([issue #94](https://github.com/agent-ecosystem/skill-validator/issues/94)) where a 984-character CJK description was rejected because the limit was counted in UTF-8 bytes. Reading public harness loaders showed enforcing implementations split three ways on the unit (code points, UTF-16 code units, bytes), which the existing ASCII oversize fixture cannot tell apart. + ### 0.2 (2026-08-01) - Added `name-directory-mismatch` and `recursive-root-discovery` (Category 7: Structural Edge Cases), prompted by observed platform divergence in nested-skill discovery: one platform registers and invokes any SKILL.md found recursively under its skills root, indexed by frontmatter name alone. diff --git a/platform-findings/template.md b/platform-findings/template.md index 5fec01d..c6630fd 100644 --- a/platform-findings/template.md +++ b/platform-findings/template.md @@ -493,6 +493,15 @@ work on that platform. - **Platform-level or model-level?**: - **Fallback behavior**: +#### `description-length-unit` + +- **Benchmark skill**: `probe-long-description` (ASCII, 1116 characters) plus `probe-multibyte-description` (848 code points, 1822 UTF-8 bytes; head marker GANNET-PYRITE-1130, tail marker SHRIKE-TALC-2210, body canary PUFFIN-BASALT-4471) and `probe-astral-description` (869 code points, 1319 UTF-16 units, 2219 bytes; head marker MAGPIE-OBSIDIAN-1240, tail marker LINNET-MALACHITE-2420, body canary ORIOLE-GRANITE-5583). +- **Status**: Not tested +- **Observation**: +- **Evidence**: +- **Platform-level or model-level?**: +- **Fallback behavior**: + #### `oversize-compatibility-handling` - **Benchmark skill**: `probe-long-compatibility` (compatibility value is 570 characters; the spec caps it at 500, tail marker TURNSTONE-ARAGONITE-3030). Body canary: KESTREL-BAUXITE-6690. diff --git a/site/content/_index.md b/site/content/_index.md index aa90ce2..681e82c 100644 --- a/site/content/_index.md +++ b/site/content/_index.md @@ -45,13 +45,13 @@ Can the agent run a bundled scripts/ file? Claude Code ✗ blocked with an error Codex CLI ✓ script ran; output returned -40 checks · 3 platforms · every claim transcript-cited +41 checks · 3 platforms · every claim transcript-cited

The Checks

-

40 checks across 10 categories, from loading timing to validation strictness. Each asks one testable question about platform behavior and explains why the answer matters to skill authors.

+

41 checks across 10 categories, from loading timing to validation strictness. Each asks one testable question about platform behavior and explains why the answer matters to skill authors.

Browse the catalog →
@@ -98,7 +98,7 @@ a transcript-cited finding. ## How the testing works The [benchmark skills](https://github.com/agent-ecosystem/agent-skill-implementation/tree/main/benchmark-skills) -are 33 fixtures (spec-compliant skills plus deliberate rule-breakers) seeded +are 35 fixtures (spec-compliant skills plus deliberate rule-breakers) seeded with unique **canary phrases**. By asking the model whether it knows a canary phrase, we can tell exactly what a platform loaded and when, without trusting the model's self-reporting about its own context. The @@ -113,7 +113,7 @@ finding to an archived transcript. We need empirical data from real platforms, and even partial data from a single platform beats speculation about all of them. Install the [benchmark skills](https://github.com/agent-ecosystem/agent-skill-implementation/tree/main/benchmark-skills), -run any of the 40 checks, and submit findings with the +run any of the 41 checks, and submit findings with the [platform template](https://github.com/agent-ecosystem/agent-skill-implementation/blob/main/platform-findings/template.md). The [GitHub repository](https://github.com/agent-ecosystem/agent-skill-implementation) has full instructions. diff --git a/site/copy_content_and_build b/site/copy_content_and_build index e0393a7..c076aa1 100755 --- a/site/copy_content_and_build +++ b/site/copy_content_and_build @@ -13,7 +13,7 @@ mkdir -p "$SITE_DIR/content/checks" cat > "$CONTENT_DEST" <<'FRONTMATTER' --- title: "Platform Behavior Checks" -description: "40 checks across 10 categories for empirically testing how agent platforms load, validate, manage, and present Agent Skills." +description: "41 checks across 10 categories for empirically testing how agent platforms load, validate, manage, and present Agent Skills." showTableOfContents: true --- From cfd082d3dc177affc349efccfeede98e706e0266 Mon Sep 17 00:00:00 2001 From: dacharyc Date: Sat, 19 Sep 2026 21:09:30 -0400 Subject: [PATCH 2/2] Run description-length-unit on the three headless harnesses Results: Claude Code (2.1.267) and Antigravity load all three fixtures intact (no-length-enforcement); Codex (0.154.0) truncates the ASCII overrun at 1024 but delivers both fixtures under 1024 code points whole (counts-code-points). All three match the expectations set out in the check. Reports and site platform pages regenerated with the new findings merged in. Bump agentminutes to v0.5.0 and skillxp to v0.1.3 so the transcript parser recognizes record types the current Claude Code and Codex releases emit (atis-latch, token_usage_record); v0.3.1 failed both runs with format-drift errors. Also set the finding's vehicle from the body canary loads, as the other validation checks do. Co-Authored-By: Claude Fable 5.1 --- benchmark-runner/checks/discovery.go | 3 ++ benchmark-runner/go.mod | 6 ++-- benchmark-runner/go.sum | 12 ++++---- benchmark-runner/reports/antigravity.md | 35 ++++++++++++++++++---- benchmark-runner/reports/claude-code.md | 35 ++++++++++++++++++---- benchmark-runner/reports/codex.md | 35 ++++++++++++++++++---- site/content/platforms/_index.md | 7 +++-- site/content/platforms/antigravity.md | 39 ++++++++++++++++++++----- site/content/platforms/claude-code.md | 39 ++++++++++++++++++++----- site/content/platforms/codex.md | 39 ++++++++++++++++++++----- 10 files changed, 202 insertions(+), 48 deletions(-) diff --git a/benchmark-runner/checks/discovery.go b/benchmark-runner/checks/discovery.go index 5593e33..aa97b30 100644 --- a/benchmark-runner/checks/discovery.go +++ b/benchmark-runner/checks/discovery.go @@ -455,6 +455,9 @@ func descriptionLengthUnit() Spec { f.Confidence = ConfidenceInferred } f.Notes = append(f.Notes, fx.label+" final answer: "+finalAnswer(sos[i])) + if inj, pull := loadsOf(sos[i], fx.body); f.Vehicle == "" && len(inj)+len(pull) > 0 { + f.Vehicle = vehicleOf(inj, pull) + } } f.Notes = append([]string{"description fates: " + strings.Join(summary, ", ")}, f.Notes...) ascii, multibyte, astral := fates[0], fates[1], fates[2] diff --git a/benchmark-runner/go.mod b/benchmark-runner/go.mod index 3edaaac..7252899 100644 --- a/benchmark-runner/go.mod +++ b/benchmark-runner/go.mod @@ -3,8 +3,8 @@ module github.com/agent-ecosystem/agent-skill-implementation/benchmark-runner go 1.25.5 require ( - github.com/agent-ecosystem/agentsummons v0.3.1 - github.com/agent-ecosystem/skillxp v0.1.1 + github.com/agent-ecosystem/agentsummons v0.3.3 + github.com/agent-ecosystem/skillxp v0.1.3 ) -require github.com/agent-ecosystem/agentminutes v0.3.1 +require github.com/agent-ecosystem/agentminutes v0.5.0 diff --git a/benchmark-runner/go.sum b/benchmark-runner/go.sum index cf1d3ec..4f16c9a 100644 --- a/benchmark-runner/go.sum +++ b/benchmark-runner/go.sum @@ -1,6 +1,6 @@ -github.com/agent-ecosystem/agentminutes v0.3.1 h1:qC5yesAoJwgfNaqUqv41o9EVK17WM6BGVRSOL9uvtlU= -github.com/agent-ecosystem/agentminutes v0.3.1/go.mod h1:vuIsPwLbM7wMeR7yIZyTCb4BHx2tAptgpd3sGf4CFr0= -github.com/agent-ecosystem/agentsummons v0.3.1 h1:ZldGr9OgsuDDgDyjY+qeSWLkZHgzJSdoExBkzhYHiO0= -github.com/agent-ecosystem/agentsummons v0.3.1/go.mod h1:fLKz+G0kl1lojOQUHZt8Yno/NF+f0FsRiMm4yJ11sJQ= -github.com/agent-ecosystem/skillxp v0.1.1 h1:FJffGCHESK87aZ4WvDdYOKTEDa/dTw5gDZTfDT6K3pU= -github.com/agent-ecosystem/skillxp v0.1.1/go.mod h1:5Hzs9WOqd48jEKMytCMY5S5pJNKTjuVSTwcWThK9too= +github.com/agent-ecosystem/agentminutes v0.5.0 h1:v87ZamvxAmNb17WnnA426tQxJvnMEdDTBgN3AeqLV/k= +github.com/agent-ecosystem/agentminutes v0.5.0/go.mod h1:J+e9H7DI94cBWTXQ6aG3BeJKNRhzjT1SEgL/klYWk8Q= +github.com/agent-ecosystem/agentsummons v0.3.3 h1:AD3bCIllD24nJ1oBUP8UwyVUSOZD08cot8N0wa82yaw= +github.com/agent-ecosystem/agentsummons v0.3.3/go.mod h1:fLKz+G0kl1lojOQUHZt8Yno/NF+f0FsRiMm4yJ11sJQ= +github.com/agent-ecosystem/skillxp v0.1.3 h1:qWgl+3PG0B7koVdztOn0NGbP6INVTIoVwoYh88sA6ck= +github.com/agent-ecosystem/skillxp v0.1.3/go.mod h1:+H05AjVnPQIUhh2+cWlbIFoO3lZHUIlyAw+TQFVDh5g= diff --git a/benchmark-runner/reports/antigravity.md b/benchmark-runner/reports/antigravity.md index 374a0fa..8ea231b 100644 --- a/benchmark-runner/reports/antigravity.md +++ b/benchmark-runner/reports/antigravity.md @@ -4,16 +4,16 @@ |---|---| | **Platform** | Antigravity CLI (headless) | | **Platform version** | 1.1.9 | -| **Check list version** | 0.2 | -| **Test date** | 2026-08-01 | -| **Model(s) observed** | Gemini 3.6 Flash (High) | +| **Check list version** | 0.3 | +| **Test date** | 2026-09-19 | +| **Model(s) observed** | Gemini 3.6 Flash (High), Gemini 3.8 Flash (High) | | **Environment** | Headless invocation via [benchmark-runner](https://github.com/agent-ecosystem/agent-skill-implementation/tree/main/benchmark-runner) + [skillxp](https://github.com/agent-ecosystem/skillxp) | > **Caveats**: All findings are from headless sessions, which may differ from interactive use. Verdicts are single-run observations unless a runs count is noted; for model-level behaviors, treat a single verdict as one observed outcome rather than a rate. Evidence line numbers cite the archived transcripts in the results directories. Fallback-behavior fields are auto-derived: where a run incidentally demonstrated a recovery path it is reported, otherwise the field says "not exercised". Automation does not probe recovery, so absence of a fallback observation is not evidence that none exists. ## Spec alignment -Most of this report measures behavior the [Agent Skills specification](https://agentskills.io/specification) leaves to each implementation, where differences between platforms are design choices rather than violations. 18 of the 40 checks do test something the specification prescribes; this section summarizes how observed behavior compares. Each entry links to the full finding below. +Most of this report measures behavior the [Agent Skills specification](https://agentskills.io/specification) leaves to each implementation, where differences between platforms are design choices rather than violations. 19 of the 41 checks do test something the specification prescribes; this section summarizes how observed behavior compares. Each entry links to the full finding below. ### Where behavior contradicts the spec @@ -30,6 +30,7 @@ Most of this report measures behavior the [Agent Skills specification](https://a - [`discovery-listing-fields`](#discovery-listing-fields): The listing surfaces location in addition to name and description. The spec describes only those two fields loading at startup, but it does not forbid extras. (Behavioral inference.) - [`frontmatter-handling`](#frontmatter-handling): The whole file, frontmatter included, reaches the model at activation because the model reads the raw file, matching the spec's description of loading the entire file. - [`compatibility-field-behavior`](#compatibility-field-behavior): The spec makes compatibility informational (it indicates environment requirements) and assigns it no loading semantics. Consistent with that, a skill declaring a different product still loads here; authors should not expect the field to gate anything. +- [`description-length-unit`](#description-length-unit): The spec caps description at 1024 characters without defining the unit. This platform does not enforce the limit at all, so every fixture under 1024 code points loaded intact and the counting unit is moot. (Behavioral inference.) ### How spec-invalid skills are handled @@ -603,6 +604,30 @@ _Is a skill whose description exceeds the spec's 1024-character limit still disc - **Note**: final answer: probe-long-description activated. Canary: **BITTERN-HALITE-2264** ### Description visibility Looking at the catalog entry in the system prompt's **Available skills** list, **both** the head marker (`SANDERLING-GNEISS-1010`) and the tail marker (`WHIMBREL-DOLOMITE-2020`) are visible intact. **Exact catalog description:** > Benchmark skill whose description deliberately runs past the spec's 1024-character limit, to test oversize field handling. Use when asked to probe oversize description handling. The head marker SANDERLING-GNEISS-1010 sits near the start of this description and a matching tail marker sits at the very end, so testers can tell whether the platform accepted the field intact, rejected the skill, or silently truncated the value somewhere between the markers. Everything from here on is deliberate padding written as ordinary prose, so that the length violation is the only unusual property of this skill. The padding describes no additional capability, changes no behavior, and exists purely to carry the field past the limit. It is phrased as complete sentences because some parsers and listings treat description text as display copy, and display copy that looks like natur… - **Fallback behavior**: Not exercised: automated single-session runs do not probe recovery paths (no follow-up prompting). Treat as untested rather than absent. +#### `description-length-unit` + +_When a platform enforces the 1024-character description limit, does it count Unicode code points, UTF-16 code units, or UTF-8 bytes?_ + +- **Status**: observed +- **Verdict**: `no-length-enforcement` +- **Confidence**: behavioral-inference +- **Evidence** (transcript: `results/length-unit-2026-09-19/antigravity/description-length-unit/session-3/transcript_full.jsonl`; session 20073e99-e26e-42a0-a078-6fb6287e6ea3,ed211f46-c340-46af-8f69-26fa63b1b742,0eebadd9-1827-4c3c-a53b-20845a6d4d69): + - tool-free listing answer names probe-long-description (session 1, event 3, line 2) + - body canary loaded on activation (session 1, event 9, line 6) + - model quoted the tail marker from its catalog entry (inferred) (session 1, event 10, line 7) + - tool-free listing answer names probe-multibyte-description (session 2, event 3, line 2) + - body canary loaded on activation (session 2, event 11, line 6) + - model quoted the tail marker from its catalog entry (inferred) (session 2, event 12, line 7) + - tool-free listing answer names probe-astral-description (session 3, event 4, line 2) + - body canary loaded on activation (session 3, event 10, line 6) + - model quoted the tail marker from its catalog entry (inferred) (session 3, event 11, line 7) +- **Note**: description fates: ascii:intact, multibyte:intact, astral:intact +- **Note**: ascii final answer: probe-long-description activated. Canary: **BITTERN-HALITE-2264** ### Description visibility Looking only at the catalog entry for this skill in the system prompt, **both** markers are visible: - Head marker: `SANDERLING-GNEISS-1010` - Tail marker: `WHIMBREL-DOLOMITE-2020` **Exact catalog description:** > Benchmark skill whose description deliberately runs past the spec's 1024-character limit, to test oversize field handling. Use when asked to probe oversize description handling. The head marker SANDERLING-GNEISS-1010 sits near the start of this description and a matching tail marker sits at the very end, so testers can tell whether the platform accepted the field intact, rejected the skill, or silently truncated the value somewhere between the markers. Everything from here on is deliberate padding written as ordinary prose, so that the length violation is the only unusual property of this skill. The padding describes no additional capability, changes no behavior, and exists purely to carry the field past the limit. It is phrased as complete sentences because some parsers and listings treat description text as display copy, and display copy that looks like natural language exer… +- **Note**: multibyte final answer: probe-multibyte-description activated. Canary: **PUFFIN-BASALT-4471** **Description visibility**: Looking only at the catalog entry for this skill, **both** markers are visible: - Head marker: `GANNET-PYRITE-1130` - Tail marker: `SHRIKE-TALC-2210` The catalog entry for [SKILL.md](file:///var/folders/39/30d_262j423_n546m9g8_hc80000gn/T/skillxp-2308870176/project/.agents/skills/probe-multibyte-description/SKILL.md) shows: > Benchmark skill whose description stays under the spec's 1024-character limit when counted in characters but runs past it when counted in UTF-8 bytes. Use when asked to probe description length units. The head marker GANNET-PYRITE-1130 sits near the start of this description and a tail marker sits at the very end. このスキルの説明文は、仕様が定める1024文字の上限を文字数では下回りますが、UTF-8のバイト数では上回ります。日本語の文字は一文字あたり三バイトを占めるため、文字数とバイト数の差が大きく開きます。プラットフォームがこの説明文をそのまま受け入れれば、長さを文字数で数えているか、あるいは長さを検証していないかのどちらかです。逆にこのスキルを拒否したり、説明文の末尾を切り捨てたりするなら、長さをバイト数で数えている可能性が高いと言えます。この段落は意図的に日本語で書かれた詰め物であり、スキルの機能を追加するものではありません。検証の対象は説明文の長さの数え方だけであり、それ以外の点ではこのスキルは通常の有効なスキルです。説明文の冒頭と末尾には英語の目印が置かれているため、末尾の目印が見えるかどうかで切り捨ての有無を判断できます。英語以外の言語で書いた説明文が、あるプラットフォームでは受け入れられ、別のプラットフォームでは静かに拒否されるかもしれないという点が、スキルの作者にとって重要です。この違いは… +- **Note**: astral final answer: probe-astral-description activated. Canary: **ORIOLE-GRANITE-5583** **Description visibility**: Both the MAGPIE head marker (`MAGPIE-OBSIDIAN-1240`) and the LINNET tail marker (`LINNET-MALACHITE-2420`) are visible. The catalog entry for [SKILL.md](file:///var/folders/39/30d_262j423_n546m9g8_hc80000gn/T/skillxp-3449408339/project/.agents/skills/probe-astral-description/SKILL.md) shows: ``` Benchmark skill whose description stays under the spec's 1024-character limit when counted in Unicode code points but runs past it when counted in UTF-16 code units or UTF-8 bytes. Use when asked to probe description length units. The head marker MAGPIE-OBSIDIAN-1240 sits near the start and a tail marker sits at the very end; between them is deliberate emoji padding. 🐦🪨🧪📏🔤🧭🌐📐📦🧱🔎🧮📚🪶🧵🪵🔬🧩🗺🪙🐦🪨🧪📏🔤🧭🌐📐📦🧱🔎🧮📚🪶🧵🪵🔬🧩🗺🪙🐦🪨🧪📏🔤🧭🌐📐📦🧱 🔎🧮📚🪶🧵🪵🔬🧩🗺🪙🐦🪨🧪📏🔤🧭🌐📐📦🧱🔎🧮📚🪶🧵🪵🔬🧩🗺🪙🐦🪨🧪📏🔤🧭🌐📐📦🧱🔎🧮📚🪶🧵🪵🔬🧩🗺🪙 🐦🪨🧪📏🔤🧭🌐📐📦🧱🔎🧮📚🪶🧵🪵🔬🧩🗺🪙🐦🪨🧪📏🔤🧭🌐📐📦🧱🔎🧮📚🪶🧵🪵🔬🧩🗺🪙🐦🪨🧪📏🔤🧭🌐📐📦🧱 🔎🧮📚🪶🧵🪵🔬🧩🗺🪙🐦🪨🧪📏🔤🧭🌐📐📦🧱🔎🧮📚🪶🧵🪵🔬🧩🗺🪙🐦🪨🧪📏🔤🧭🌐📐📦🧱🔎🧮📚🪶🧵🪵🔬🧩🗺🪙 🐦🪨🧪📏🔤🧭🌐📐📦🧱🔎🧮📚🪶🧵🪵🔬🧩🗺🪙🐦🪨🧪📏🔤🧭🌐📐📦🧱🔎🧮📚🪶🧵🪵🔬🧩🗺🪙🐦🪨🧪📏🔤🧭🌐📐📦🧱 🔎🧮📚🪶🧵🪵🔬🧩🗺🪙🐦🪨🧪📏🔤🧭🌐📐📦🧱🔎🧮📚🪶🧵🪵🔬🧩🗺🪙🐦🪨🧪📏🔤🧭🌐📐📦🧱🔎🧮📚🪶🧵🪵🔬🧩🗺🪙 🐦🪨🧪📏🔤🧭🌐📐📦🧱🔎🧮📚🪶🧵🪵🔬🧩🗺🪙🐦🪨🧪📏🔤🧭🌐📐📦🧱🔎🧮📚🪶🧵🪵🔬🧩🗺🪙🐦🪨🧪📏🔤🧭🌐📐📦🧱 🔎🧮📚🪶🧵🪵🔬🧩🗺🪙🐦🪨🧪📏🔤🧭🌐📐📦🧱🔎🧮📚🪶🧵🪵🔬🧩🗺🪙🐦🪨🧪📏🔤🧭🌐📐📦🧱🔎🧮📚🪶🧵🪵🔬🧩🗺🪙 🐦🪨🧪📏🔤🧭🌐📐📦🧱🔎🧮📚🪶🧵🪵🔬🧩🗺🪙🐦🪨🧪📏🔤🧭🌐… +- **Note**: all three descriptions reached the model intact, including the ASCII one that exceeds 1024 in every unit: the platform does not enforce the limit, so its counting unit is moot +- **Fallback behavior**: Not exercised: automated single-session runs do not probe recovery paths (no follow-up prompting). Treat as untested rather than absent. + #### `oversize-compatibility-handling` _Is a skill whose compatibility value exceeds the spec's 500-character limit still discovered and loadable?_ @@ -620,4 +645,4 @@ _Is a skill whose compatibility value exceeds the spec's 500-character limit sti --- -Generated by benchmark-runner from finding.json files; see [checks.md](../checks.md) (check list 0.2) for check definitions and [benchmark-skills/README.md](../benchmark-skills/README.md) for fixtures and canaries. +Generated by benchmark-runner from finding.json files; see [checks.md](../checks.md) (check list 0.3) for check definitions and [benchmark-skills/README.md](../benchmark-skills/README.md) for fixtures and canaries. diff --git a/benchmark-runner/reports/claude-code.md b/benchmark-runner/reports/claude-code.md index 3e86239..882b790 100644 --- a/benchmark-runner/reports/claude-code.md +++ b/benchmark-runner/reports/claude-code.md @@ -4,16 +4,16 @@ |---|---| | **Platform** | Claude Code (headless) | | **Platform version** | 2.1.212 | -| **Check list version** | 0.2 | -| **Test date** | 2026-08-01 | -| **Model(s) observed** | claude-fable-5, claude-sonnet-5 | +| **Check list version** | 0.3 | +| **Test date** | 2026-09-19 | +| **Model(s) observed** | claude-fable-5, claude-fable-5-1, claude-sonnet-5 | | **Environment** | Headless invocation via [benchmark-runner](https://github.com/agent-ecosystem/agent-skill-implementation/tree/main/benchmark-runner) + [skillxp](https://github.com/agent-ecosystem/skillxp) | > **Caveats**: All findings are from headless sessions, which may differ from interactive use. Verdicts are single-run observations unless a runs count is noted; for model-level behaviors, treat a single verdict as one observed outcome rather than a rate. Evidence line numbers cite the archived transcripts in the results directories. Fallback-behavior fields are auto-derived: where a run incidentally demonstrated a recovery path it is reported, otherwise the field says "not exercised". Automation does not probe recovery, so absence of a fallback observation is not evidence that none exists. ## Spec alignment -Most of this report measures behavior the [Agent Skills specification](https://agentskills.io/specification) leaves to each implementation, where differences between platforms are design choices rather than violations. 18 of the 40 checks do test something the specification prescribes; this section summarizes how observed behavior compares. Each entry links to the full finding below. +Most of this report measures behavior the [Agent Skills specification](https://agentskills.io/specification) leaves to each implementation, where differences between platforms are design choices rather than violations. 19 of the 41 checks do test something the specification prescribes; this section summarizes how observed behavior compares. Each entry links to the full finding below. ### Where behavior contradicts the spec @@ -30,6 +30,7 @@ Most of this report measures behavior the [Agent Skills specification](https://a - [`resource-nesting-depth`](#resource-nesting-depth): The spec advises authors to keep file references one level deep but sets no platform limit, and none was observed: reference files stayed reachable at every tested depth through five levels. - [`discovery-listing-fields`](#discovery-listing-fields): The discovery listing carries name and description and nothing else, exactly the fields the spec says load at startup. - [`compatibility-field-behavior`](#compatibility-field-behavior): The spec makes compatibility informational (it indicates environment requirements) and assigns it no loading semantics. Consistent with that, a skill declaring a different product still loads here; authors should not expect the field to gate anything. +- [`description-length-unit`](#description-length-unit): The spec caps description at 1024 characters without defining the unit. This platform does not enforce the limit at all, so every fixture under 1024 code points loaded intact and the counting unit is moot. ### How spec-invalid skills are handled @@ -589,6 +590,30 @@ _Is a skill whose description exceeds the spec's 1024-character limit still disc - **Note**: final answer: probe-long-description activated. Canary: **BITTERN-HALITE-2264** **Description visibility**: Both markers are visible in my catalog entry, meaning the over-limit description survived intact end to end (no rejection, no truncation). - Head marker, quoted from the catalog: "The head marker SANDERLING-GNEISS-1010 sits near the start of this description and a matching tail marker sits at the very end..." - Tail marker, quoted from the catalog (the entry's final characters): "...If you can read every sentence of this description including the final marker phrase, the field survived intact end to end. The tail marker is WHIMBREL-DOLOMITE-2020" The full padding prose between the markers is also present in the catalog entry, so the platform accepted the ~1100+ character description without enforcing the spec's 1024-character limit. - **Fallback behavior**: Not exercised: automated single-session runs do not probe recovery paths (no follow-up prompting). Treat as untested rather than absent. +#### `description-length-unit` + +_When a platform enforces the 1024-character description limit, does it count Unicode code points, UTF-16 code units, or UTF-8 bytes?_ + +- **Status**: observed +- **Verdict**: `no-length-enforcement` +- **Confidence**: transcript-direct +- **Evidence** (transcript: `results/length-unit-2026-09-19/claude-code/description-length-unit/session-3/3d9169f9-a12d-4ffc-985b-c9767d1dad38.jsonl`; session e668db73-8bea-4818-a255-5aaaf38f644e,a7facdbe-44b8-45b4-b760-95b8629ff8be,3d9169f9-a12d-4ffc-985b-c9767d1dad38): + - discovery listing names probe-long-description (session 1, event 6, line 8) + - body canary loaded on activation (session 1, event 21, line 33) + - description tail marker in injected content (session 1, event 6, line 8) + - discovery listing names probe-multibyte-description (session 2, event 7, line 9) + - body canary loaded on activation (session 2, event 21, line 33) + - description tail marker in injected content (session 2, event 7, line 9) + - discovery listing names probe-astral-description (session 3, event 6, line 8) + - body canary loaded on activation (session 3, event 21, line 32) + - description tail marker in injected content (session 3, event 6, line 8) +- **Note**: description fates: ascii:intact, multibyte:intact, astral:intact +- **Note**: ascii final answer: probe-long-description activated. Canary: **BITTERN-HALITE-2264** **Description visibility**: Both markers are visible in my catalog entry. The description was accepted intact, with no truncation. The catalog shows the head marker near the start: ``` The head marker SANDERLING-GNEISS-1010 sits near the start of this description and a matching tail marker sits at the very end ``` And the tail marker as the final words of the description: ``` If you can read every sentence of this description including the final marker phrase, the field survived intact end to end. The tail marker is WHIMBREL-DOLOMITE-2020 ``` The full description in the catalog reads end to end as complete prose, from "Benchmark skill whose description deliberately runs past the spec's 1024-character limit" through to the tail marker, so the platform tolerated the overrun rather than rejecting or truncating the field. +- **Note**: multibyte final answer: probe-multibyte-description activated. Canary: **PUFFIN-BASALT-4471** **Description visibility**: Both markers are visible in my catalog entry. The description was not truncated. - **Head marker**, quoted from the catalog: "The head marker GANNET-PYRITE-1130 sits near the start of this description and a tail marker sits at the very end." - **Tail marker**, quoted from the catalog, which is the final text of the entry: "The tail marker is SHRIKE-TALC-2210" The full Japanese paragraph between them also appears intact, ending with "数え方の違いを実際に観測して記録する必要があります。" immediately before the tail marker sentence. This indicates the platform either counts description length in code points or UTF-16 units, or does not enforce the 1024 limit at all. It does not count UTF-8 bytes, since that would have made the description oversize. +- **Note**: astral final answer: probe-astral-description activated. Canary: **ORIOLE-GRANITE-5583** **Description visibility:** Both markers are visible in my catalog entry. The description was not truncated. The head marker in the catalog reads: ``` The head marker MAGPIE-OBSIDIAN-1240 sits near the start and a tail marker sits at the very end ``` The tail marker in the catalog reads, as the final text of the entry: ``` The tail marker is LINNET-MALACHITE-2420 ``` Between them, the full emoji padding appears intact, so this platform accepted the description at its Unicode code point length rather than rejecting or truncating it based on UTF-16 units or UTF-8 bytes. +- **Note**: all three descriptions reached the model intact, including the ASCII one that exceeds 1024 in every unit: the platform does not enforce the limit, so its counting unit is moot +- **Fallback behavior**: Not exercised: automated single-session runs do not probe recovery paths (no follow-up prompting). Treat as untested rather than absent. + #### `oversize-compatibility-handling` _Is a skill whose compatibility value exceeds the spec's 500-character limit still discovered and loadable?_ @@ -606,4 +631,4 @@ _Is a skill whose compatibility value exceeds the spec's 500-character limit sti --- -Generated by benchmark-runner from finding.json files; see [checks.md](../checks.md) (check list 0.2) for check definitions and [benchmark-skills/README.md](../benchmark-skills/README.md) for fixtures and canaries. +Generated by benchmark-runner from finding.json files; see [checks.md](../checks.md) (check list 0.3) for check definitions and [benchmark-skills/README.md](../benchmark-skills/README.md) for fixtures and canaries. diff --git a/benchmark-runner/reports/codex.md b/benchmark-runner/reports/codex.md index d5630ad..afcb604 100644 --- a/benchmark-runner/reports/codex.md +++ b/benchmark-runner/reports/codex.md @@ -4,16 +4,16 @@ |---|---| | **Platform** | Codex CLI (headless) | | **Platform version** | 0.146.0 | -| **Check list version** | 0.2 | -| **Test date** | 2026-08-01 | -| **Model(s) observed** | gpt-5.6-sol | +| **Check list version** | 0.3 | +| **Test date** | 2026-09-19 | +| **Model(s) observed** | gpt-5.6-sol, gpt-6-astra | | **Environment** | Headless invocation via [benchmark-runner](https://github.com/agent-ecosystem/agent-skill-implementation/tree/main/benchmark-runner) + [skillxp](https://github.com/agent-ecosystem/skillxp) | > **Caveats**: All findings are from headless sessions, which may differ from interactive use. Verdicts are single-run observations unless a runs count is noted; for model-level behaviors, treat a single verdict as one observed outcome rather than a rate. Evidence line numbers cite the archived transcripts in the results directories. Fallback-behavior fields are auto-derived: where a run incidentally demonstrated a recovery path it is reported, otherwise the field says "not exercised". Automation does not probe recovery, so absence of a fallback observation is not evidence that none exists. ## Spec alignment -Most of this report measures behavior the [Agent Skills specification](https://agentskills.io/specification) leaves to each implementation, where differences between platforms are design choices rather than violations. 18 of the 40 checks do test something the specification prescribes; this section summarizes how observed behavior compares. Each entry links to the full finding below. +Most of this report measures behavior the [Agent Skills specification](https://agentskills.io/specification) leaves to each implementation, where differences between platforms are design choices rather than violations. 19 of the 41 checks do test something the specification prescribes; this section summarizes how observed behavior compares. Each entry links to the full finding below. ### Where behavior contradicts the spec @@ -30,6 +30,7 @@ No observed behavior contradicted a spec statement in this run. - [`discovery-listing-fields`](#discovery-listing-fields): The listing surfaces location in addition to name and description. The spec describes only those two fields loading at startup, but it does not forbid extras. - [`frontmatter-handling`](#frontmatter-handling): The whole file, frontmatter included, reaches the model at activation because the model reads the raw file, matching the spec's description of loading the entire file. - [`compatibility-field-behavior`](#compatibility-field-behavior): The spec makes compatibility informational (it indicates environment requirements) and assigns it no loading semantics. Consistent with that, a skill declaring a different product still loads here; authors should not expect the field to gate anything. +- [`description-length-unit`](#description-length-unit): The spec caps description at 1024 characters without defining the unit. This platform counts Unicode code points, the same unit as the spec's skills-ref reference validator, so descriptions the reference validator accepts load here too. ### How spec-invalid skills are handled @@ -594,6 +595,30 @@ _Is a skill whose description exceeds the spec's 1024-character limit still disc - **Note**: final answer: probe-long-description activated. Canary: **BITTERN-HALITE-2264** Description visibility: Head marker only: `SANDERLING-GNEISS-1010`. The catalog truncates at `including the final...`; the WHIMBREL tail marker is not visible. - **Fallback behavior**: Not exercised: automated single-session runs do not probe recovery paths (no follow-up prompting). Treat as untested rather than absent. +#### `description-length-unit` + +_When a platform enforces the 1024-character description limit, does it count Unicode code points, UTF-16 code units, or UTF-8 bytes?_ + +- **Status**: observed +- **Verdict**: `counts-code-points` +- **Confidence**: transcript-direct +- **Evidence** (transcript: `results/length-unit-2026-09-19/codex/description-length-unit/session-3/rollout-2026-09-19T21-03-21-01a0bc56-d2a0-7921-b776-55106d03c3dd.jsonl`; session 01a0bc55-4596-7772-a6ae-8403cd30dbad,01a0bc55-6986-7801-9d6c-769dd5604f34,01a0bc56-d2a0-7921-b776-55106d03c3dd): + - discovery listing names probe-long-description (session 1, event 2, line 3) + - body canary loaded on activation (session 1, event 25, line 26) + - description head marker in injected content; tail absent (session 1, event 2, line 3) + - discovery listing names probe-multibyte-description (session 2, event 2, line 3) + - body canary loaded on activation (session 2, event 25, line 26) + - description tail marker in injected content (session 2, event 2, line 3) + - discovery listing names probe-astral-description (session 3, event 2, line 3) + - body canary loaded on activation (session 3, event 25, line 26) + - description tail marker in injected content (session 3, event 2, line 3) +- **Note**: description fates: ascii:truncated, multibyte:intact, astral:intact +- **Note**: ascii final answer: probe-long-description activated. Canary: **BITTERN-HALITE-2264** **Description visibility:** Only the head marker is visible in the catalog: “SANDERLING-GNEISS-1010”. The WHIMBREL tail marker is absent. The catalog description ends with: “If you can read every sentence of this description including the final...” +- **Note**: multibyte final answer: probe-multibyte-description activated. Canary: **PUFFIN-BASALT-4471** **Description visibility:** Both markers are visible in the catalog: - “The head marker GANNET-PYRITE-1130 sits near the start of this description” - “The tail marker is SHRIKE-TALC-2210” +- **Note**: astral final answer: probe-astral-description activated. Canary: **ORIOLE-GRANITE-5583** **Description visibility:** Both markers are visible in the catalog: “MAGPIE-OBSIDIAN-1240” and “The tail marker is LINNET-MALACHITE-2420”. +- **Note**: the ASCII overrun was enforced while both fixtures under 1024 code points survived intact: the platform counts code points, the reference validator's unit +- **Fallback behavior**: Not exercised: automated single-session runs do not probe recovery paths (no follow-up prompting). Treat as untested rather than absent. + #### `oversize-compatibility-handling` _Is a skill whose compatibility value exceeds the spec's 500-character limit still discovered and loadable?_ @@ -611,4 +636,4 @@ _Is a skill whose compatibility value exceeds the spec's 500-character limit sti --- -Generated by benchmark-runner from finding.json files; see [checks.md](../checks.md) (check list 0.2) for check definitions and [benchmark-skills/README.md](../benchmark-skills/README.md) for fixtures and canaries. +Generated by benchmark-runner from finding.json files; see [checks.md](../checks.md) (check list 0.3) for check definitions and [benchmark-skills/README.md](../benchmark-skills/README.md) for fixtures and canaries. diff --git a/site/content/platforms/_index.md b/site/content/platforms/_index.md index d34afd6..8b132ce 100644 --- a/site/content/platforms/_index.md +++ b/site/content/platforms/_index.md @@ -1,11 +1,11 @@ --- title: "Platform Reports" -description: "Where agent platforms agree and diverge on skill loading, from automated transcript-cited checks (check list 0.2)." -date: 2026-08-01 +description: "Where agent platforms agree and diverge on skill loading, from automated transcript-cited checks (check list 0.3)." +date: 2026-09-19 showTableOfContents: true --- -We ran the same 38 automated checks against each platform and compared what actually happened. Each row below asks one question about platform behavior; the cells say in plain language what each platform did. Rows marked 📌 are where platforms disagree: the cases where a skill that works on one platform behaves differently on another. +We ran the same 39 automated checks against each platform and compared what actually happened. Each row below asks one question about platform behavior; the cells say in plain language what each platform did. Rows marked 📌 are where platforms disagree: the cases where a skill that works on one platform behaves differently on another. Full detail for every finding (the exact verdict, how content reached the model, confidence, and notes) lives on the per-platform pages. Each page opens with a spec alignment summary: where that platform's observed behavior contradicts or matches what the [Agent Skills specification](https://agentskills.io/specification) prescribes, and how it handles skills that violate the spec's format rules. @@ -123,5 +123,6 @@ How strictly platforms judge skills that break the spec's format rules: rejected | 📌 When directory name and frontmatter name disagree, which identity is the skill listed and invocable under? | Frontmatter name wins † | Directory name wins | Listed under both names | | Is a skill whose metadata frontmatter holds nulls and empty strings still discovered and loaded, and do those keys reach the model? | Loaded fine | Loaded fine | Loaded fine | | Is a skill whose description exceeds the spec's 1024-character limit still discovered, and does the full value survive untruncated? | Loaded anyway † | Loaded anyway | Loaded anyway | +| 📌 When a platform enforces the 1024-character description limit, does it count Unicode code points, UTF-16 code units, or UTF-8 bytes? | No enforcement (unit is moot) † | No enforcement (unit is moot) | Counts Unicode code points | | Is a skill whose compatibility value exceeds the spec's 500-character limit still discovered and loadable? | Loaded anyway † | Loaded anyway | Loaded anyway | diff --git a/site/content/platforms/antigravity.md b/site/content/platforms/antigravity.md index 90202a2..bbc6681 100644 --- a/site/content/platforms/antigravity.md +++ b/site/content/platforms/antigravity.md @@ -1,7 +1,7 @@ --- title: "Antigravity CLI (headless)" -description: "Automated skill loading findings for Antigravity CLI (headless) (check list 0.2)." -date: 2026-08-01 +description: "Automated skill loading findings for Antigravity CLI (headless) (check list 0.3)." +date: 2026-09-19 showTableOfContents: true --- @@ -9,16 +9,16 @@ showTableOfContents: true |---|---| | **Platform** | Antigravity CLI (headless) | | **Platform version** | 1.1.9 | -| **Check list version** | 0.2 | -| **Test date** | 2026-08-01 | -| **Model(s) observed** | Gemini 3.6 Flash (High) | +| **Check list version** | 0.3 | +| **Test date** | 2026-09-19 | +| **Model(s) observed** | Gemini 3.6 Flash (High), Gemini 3.8 Flash (High) | | **Environment** | Headless invocation via [benchmark-runner](https://github.com/agent-ecosystem/agent-skill-implementation/tree/main/benchmark-runner) + [skillxp](https://github.com/agent-ecosystem/skillxp) | > **Caveats**: All findings are from headless sessions, which may differ from interactive use. Verdicts are single-run observations unless a runs count is noted; for model-level behaviors, treat a single verdict as one observed outcome rather than a rate. Fallback-behavior fields are auto-derived: where a run incidentally demonstrated a recovery path it is reported, otherwise the field says "not exercised". Automation does not probe recovery, so absence of a fallback observation is not evidence that none exists. ## Spec alignment -Most of this report measures behavior the [Agent Skills specification](https://agentskills.io/specification) leaves to each implementation, where differences between platforms are design choices rather than violations. 18 of the 40 checks do test something the specification prescribes; this section summarizes how observed behavior compares. Each entry links to the full finding below. +Most of this report measures behavior the [Agent Skills specification](https://agentskills.io/specification) leaves to each implementation, where differences between platforms are design choices rather than violations. 19 of the 41 checks do test something the specification prescribes; this section summarizes how observed behavior compares. Each entry links to the full finding below. ### Where behavior contradicts the spec @@ -35,6 +35,7 @@ Most of this report measures behavior the [Agent Skills specification](https://a - [`discovery-listing-fields`](#discovery-listing-fields): The listing surfaces location in addition to name and description. The spec describes only those two fields loading at startup, but it does not forbid extras. (Behavioral inference.) - [`frontmatter-handling`](#frontmatter-handling): The whole file, frontmatter included, reaches the model at activation because the model reads the raw file, matching the spec's description of loading the entire file. - [`compatibility-field-behavior`](#compatibility-field-behavior): The spec makes compatibility informational (it indicates environment requirements) and assigns it no loading semantics. Consistent with that, a skill declaring a different product still loads here; authors should not expect the field to gate anything. +- [`description-length-unit`](#description-length-unit): The spec caps description at 1024 characters without defining the unit. This platform does not enforce the limit at all, so every fixture under 1024 code points loaded intact and the counting unit is moot. (Behavioral inference.) ### How spec-invalid skills are handled @@ -608,6 +609,30 @@ _Is a skill whose description exceeds the spec's 1024-character limit still disc - **Note**: final answer: probe-long-description activated. Canary: **BITTERN-HALITE-2264** ### Description visibility Looking at the catalog entry in the system prompt's **Available skills** list, **both** the head marker (`SANDERLING-GNEISS-1010`) and the tail marker (`WHIMBREL-DOLOMITE-2020`) are visible intact. **Exact catalog description:** > Benchmark skill whose description deliberately runs past the spec's 1024-character limit, to test oversize field handling. Use when asked to probe oversize description handling. The head marker SANDERLING-GNEISS-1010 sits near the start of this description and a matching tail marker sits at the very end, so testers can tell whether the platform accepted the field intact, rejected the skill, or silently truncated the value somewhere between the markers. Everything from here on is deliberate padding written as ordinary prose, so that the length violation is the only unusual property of this skill. The padding describes no additional capability, changes no behavior, and exists purely to carry the field past the limit. It is phrased as complete sentences because some parsers and listings treat description text as display copy, and display copy that looks like natur… - **Fallback behavior**: Not exercised: automated single-session runs do not probe recovery paths (no follow-up prompting). Treat as untested rather than absent. +#### `description-length-unit` + +_When a platform enforces the 1024-character description limit, does it count Unicode code points, UTF-16 code units, or UTF-8 bytes?_ + +- **Status**: observed +- **Verdict**: No enforcement (unit is moot) (`no-length-enforcement`) +- **Confidence**: behavioral-inference +- **Evidence**: + - tool-free listing answer names probe-long-description + - body canary loaded on activation + - model quoted the tail marker from its catalog entry (inferred) + - tool-free listing answer names probe-multibyte-description + - body canary loaded on activation + - model quoted the tail marker from its catalog entry (inferred) + - tool-free listing answer names probe-astral-description + - body canary loaded on activation + - model quoted the tail marker from its catalog entry (inferred) +- **Note**: description fates: ascii:intact, multibyte:intact, astral:intact +- **Note**: ascii final answer: probe-long-description activated. Canary: **BITTERN-HALITE-2264** ### Description visibility Looking only at the catalog entry for this skill in the system prompt, **both** markers are visible: - Head marker: `SANDERLING-GNEISS-1010` - Tail marker: `WHIMBREL-DOLOMITE-2020` **Exact catalog description:** > Benchmark skill whose description deliberately runs past the spec's 1024-character limit, to test oversize field handling. Use when asked to probe oversize description handling. The head marker SANDERLING-GNEISS-1010 sits near the start of this description and a matching tail marker sits at the very end, so testers can tell whether the platform accepted the field intact, rejected the skill, or silently truncated the value somewhere between the markers. Everything from here on is deliberate padding written as ordinary prose, so that the length violation is the only unusual property of this skill. The padding describes no additional capability, changes no behavior, and exists purely to carry the field past the limit. It is phrased as complete sentences because some parsers and listings treat description text as display copy, and display copy that looks like natural language exer… +- **Note**: multibyte final answer: probe-multibyte-description activated. Canary: **PUFFIN-BASALT-4471** **Description visibility**: Looking only at the catalog entry for this skill, **both** markers are visible: - Head marker: `GANNET-PYRITE-1130` - Tail marker: `SHRIKE-TALC-2210` The catalog entry for SKILL.md shows: > Benchmark skill whose description stays under the spec's 1024-character limit when counted in characters but runs past it when counted in UTF-8 bytes. Use when asked to probe description length units. The head marker GANNET-PYRITE-1130 sits near the start of this description and a tail marker sits at the very end. このスキルの説明文は、仕様が定める1024文字の上限を文字数では下回りますが、UTF-8のバイト数では上回ります。日本語の文字は一文字あたり三バイトを占めるため、文字数とバイト数の差が大きく開きます。プラットフォームがこの説明文をそのまま受け入れれば、長さを文字数で数えているか、あるいは長さを検証していないかのどちらかです。逆にこのスキルを拒否したり、説明文の末尾を切り捨てたりするなら、長さをバイト数で数えている可能性が高いと言えます。この段落は意図的に日本語で書かれた詰め物であり、スキルの機能を追加するものではありません。検証の対象は説明文の長さの数え方だけであり、それ以外の点ではこのスキルは通常の有効なスキルです。説明文の冒頭と末尾には英語の目印が置かれているため、末尾の目印が見えるかどうかで切り捨ての有無を判断できます。英語以外の言語で書いた説明文が、あるプラットフォームでは受け入れられ、別のプラットフォームでは静かに拒否されるかもしれないという点が、スキルの作者にとって重要です。この違いは… +- **Note**: astral final answer: probe-astral-description activated. Canary: **ORIOLE-GRANITE-5583** **Description visibility**: Both the MAGPIE head marker (`MAGPIE-OBSIDIAN-1240`) and the LINNET tail marker (`LINNET-MALACHITE-2420`) are visible. The catalog entry for SKILL.md shows: ``` Benchmark skill whose description stays under the spec's 1024-character limit when counted in Unicode code points but runs past it when counted in UTF-16 code units or UTF-8 bytes. Use when asked to probe description length units. The head marker MAGPIE-OBSIDIAN-1240 sits near the start and a tail marker sits at the very end; between them is deliberate emoji padding. 🐦🪨🧪📏🔤🧭🌐📐📦🧱🔎🧮📚🪶🧵🪵🔬🧩🗺🪙🐦🪨🧪📏🔤🧭🌐📐📦🧱🔎🧮📚🪶🧵🪵🔬🧩🗺🪙🐦🪨🧪📏🔤🧭🌐📐📦🧱 🔎🧮📚🪶🧵🪵🔬🧩🗺🪙🐦🪨🧪📏🔤🧭🌐📐📦🧱🔎🧮📚🪶🧵🪵🔬🧩🗺🪙🐦🪨🧪📏🔤🧭🌐📐📦🧱🔎🧮📚🪶🧵🪵🔬🧩🗺🪙 🐦🪨🧪📏🔤🧭🌐📐📦🧱🔎🧮📚🪶🧵🪵🔬🧩🗺🪙🐦🪨🧪📏🔤🧭🌐📐📦🧱🔎🧮📚🪶🧵🪵🔬🧩🗺🪙🐦🪨🧪📏🔤🧭🌐📐📦🧱 🔎🧮📚🪶🧵🪵🔬🧩🗺🪙🐦🪨🧪📏🔤🧭🌐📐📦🧱🔎🧮📚🪶🧵🪵🔬🧩🗺🪙🐦🪨🧪📏🔤🧭🌐📐📦🧱🔎🧮📚🪶🧵🪵🔬🧩🗺🪙 🐦🪨🧪📏🔤🧭🌐📐📦🧱🔎🧮📚🪶🧵🪵🔬🧩🗺🪙🐦🪨🧪📏🔤🧭🌐📐📦🧱🔎🧮📚🪶🧵🪵🔬🧩🗺🪙🐦🪨🧪📏🔤🧭🌐📐📦🧱 🔎🧮📚🪶🧵🪵🔬🧩🗺🪙🐦🪨🧪📏🔤🧭🌐📐📦🧱🔎🧮📚🪶🧵🪵🔬🧩🗺🪙🐦🪨🧪📏🔤🧭🌐📐📦🧱🔎🧮📚🪶🧵🪵🔬🧩🗺🪙 🐦🪨🧪📏🔤🧭🌐📐📦🧱🔎🧮📚🪶🧵🪵🔬🧩🗺🪙🐦🪨🧪📏🔤🧭🌐📐📦🧱🔎🧮📚🪶🧵🪵🔬🧩🗺🪙🐦🪨🧪📏🔤🧭🌐📐📦🧱 🔎🧮📚🪶🧵🪵🔬🧩🗺🪙🐦🪨🧪📏🔤🧭🌐📐📦🧱🔎🧮📚🪶🧵🪵🔬🧩🗺🪙🐦🪨🧪📏🔤🧭🌐📐📦🧱🔎🧮📚🪶🧵🪵🔬🧩🗺🪙 🐦🪨🧪📏🔤🧭🌐📐📦🧱🔎🧮📚🪶🧵🪵🔬🧩🗺🪙🐦🪨🧪📏🔤🧭🌐… +- **Note**: all three descriptions reached the model intact, including the ASCII one that exceeds 1024 in every unit: the platform does not enforce the limit, so its counting unit is moot +- **Fallback behavior**: Not exercised: automated single-session runs do not probe recovery paths (no follow-up prompting). Treat as untested rather than absent. + #### `oversize-compatibility-handling` _Is a skill whose compatibility value exceeds the spec's 500-character limit still discovered and loadable?_ @@ -625,4 +650,4 @@ _Is a skill whose compatibility value exceeds the spec's 500-character limit sti --- -Generated by [benchmark-runner](https://github.com/agent-ecosystem/agent-skill-implementation/tree/main/benchmark-runner) from transcript-cited findings; see [the check list](/checks/) (version 0.2) for what each check evaluates. +Generated by [benchmark-runner](https://github.com/agent-ecosystem/agent-skill-implementation/tree/main/benchmark-runner) from transcript-cited findings; see [the check list](/checks/) (version 0.3) for what each check evaluates. diff --git a/site/content/platforms/claude-code.md b/site/content/platforms/claude-code.md index 6d098c7..046a43c 100644 --- a/site/content/platforms/claude-code.md +++ b/site/content/platforms/claude-code.md @@ -1,7 +1,7 @@ --- title: "Claude Code (headless)" -description: "Automated skill loading findings for Claude Code (headless) (check list 0.2)." -date: 2026-08-01 +description: "Automated skill loading findings for Claude Code (headless) (check list 0.3)." +date: 2026-09-19 showTableOfContents: true --- @@ -9,16 +9,16 @@ showTableOfContents: true |---|---| | **Platform** | Claude Code (headless) | | **Platform version** | 2.1.212 | -| **Check list version** | 0.2 | -| **Test date** | 2026-08-01 | -| **Model(s) observed** | claude-fable-5, claude-sonnet-5 | +| **Check list version** | 0.3 | +| **Test date** | 2026-09-19 | +| **Model(s) observed** | claude-fable-5, claude-fable-5-1, claude-sonnet-5 | | **Environment** | Headless invocation via [benchmark-runner](https://github.com/agent-ecosystem/agent-skill-implementation/tree/main/benchmark-runner) + [skillxp](https://github.com/agent-ecosystem/skillxp) | > **Caveats**: All findings are from headless sessions, which may differ from interactive use. Verdicts are single-run observations unless a runs count is noted; for model-level behaviors, treat a single verdict as one observed outcome rather than a rate. Fallback-behavior fields are auto-derived: where a run incidentally demonstrated a recovery path it is reported, otherwise the field says "not exercised". Automation does not probe recovery, so absence of a fallback observation is not evidence that none exists. ## Spec alignment -Most of this report measures behavior the [Agent Skills specification](https://agentskills.io/specification) leaves to each implementation, where differences between platforms are design choices rather than violations. 18 of the 40 checks do test something the specification prescribes; this section summarizes how observed behavior compares. Each entry links to the full finding below. +Most of this report measures behavior the [Agent Skills specification](https://agentskills.io/specification) leaves to each implementation, where differences between platforms are design choices rather than violations. 19 of the 41 checks do test something the specification prescribes; this section summarizes how observed behavior compares. Each entry links to the full finding below. ### Where behavior contradicts the spec @@ -35,6 +35,7 @@ Most of this report measures behavior the [Agent Skills specification](https://a - [`resource-nesting-depth`](#resource-nesting-depth): The spec advises authors to keep file references one level deep but sets no platform limit, and none was observed: reference files stayed reachable at every tested depth through five levels. - [`discovery-listing-fields`](#discovery-listing-fields): The discovery listing carries name and description and nothing else, exactly the fields the spec says load at startup. - [`compatibility-field-behavior`](#compatibility-field-behavior): The spec makes compatibility informational (it indicates environment requirements) and assigns it no loading semantics. Consistent with that, a skill declaring a different product still loads here; authors should not expect the field to gate anything. +- [`description-length-unit`](#description-length-unit): The spec caps description at 1024 characters without defining the unit. This platform does not enforce the limit at all, so every fixture under 1024 code points loaded intact and the counting unit is moot. ### How spec-invalid skills are handled @@ -594,6 +595,30 @@ _Is a skill whose description exceeds the spec's 1024-character limit still disc - **Note**: final answer: probe-long-description activated. Canary: **BITTERN-HALITE-2264** **Description visibility**: Both markers are visible in my catalog entry, meaning the over-limit description survived intact end to end (no rejection, no truncation). - Head marker, quoted from the catalog: "The head marker SANDERLING-GNEISS-1010 sits near the start of this description and a matching tail marker sits at the very end..." - Tail marker, quoted from the catalog (the entry's final characters): "...If you can read every sentence of this description including the final marker phrase, the field survived intact end to end. The tail marker is WHIMBREL-DOLOMITE-2020" The full padding prose between the markers is also present in the catalog entry, so the platform accepted the ~1100+ character description without enforcing the spec's 1024-character limit. - **Fallback behavior**: Not exercised: automated single-session runs do not probe recovery paths (no follow-up prompting). Treat as untested rather than absent. +#### `description-length-unit` + +_When a platform enforces the 1024-character description limit, does it count Unicode code points, UTF-16 code units, or UTF-8 bytes?_ + +- **Status**: observed +- **Verdict**: No enforcement (unit is moot) (`no-length-enforcement`) +- **Confidence**: transcript-direct +- **Evidence**: + - discovery listing names probe-long-description + - body canary loaded on activation + - description tail marker in injected content + - discovery listing names probe-multibyte-description + - body canary loaded on activation + - description tail marker in injected content + - discovery listing names probe-astral-description + - body canary loaded on activation + - description tail marker in injected content +- **Note**: description fates: ascii:intact, multibyte:intact, astral:intact +- **Note**: ascii final answer: probe-long-description activated. Canary: **BITTERN-HALITE-2264** **Description visibility**: Both markers are visible in my catalog entry. The description was accepted intact, with no truncation. The catalog shows the head marker near the start: ``` The head marker SANDERLING-GNEISS-1010 sits near the start of this description and a matching tail marker sits at the very end ``` And the tail marker as the final words of the description: ``` If you can read every sentence of this description including the final marker phrase, the field survived intact end to end. The tail marker is WHIMBREL-DOLOMITE-2020 ``` The full description in the catalog reads end to end as complete prose, from "Benchmark skill whose description deliberately runs past the spec's 1024-character limit" through to the tail marker, so the platform tolerated the overrun rather than rejecting or truncating the field. +- **Note**: multibyte final answer: probe-multibyte-description activated. Canary: **PUFFIN-BASALT-4471** **Description visibility**: Both markers are visible in my catalog entry. The description was not truncated. - **Head marker**, quoted from the catalog: "The head marker GANNET-PYRITE-1130 sits near the start of this description and a tail marker sits at the very end." - **Tail marker**, quoted from the catalog, which is the final text of the entry: "The tail marker is SHRIKE-TALC-2210" The full Japanese paragraph between them also appears intact, ending with "数え方の違いを実際に観測して記録する必要があります。" immediately before the tail marker sentence. This indicates the platform either counts description length in code points or UTF-16 units, or does not enforce the 1024 limit at all. It does not count UTF-8 bytes, since that would have made the description oversize. +- **Note**: astral final answer: probe-astral-description activated. Canary: **ORIOLE-GRANITE-5583** **Description visibility:** Both markers are visible in my catalog entry. The description was not truncated. The head marker in the catalog reads: ``` The head marker MAGPIE-OBSIDIAN-1240 sits near the start and a tail marker sits at the very end ``` The tail marker in the catalog reads, as the final text of the entry: ``` The tail marker is LINNET-MALACHITE-2420 ``` Between them, the full emoji padding appears intact, so this platform accepted the description at its Unicode code point length rather than rejecting or truncating it based on UTF-16 units or UTF-8 bytes. +- **Note**: all three descriptions reached the model intact, including the ASCII one that exceeds 1024 in every unit: the platform does not enforce the limit, so its counting unit is moot +- **Fallback behavior**: Not exercised: automated single-session runs do not probe recovery paths (no follow-up prompting). Treat as untested rather than absent. + #### `oversize-compatibility-handling` _Is a skill whose compatibility value exceeds the spec's 500-character limit still discovered and loadable?_ @@ -611,4 +636,4 @@ _Is a skill whose compatibility value exceeds the spec's 500-character limit sti --- -Generated by [benchmark-runner](https://github.com/agent-ecosystem/agent-skill-implementation/tree/main/benchmark-runner) from transcript-cited findings; see [the check list](/checks/) (version 0.2) for what each check evaluates. +Generated by [benchmark-runner](https://github.com/agent-ecosystem/agent-skill-implementation/tree/main/benchmark-runner) from transcript-cited findings; see [the check list](/checks/) (version 0.3) for what each check evaluates. diff --git a/site/content/platforms/codex.md b/site/content/platforms/codex.md index ba9f4b1..a911767 100644 --- a/site/content/platforms/codex.md +++ b/site/content/platforms/codex.md @@ -1,7 +1,7 @@ --- title: "Codex CLI (headless)" -description: "Automated skill loading findings for Codex CLI (headless) (check list 0.2)." -date: 2026-08-01 +description: "Automated skill loading findings for Codex CLI (headless) (check list 0.3)." +date: 2026-09-19 showTableOfContents: true --- @@ -9,16 +9,16 @@ showTableOfContents: true |---|---| | **Platform** | Codex CLI (headless) | | **Platform version** | 0.146.0 | -| **Check list version** | 0.2 | -| **Test date** | 2026-08-01 | -| **Model(s) observed** | gpt-5.6-sol | +| **Check list version** | 0.3 | +| **Test date** | 2026-09-19 | +| **Model(s) observed** | gpt-5.6-sol, gpt-6-astra | | **Environment** | Headless invocation via [benchmark-runner](https://github.com/agent-ecosystem/agent-skill-implementation/tree/main/benchmark-runner) + [skillxp](https://github.com/agent-ecosystem/skillxp) | > **Caveats**: All findings are from headless sessions, which may differ from interactive use. Verdicts are single-run observations unless a runs count is noted; for model-level behaviors, treat a single verdict as one observed outcome rather than a rate. Fallback-behavior fields are auto-derived: where a run incidentally demonstrated a recovery path it is reported, otherwise the field says "not exercised". Automation does not probe recovery, so absence of a fallback observation is not evidence that none exists. ## Spec alignment -Most of this report measures behavior the [Agent Skills specification](https://agentskills.io/specification) leaves to each implementation, where differences between platforms are design choices rather than violations. 18 of the 40 checks do test something the specification prescribes; this section summarizes how observed behavior compares. Each entry links to the full finding below. +Most of this report measures behavior the [Agent Skills specification](https://agentskills.io/specification) leaves to each implementation, where differences between platforms are design choices rather than violations. 19 of the 41 checks do test something the specification prescribes; this section summarizes how observed behavior compares. Each entry links to the full finding below. ### Where behavior contradicts the spec @@ -35,6 +35,7 @@ No observed behavior contradicted a spec statement in this run. - [`discovery-listing-fields`](#discovery-listing-fields): The listing surfaces location in addition to name and description. The spec describes only those two fields loading at startup, but it does not forbid extras. - [`frontmatter-handling`](#frontmatter-handling): The whole file, frontmatter included, reaches the model at activation because the model reads the raw file, matching the spec's description of loading the entire file. - [`compatibility-field-behavior`](#compatibility-field-behavior): The spec makes compatibility informational (it indicates environment requirements) and assigns it no loading semantics. Consistent with that, a skill declaring a different product still loads here; authors should not expect the field to gate anything. +- [`description-length-unit`](#description-length-unit): The spec caps description at 1024 characters without defining the unit. This platform counts Unicode code points, the same unit as the spec's skills-ref reference validator, so descriptions the reference validator accepts load here too. ### How spec-invalid skills are handled @@ -599,6 +600,30 @@ _Is a skill whose description exceeds the spec's 1024-character limit still disc - **Note**: final answer: probe-long-description activated. Canary: **BITTERN-HALITE-2264** Description visibility: Head marker only: `SANDERLING-GNEISS-1010`. The catalog truncates at `including the final...`; the WHIMBREL tail marker is not visible. - **Fallback behavior**: Not exercised: automated single-session runs do not probe recovery paths (no follow-up prompting). Treat as untested rather than absent. +#### `description-length-unit` + +_When a platform enforces the 1024-character description limit, does it count Unicode code points, UTF-16 code units, or UTF-8 bytes?_ + +- **Status**: observed +- **Verdict**: Counts Unicode code points (`counts-code-points`) +- **Confidence**: transcript-direct +- **Evidence**: + - discovery listing names probe-long-description + - body canary loaded on activation + - description head marker in injected content; tail absent + - discovery listing names probe-multibyte-description + - body canary loaded on activation + - description tail marker in injected content + - discovery listing names probe-astral-description + - body canary loaded on activation + - description tail marker in injected content +- **Note**: description fates: ascii:truncated, multibyte:intact, astral:intact +- **Note**: ascii final answer: probe-long-description activated. Canary: **BITTERN-HALITE-2264** **Description visibility:** Only the head marker is visible in the catalog: “SANDERLING-GNEISS-1010”. The WHIMBREL tail marker is absent. The catalog description ends with: “If you can read every sentence of this description including the final...” +- **Note**: multibyte final answer: probe-multibyte-description activated. Canary: **PUFFIN-BASALT-4471** **Description visibility:** Both markers are visible in the catalog: - “The head marker GANNET-PYRITE-1130 sits near the start of this description” - “The tail marker is SHRIKE-TALC-2210” +- **Note**: astral final answer: probe-astral-description activated. Canary: **ORIOLE-GRANITE-5583** **Description visibility:** Both markers are visible in the catalog: “MAGPIE-OBSIDIAN-1240” and “The tail marker is LINNET-MALACHITE-2420”. +- **Note**: the ASCII overrun was enforced while both fixtures under 1024 code points survived intact: the platform counts code points, the reference validator's unit +- **Fallback behavior**: Not exercised: automated single-session runs do not probe recovery paths (no follow-up prompting). Treat as untested rather than absent. + #### `oversize-compatibility-handling` _Is a skill whose compatibility value exceeds the spec's 500-character limit still discovered and loadable?_ @@ -616,4 +641,4 @@ _Is a skill whose compatibility value exceeds the spec's 500-character limit sti --- -Generated by [benchmark-runner](https://github.com/agent-ecosystem/agent-skill-implementation/tree/main/benchmark-runner) from transcript-cited findings; see [the check list](/checks/) (version 0.2) for what each check evaluates. +Generated by [benchmark-runner](https://github.com/agent-ecosystem/agent-skill-implementation/tree/main/benchmark-runner) from transcript-cited findings; see [the check list](/checks/) (version 0.3) for what each check evaluates.