diff --git a/.agents/checks/attribution-lines.sh b/.agents/checks/attribution-lines.sh new file mode 100644 index 00000000..2005cc46 --- /dev/null +++ b/.agents/checks/attribution-lines.sh @@ -0,0 +1,139 @@ +#!/usr/bin/env bash +# The attribution-line classifier. `.agents/checks/attribution.sh` holds it over every +# reachable commit message and `.agents/checks/attribution-metadata.sh` holds it over a +# pull request's title and body, so one line reaches one verdict wherever it is written. +# This file is sourced, never run. +# +# It reads a whole line at a time, never a substring of one, and holds three rules: +# +# 1. A `Co-Authored-By:` trailer, whoever it names. This repository's commits carry +# one author, so the actor cannot change the verdict. +# 2. An attribution trailer: a trailer whose key attributes the work and whose value +# names a generated actor, such as `Generated-by: Claude Code`. +# 3. A generated footer: a line that is nothing but an attribution phrase, such as +# `Generated with [Claude Code](https://claude.com/claude-code)`. +# +# Rule 3 has no key to recognise, so it has to read the whole line. The line must open +# with the attribution phrase, and every word after it must be a word a footer uses. +# That closed vocabulary is what keeps ordinary prose valid: a sentence about Claude, +# Codex, Gemini, Copilot, or about this detector, carries some word no footer carries, +# so it cannot match. The cost is a bespoke attribution sentence that no known tool +# writes, which this check does not claim to catch. +# +# The rule phrases below name the broken rule and nothing else. A caller reports the +# phrase and the place, so untrusted message content never reaches a log. +# +# Callers must export `LC_ALL=C`: the folding and the character classes here are ASCII, +# and a commit message or a pull request body may hold any bytes at all. + +# The words that name a generated actor. Whole words only, because the classifier +# splits a line into words first: `ai` cannot match inside `said`, and `gpt` cannot +# match inside a hash. +attribution_actor_word() { + case $1 in + claude | anthropic | chatgpt | gpt | openai | codex | gemini | copilot | \ + llm | llms | ai | agent | agents | assistant | assistants | \ + bot | bots | model | models) return 0 ;; + esac + return 1 +} + +# The rest of a footer's vocabulary: the connectives a footer puts between its words, +# the vendor and edition words that finish an actor's name, and the words that are left +# of a link to the actor's home page once the punctuation is gone. +# +# The bare `o` is an edition word too. A model number is a separator, so a versioned GPT +# name leaves it behind on its own: `GPT-4o` splits into `gpt` and `o`, and `o3` into +# `o`. Without it those footers would carry a word no footer carries and read as +# sentences. It names no actor by itself, so a line still has to name one to match. +attribution_footer_word() { + case $1 in + a | an | the | and | my | our | its | this | of | in | on | at | to | for | \ + with | by | using | via | from | \ + google | github | microsoft | amazon | mistral | deepseek | \ + code | coding | cli | app | chat | desktop | web | studio | \ + assist | assistance | assisted | help | tool | tools | \ + o | pro | flash | ultra | mini | nano | turbo | preview | latest | \ + sonnet | opus | haiku | \ + https | http | www | com | org | net | io | dev | sh) return 0 ;; + esac + return 1 +} + +# The ASCII letter words of an already folded line, in order. Everything else is a +# separator: a footer's version number, its link punctuation, and whatever bytes a +# message carries are noise, and the rules are about the words around them. +attribution_split() { + local IFS=$' \t\n' text=${1//[^a-z]/ } + # The unquoted expansion is the split itself, and the text holds only letters and + # spaces by now, so there is nothing here for a glob to match. + attribution_words=(${text}) +} + +attribution_names_actor() { + local word + for word in "${attribution_words[@]}"; do + if attribution_actor_word "${word}"; then + return 0 + fi + done + return 1 +} + +# A footer phrase names an actor and says nothing else. One word outside the vocabulary +# is enough to make the line a sentence rather than a footer. +attribution_is_footer_phrase() { + local word named=1 + for word in "${attribution_words[@]}"; do + if attribution_actor_word "${word}"; then + named=0 + continue + fi + if ! attribution_footer_word "${word}"; then + return 1 + fi + done + return "${named}" +} + +# The trailer keys that attribute the work. `Co-Authored-By` is rule 1 and is not here, +# because rule 1 needs no actor. +readonly attribution_trailer_key='(generated-by|generated-with|generated-using|created-by|created-with|authored-by|written-by|made-by|made-with|built-by|built-with|assisted-by|committed-by|signed-off-by|co-authored-with|co-written-by|on-behalf-of|attribution|agent|assistant|model|llm|ai|bot|tool)' + +# The opening of a generated footer: the decoration a footer leads with, such as the +# robot emoji Claude Code writes or a list marker; an optional subject clause, so +# `This pull request was created by ...` reads as one phrase; then the verb and the +# preposition that start the attribution. +readonly attribution_footer_head='^[^[:alpha:]]*((this|these|it|they|the)[[:blank:]]+([a-z]+[[:blank:]]+){0,2}(was|were|is|are)[[:blank:]]+)?(co-)?(generated|created|authored|written|produced|made|built|drafted|committed|assisted|developed|implemented)[[:blank:]]+(with|by|using|via|from)[[:blank:]]+' + +# The one entry point. It sets `attribution_rule` to the phrase that names the broken +# rule and returns 0 when the line is an attribution line, and clears the phrase and +# returns 1 when it is not. +attribution_classify() { + local folded=${1,,} + attribution_rule='' + + if [[ ${folded} =~ ^[[:blank:]]*co-authored-by[[:blank:]]*: ]]; then + attribution_rule='a Co-Authored-By trailer' + return 0 + fi + + # `BASH_REMATCH[0]` ends at the colon, so the rest of the line is the value. + if [[ ${folded} =~ ^[[:blank:]]*${attribution_trailer_key}[[:blank:]]*: ]]; then + attribution_split "${folded:${#BASH_REMATCH[0]}}" + if attribution_names_actor; then + attribution_rule='an attribution trailer that names a generated actor' + return 0 + fi + fi + + if [[ ${folded} =~ ${attribution_footer_head} ]]; then + attribution_split "${folded:${#BASH_REMATCH[0]}}" + if attribution_is_footer_phrase; then + attribution_rule='a generated attribution footer' + return 0 + fi + fi + + return 1 +} diff --git a/.agents/checks/attribution-metadata.sh b/.agents/checks/attribution-metadata.sh new file mode 100755 index 00000000..eaef84e4 --- /dev/null +++ b/.agents/checks/attribution-metadata.sh @@ -0,0 +1,135 @@ +#!/usr/bin/env bash +# The pull request metadata gate. It holds `.agents/checks/attribution-lines.sh`, the +# classifier `.agents/checks/attribution.sh` holds over commit messages, over the title +# and body of the pull request the run is for. +# +# GitHub writes the event as JSON at `GITHUB_EVENT_PATH`. That file carries text a +# stranger wrote, so the text never reaches the shell as anything but a string: `jq` +# parses the file, the fields arrive through a quoted command substitution, they are +# compared against fixed patterns, and no path here expands or evaluates them. `jq`'s +# own diagnostics are dropped for the same reason, because a parse error quotes the +# text that failed to parse. The workflow must not interpolate the title or the body +# into the run step either; it passes only the event name and the event path, which +# GitHub itself controls. +# +# A run that is not for a pull request has no title or body and says so. A run that is +# for one has to be able to read both: a missing, unreadable, malformed, or wrongly +# shaped event fails, because the alternative is a check that passes quietly exactly +# when the metadata it guards cannot be read. +# +# Read-only: it reads one file and writes nothing. +# +# Usage: attribution-metadata.sh +# +# `GITHUB_EVENT_NAME` names the event and `GITHUB_EVENT_PATH` locates it. GitHub CI +# sets both. A local run sets neither, so the check reports that there is nothing to +# read rather than inventing a pull request. +set -euo pipefail + +# ASCII case folding and ASCII character classes, whatever locale the host sets. The +# classifier is sourced after it, because it reads that setting rather than its own. +export LC_ALL=C + +source "$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)/attribution-lines.sh" + +event_name=${GITHUB_EVENT_NAME:-} +event_path=${GITHUB_EVENT_PATH:-} + +refuse() { + printf 'error: pull request event: %s\n' "$1" >&2 + exit 1 +} + +# The event name is GitHub's, but a local caller can set anything, so it is printed +# only when it looks like one of the names GitHub uses. +if [[ ${event_name} =~ ^[a-z_]+$ ]]; then + event_label="event ${event_name}" +else + event_label='an event this check cannot name' +fi + +case ${event_name} in +pull_request | pull_request_target) ;; +'') + printf 'attribution metadata: no GitHub event is set, so there is no pull request title or body to check\n' + exit 0 + ;; +*) + printf 'attribution metadata: %s is not a pull request, so there is no title or body to check\n' \ + "${event_label}" + exit 0 + ;; +esac + +if ! command -v jq >/dev/null 2>&1; then + refuse 'jq is not on PATH, so the event file cannot be parsed' +fi + +if [[ -z ${event_path} ]]; then + refuse 'GITHUB_EVENT_PATH is not set, so the title and body cannot be read' +fi + +if [[ ! -f ${event_path} || ! -r ${event_path} ]]; then + refuse 'the event file is missing or unreadable' +fi + +# One pass over the event that reports a fixed word for each way it can fail to hold a +# title and a body. Only that word crosses back into the shell. A null body is the +# empty description GitHub writes for a pull request that has none, so it is a body; +# any other type means the file is not the payload it claims to be. +shape=$(jq -r ' + if (type != "object") then "event-not-an-object" + elif (has("pull_request") | not) then "no-pull-request" + elif ((.pull_request | type) != "object") then "pull-request-not-an-object" + elif ((.pull_request | has("title")) | not) then "no-title" + elif ((.pull_request.title | type) != "string") then "title-not-a-string" + elif ((.pull_request | has("body")) | not) then "no-body" + elif ((.pull_request.body | type) as $t | ($t != "string" and $t != "null")) then "body-not-a-string" + else "ok" + end' <"${event_path}" 2>/dev/null) || shape='not-json' + +case ${shape} in +ok) ;; +not-json) refuse 'the event file is not valid JSON' ;; +event-not-an-object) refuse 'the event file is not a JSON object' ;; +no-pull-request) refuse 'the event has no pull_request object' ;; +pull-request-not-an-object) refuse 'the event pull_request value is not an object' ;; +no-title) refuse 'the pull request has no title field' ;; +title-not-a-string) refuse 'the pull request title is not a string' ;; +no-body) refuse 'the pull request has no body field' ;; +body-not-a-string) refuse 'the pull request body is neither a string nor null' ;; +*) refuse 'the event file could not be read' ;; +esac + +# The report names the field and the rule, never the line: the title and the body are +# untrusted text, and a run's own metadata already locates them. +check_field() { + local field=$1 filter=$2 text line status=0 + if ! text=$(jq -r "${filter}" <"${event_path}" 2>/dev/null); then + printf 'error: pull request event: the %s could not be read\n' "${field}" >&2 + return 1 + fi + # A title with a newline in it is several lines, and each of them is a line the + # rules are about. A body edited through a browser arrives with carriage returns. + while IFS= read -r line; do + line=${line%$'\r'} + if attribution_classify "${line}"; then + printf 'error: the pull request %s has %s\n' "${field}" "${attribution_rule}" >&2 + status=1 + break + fi + done <<<"${text}" + return "${status}" +} + +failures=0 +check_field title '.pull_request.title' || failures=$((failures + 1)) +check_field body '.pull_request.body // ""' || failures=$((failures + 1)) + +if ((failures > 0)); then + printf 'error: attribution metadata: %d of the 2 pull request fields break a rule\n' \ + "${failures}" >&2 + exit 1 +fi + +printf 'attribution metadata: the pull request title and body carry no attribution line\n' diff --git a/.agents/checks/attribution-test.sh b/.agents/checks/attribution-test.sh index 4f383969..2411a80d 100755 --- a/.agents/checks/attribution-test.sh +++ b/.agents/checks/attribution-test.sh @@ -1,12 +1,17 @@ #!/usr/bin/env bash -# Deterministic proof of `.agents/checks/attribution.sh`. Each case builds a -# throwaway repository under the temporary directory, so no case reads or writes -# the tracked tree, and `.agents/checks/ci-gate.sh` runs this file before it reads -# the live history. +# Deterministic proof of `.agents/checks/attribution.sh` and of +# `.agents/checks/attribution-metadata.sh`, which hold the same line rules over a +# history and over a pull request's title and body. Each case builds a throwaway +# repository or event file under the temporary directory, so no case reads or writes +# the tracked tree or GitHub's real event, and `.agents/checks/ci-gate.sh` runs this +# file before either check reads anything live. # # Every case supplies its own identity, drops host Git configuration, and clears the # variables that name a repository, so a maintainer's own name, a global mailmap, a # signing setting, or an inherited `GIT_DIR` cannot change a result. +# +# The event cases need `jq`, which the flake's development shell supplies for the same +# reason the check needs it. set -euo pipefail check=$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)/attribution.sh @@ -74,9 +79,10 @@ expect_pass() { # The report has to name what it rejected and give the reason. The subject is the # offending commit for a history violation, and the repository itself where the walk -# stops before it reads a commit. +# stops before it reads a commit. The optional last argument is text the report must +# not carry: a message rule names the rule, so the offending line stays out of the log. expect_failure() { - local label=$1 repository=$2 subject=$3 reason=$4 output status=0 + local label=$1 repository=$2 subject=$3 reason=$4 forbidden=${5:-} output status=0 cases=$((cases + 1)) output=$(ATTRIBUTION_TARGET=HEAD bash "${check}" "${repository}" 2>&1) || status=$? if ((status == 0)); then @@ -91,11 +97,17 @@ expect_failure() { record_failure "${label}" "the report does not give the reason: ${output}" return fi + if [[ -n ${forbidden} && ${output} == *"${forbidden}"* ]]; then + record_failure "${label}" "the report echoed the message content: ${output}" + return + fi printf 'ok %s\n' "${label}" } readonly foreign_author='is not the repository owner' readonly foreign_trailer='has a Co-Authored-By trailer' +readonly generated_footer='has a generated attribution footer' +readonly generated_trailer='has an attribution trailer that names a generated actor' readonly shallow_history='is shallow, so the check cannot read the complete history' repository=$(new_repository valid-history) @@ -145,6 +157,73 @@ add_commit "${repository}" "${owner_name}" "${owner_email}" \ expect_failure 'an upper-case indented co-author trailer fails' "${repository}" \ "$(git -C "${repository}" rev-parse HEAD)" "${foreign_trailer}" +# The rule is about the trailer, not about who it credits, so a plainly human co-author +# fails on the same line as an agent one. +repository=$(new_repository human-co-author) +add_commit "${repository}" "${owner_name}" "${owner_email}" \ + $'feat: add a paired change\n\nCo-authored-by: A Human ' +expect_failure 'a co-author trailer naming a human fails' "${repository}" \ + "$(git -C "${repository}" rev-parse HEAD)" "${foreign_trailer}" 'human@example.invalid' + +# The footers the current tools write, each one whole: the decoration it leads with, +# the phrase, and the link it ends on. The Claude Code case carries its robot emoji as +# an escape, so the case holds the real bytes while this file stays ASCII. +repository=$(new_repository claude-footer) +add_commit "${repository}" "${owner_name}" "${owner_email}" \ + $'feat: add a generated change\n\n\xf0\x9f\xa4\x96 Generated with [Claude Code](https://claude.com/claude-code)' +expect_failure 'the Claude Code footer fails' "${repository}" \ + "$(git -C "${repository}" rev-parse HEAD)" "${generated_footer}" 'claude.com' + +repository=$(new_repository codex-footer) +add_commit "${repository}" "${owner_name}" "${owner_email}" \ + $'feat: add a delegated change\n\nGenerated by Codex (https://chatgpt.com/codex)' +expect_failure 'a GPT or Codex footer fails' "${repository}" \ + "$(git -C "${repository}" rev-parse HEAD)" "${generated_footer}" 'chatgpt.com' + +# The same family under a versioned model name. The split drops the model number, so +# these leave a bare `o` where the Codex footer above leaves a whole word, and a +# vocabulary without it would read them as sentences and accept the history. +repository=$(new_repository gpt-model-footer) +add_commit "${repository}" "${owner_name}" "${owner_email}" \ + $'feat: add a versioned change\n\nGenerated by GPT-4o (https://chatgpt.com)' +expect_failure 'a versioned GPT model footer fails' "${repository}" \ + "$(git -C "${repository}" rev-parse HEAD)" "${generated_footer}" 'chatgpt.com' + +repository=$(new_repository gpt-edition-footer) +add_commit "${repository}" "${owner_name}" "${owner_email}" \ + $'feat: add a smaller change\n\n\xf0\x9f\xa4\x96 Generated with GPT-4o-mini' +expect_failure 'a GPT edition footer fails' "${repository}" \ + "$(git -C "${repository}" rev-parse HEAD)" "${generated_footer}" 'GPT' + +repository=$(new_repository gemini-footer) +add_commit "${repository}" "${owner_name}" "${owner_email}" \ + $'feat: add an assisted change\n\nGenerated with [Gemini CLI](https://github.com/google-gemini/gemini-cli)' +expect_failure 'a Gemini footer fails' "${repository}" \ + "$(git -C "${repository}" rev-parse HEAD)" "${generated_footer}" 'gemini-cli' + +repository=$(new_repository copilot-footer) +add_commit "${repository}" "${owner_name}" "${owner_email}" \ + $'feat: add a suggested change\n\nThis pull request was created by GitHub Copilot.' +expect_failure 'a Copilot footer fails' "${repository}" \ + "$(git -C "${repository}" rev-parse HEAD)" "${generated_footer}" 'Copilot' + +# A trailer key that attributes the work, with an actor in its value, is the other +# structural form, and it reports its own rule rather than the footer one. +repository=$(new_repository attribution-trailer) +add_commit "${repository}" "${owner_name}" "${owner_email}" \ + $'feat: add a credited change\n\nGenerated-By: Claude Code' +expect_failure 'an attribution trailer naming an actor fails' "${repository}" \ + "$(git -C "${repository}" rev-parse HEAD)" "${generated_trailer}" 'Claude Code' + +# The other half of the rule. These lines name the same tools, and one of them opens on +# the same verb a footer opens on, but each is a sentence rather than an attribution, so +# the history is valid and the check has to say so. A trailer key with no actor in its +# value is a sentence too. +repository=$(new_repository safe-mentions) +add_commit "${repository}" "${owner_name}" "${owner_email}" \ + $'docs: describe the attribution rules\n\nThe gate rejects a footer generated with Claude Code and a trailer that names Codex, Gemini or Copilot.\n\nOrdinary prose about an AI agent, an LLM, or the Copilot commit footer stays valid, because a sentence carries a word no footer carries.\n\nGenerated footers name their tool, which is what the classifier keys on.\n\nGenerated by GPT-4o and by o3, those footers now fail, and this sentence about them does not.\n\nCreated by hand, this change carries no generated footer.\n\nAssisted-by: a human reviewer' +expect_pass 'safe technical mentions pass' "${repository}" + # The rewritten history put the removed identities and trailers below the tip, so # a check that read only the tip commit would report a clean repository. repository=$(new_repository ancestor-violation) @@ -182,6 +261,182 @@ clone=${scratch}/shallow-clone git clone --quiet --depth 1 "file://${source_repository}" "${clone}" expect_failure 'a shallow clone fails' "${clone}" "${clone}" "${shallow_history}" +# The pull request half of the gate. Each case writes its own event file under the same +# temporary directory and names it explicitly, so no case reads GitHub's real event or +# another case's, and the two variables a run supplies are set per case rather than +# inherited: a real pull request run sets both, and a maintainer's shell sets neither. +metadata_check=$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)/attribution-metadata.sh + +# `jq` writes the event, so a title or a body holding quotes, backslashes, or shell +# punctuation is escaped by the same parser the check reads it back with, and the case +# proves the check rather than the fixture. +new_event() { + local name=$1 title=$2 body=$3 + local path=${scratch}/${name}.json + jq -n --arg title "${title}" --arg body "${body}" \ + '{pull_request: {title: $title, body: $body}}' >"${path}" + printf '%s' "${path}" +} + +# The cases that are about a broken payload write it byte for byte instead. +raw_event() { + local name=$1 content=$2 + local path=${scratch}/${name}.json + printf '%s' "${content}" >"${path}" + printf '%s' "${path}" +} + +expect_metadata_pass() { + local label=$1 name=$2 path=$3 output status=0 + cases=$((cases + 1)) + output=$(GITHUB_EVENT_NAME="${name}" GITHUB_EVENT_PATH="${path}" \ + bash "${metadata_check}" 2>&1) || status=$? + if ((status != 0)); then + record_failure "${label}" "the check rejected valid metadata: ${output}" + return + fi + printf 'ok %s\n' "${label}" +} + +expect_metadata_failure() { + local label=$1 name=$2 path=$3 subject=$4 reason=$5 forbidden=${6:-} output status=0 + cases=$((cases + 1)) + output=$(GITHUB_EVENT_NAME="${name}" GITHUB_EVENT_PATH="${path}" \ + bash "${metadata_check}" 2>&1) || status=$? + if ((status == 0)); then + record_failure "${label}" 'the check accepted metadata it must reject' + return + fi + if [[ ${output} != *"${subject}"* ]]; then + record_failure "${label}" "the report does not name ${subject}: ${output}" + return + fi + if [[ ${output} != *"${reason}"* ]]; then + record_failure "${label}" "the report does not give the reason: ${output}" + return + fi + if [[ -n ${forbidden} && ${output} == *"${forbidden}"* ]]; then + record_failure "${label}" "the report echoed the event content: ${output}" + return + fi + printf 'ok %s\n' "${label}" +} + +event=$(new_event clean-pull-request 'fix: reject generated attribution messages' \ + $'The body explains the change.\n\nIt mentions Claude Code and Copilot in a sentence, which is not a footer.') +expect_metadata_pass 'a clean pull request passes' pull_request "${event}" + +# The same classifier over the other surface: the title and the body break the same +# rules a commit message breaks, and the report names the field rather than the line. +event=$(new_event generated-title \ + "$(printf '\xf0\x9f\xa4\x96 Generated with [Claude Code](https://claude.com/claude-code)')" \ + 'A body that says nothing about its author.') +expect_metadata_failure 'a generated title fails' pull_request "${event}" \ + 'pull request title' "${generated_footer}" 'claude.com' + +event=$(new_event generated-body 'fix: correct the rule' \ + $'The change is small.\n\nCo-Authored-By: Claude ') +expect_metadata_failure 'a co-author trailer in the body fails' pull_request "${event}" \ + 'pull request body' "${foreign_trailer}" 'noreply@anthropic.com' + +event=$(new_event codex-body 'fix: correct the other rule' \ + $'The change is small.\n\nGenerated by Codex (https://chatgpt.com/codex)') +expect_metadata_failure 'a generated footer in the body fails' pull_request "${event}" \ + 'pull request body' "${generated_footer}" 'chatgpt.com' + +# The versioned model name reaches this surface too, and the classifier is the same one, +# so the vocabulary that closes the phrase has to hold for a body and a title as well. +event=$(new_event gpt-model-body 'fix: correct the versioned rule' \ + $'The change is small.\n\nGenerated with GPT-4o (https://chatgpt.com)') +expect_metadata_failure 'a versioned GPT model footer in the body fails' pull_request \ + "${event}" 'pull request body' "${generated_footer}" 'chatgpt.com' + +event=$(new_event gpt-model-title 'This pull request was created by GPT-4o' \ + 'A body that says nothing about its author.') +expect_metadata_failure 'a versioned GPT model footer in the title fails' pull_request \ + "${event}" 'pull request title' "${generated_footer}" 'GPT' + +# The other half of the rule on this surface: a body that names the same model in a +# sentence is prose, so it stays valid. +event=$(new_event gpt-model-mention 'fix: describe the versioned rule' \ + $'The change is small.\n\nGenerated by GPT-4o and by o3, those footers now fail, and this sentence about them does not.') +expect_metadata_pass 'a safe mention of a GPT model passes' pull_request "${event}" + +# A title split across lines would otherwise hide an attribution line behind the first +# one, so every line of both fields is classified. +event=$(new_event multi-line-title \ + $'fix: correct the rule\nGenerated with Claude Code' 'A body.') +expect_metadata_failure 'an attribution line below a title fails' pull_request "${event}" \ + 'pull request title' "${generated_footer}" 'Claude' + +# Untrusted text that looks like a command. It has to pass the rules, because it +# attributes nothing, and it has to stay text: the sentinel below exists only if some +# part of the check let the event's own words run. +sentinel=${scratch}/executed +event=$(new_event command-like-text \ + "fix: quote \$(touch ${sentinel}) and \`touch ${sentinel}\` safely" \ + "$(printf 'The body carries ; touch %s and $(touch %s) and `touch %s`.\nIt still names no author.' \ + "${sentinel}" "${sentinel}" "${sentinel}")") +expect_metadata_pass 'command-like untrusted text passes' pull_request "${event}" +cases=$((cases + 1)) +if [[ -e ${sentinel} ]]; then + record_failure 'command-like untrusted text stays inert' 'the check ran text from the event' +else + printf 'ok %s\n' 'command-like untrusted text stays inert' +fi + +# A push carries no pull request, so the metadata rules have nothing to read. The event +# here would fail every one of them, which is what makes the skip visible. +event=$(new_event pushed-event 'Generated with [Claude Code](https://claude.com/claude-code)' \ + 'Co-Authored-By: Claude ') +expect_metadata_pass 'a push skips the pull request metadata' push "${event}" +expect_metadata_pass 'a run with no event skips the pull request metadata' '' '' + +# Once the event claims to be a pull request, every way of not producing a title and a +# body is a failure. A check that skipped here would pass exactly when the metadata it +# guards became unreadable. +expect_metadata_failure 'a pull request event with no event path fails' pull_request '' \ + 'pull request event' 'GITHUB_EVENT_PATH is not set' +expect_metadata_failure 'a missing event file fails' pull_request "${scratch}/absent.json" \ + 'pull request event' 'the event file is missing or unreadable' + +event=$(raw_event malformed-event '{"pull_request": {"title": "fix: correct the rule",') +expect_metadata_failure 'a malformed event file fails' pull_request "${event}" \ + 'pull request event' 'the event file is not valid JSON' 'fix: correct the rule' + +event=$(raw_event array-event '["pull_request"]') +expect_metadata_failure 'an event that is not an object fails' pull_request "${event}" \ + 'pull request event' 'the event file is not a JSON object' + +event=$(raw_event no-pull-request-event '{"repository": {"name": "agent-scaffold"}}') +expect_metadata_failure 'an event with no pull request fails' pull_request "${event}" \ + 'pull request event' 'the event has no pull_request object' + +event=$(raw_event pull-request-not-an-object '{"pull_request": "fix: correct the rule"}') +expect_metadata_failure 'a pull request that is not an object fails' pull_request "${event}" \ + 'pull request event' 'the event pull_request value is not an object' + +event=$(raw_event no-title '{"pull_request": {"body": "A body."}}') +expect_metadata_failure 'a pull request with no title fails' pull_request "${event}" \ + 'pull request event' 'the pull request has no title field' + +event=$(raw_event title-not-a-string '{"pull_request": {"title": 12, "body": "A body."}}') +expect_metadata_failure 'a title that is not a string fails' pull_request "${event}" \ + 'pull request event' 'the pull request title is not a string' + +event=$(raw_event no-body '{"pull_request": {"title": "fix: correct the rule"}}') +expect_metadata_failure 'a pull request with no body fails' pull_request "${event}" \ + 'pull request event' 'the pull request has no body field' + +event=$(raw_event body-not-a-string '{"pull_request": {"title": "fix: correct the rule", "body": []}}') +expect_metadata_failure 'a body that is neither a string nor null fails' pull_request "${event}" \ + 'pull request event' 'the pull request body is neither a string nor null' + +# GitHub writes a null body for a pull request that has no description, so a null body +# is an empty one rather than a broken payload. +event=$(raw_event null-body '{"pull_request": {"title": "fix: correct the rule", "body": null}}') +expect_metadata_pass 'a null body is an empty description' pull_request "${event}" + if ((failures > 0)); then printf 'error: attribution tests: %d of %d cases failed\n' "${failures}" "${cases}" >&2 exit 1 diff --git a/.agents/checks/attribution.sh b/.agents/checks/attribution.sh index c26d2e54..dec7374a 100755 --- a/.agents/checks/attribution.sh +++ b/.agents/checks/attribution.sh @@ -1,7 +1,12 @@ #!/usr/bin/env bash # The attribution gate. It reads every commit reachable from its target and holds # two rules the human selected: each commit carries the repository owner's raw -# author identity, and no commit message carries a co-author trailer. +# author identity, and no commit message carries an attribution line. +# +# The second rule lives in `.agents/checks/attribution-lines.sh`, which +# `.agents/checks/attribution-metadata.sh` holds over a pull request's title and +# body, so a line a commit message may not carry is a line the pull request may not +# carry either. # # `.agents/checks/ci-gate.sh` runs `.agents/checks/attribution-test.sh` first, so # scratch repositories prove both rules before this file reads the live history. @@ -26,9 +31,12 @@ readonly owner_email='18732253+nothingnesses@users.noreply.github.com' repository=${1:-.} target=${ATTRIBUTION_TARGET:-HEAD} -# ASCII case folding and ASCII character classes, whatever locale the host sets. +# ASCII case folding and ASCII character classes, whatever locale the host sets. The +# classifier is sourced after it, because it reads that setting rather than its own. export LC_ALL=C +source "$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)/attribution-lines.sh" + # A target the repository does not hold, such as a head SHA a partial fetch left # out, would otherwise stop the walk before it read anything. if ! git -C "${repository}" rev-parse --verify --quiet "${target}^{commit}" >/dev/null; then @@ -45,7 +53,7 @@ if [[ $(git -C "${repository}" rev-parse --is-shallow-repository) == true ]]; th fi scan() { - local failures=0 record commit name email line folded + local failures=0 record commit name email line local -a lines while IFS= read -r -d '' record; do # git separates entries with a newline, so every record after the first @@ -60,13 +68,13 @@ scan() { "${commit}" "${name}" "${email}" "${owner_name}" "${owner_email}" >&2 failures=$((failures + 1)) fi - # The report names the rule and the commit, never the trailer line: the line - # holds a person's name and email, and the hash already locates it. + # The report names the rule and the commit, never the offending line: the line + # is untrusted text that may hold a person's name and email, or anything else + # an author typed, and the hash already locates it. for line in "${lines[@]:3}"; do - folded=${line,,} - if [[ ${folded} =~ ^[[:blank:]]*co-authored-by[[:blank:]]*: ]]; then - printf 'error: %s: the commit message has a Co-Authored-By trailer\n' \ - "${commit}" >&2 + if attribution_classify "${line}"; then + printf 'error: %s: the commit message has %s\n' \ + "${commit}" "${attribution_rule}" >&2 failures=$((failures + 1)) break fi @@ -90,5 +98,5 @@ if ! git -C "${repository}" log --format='%H%n%an%n%ae%n%B%x00' "${target}" | sc exit 1 fi -printf 'attribution: %s commits reachable from %s carry the owner identity and no co-author trailer\n' \ +printf 'attribution: %s commits reachable from %s carry the owner identity and no attribution line\n' \ "$(git -C "${repository}" rev-list --count "${target}")" "${target}" diff --git a/.agents/checks/ci-gate.sh b/.agents/checks/ci-gate.sh index 1bf9d8d0..3766c814 100755 --- a/.agents/checks/ci-gate.sh +++ b/.agents/checks/ci-gate.sh @@ -71,13 +71,20 @@ if unpinned=$(grep -rEn '^[[:space:]]*-?[[:space:]]*uses:' .github/workflows | fi step 'attribution tests' -# The proof runs before the live history, so a check that had stopped rejecting a -# foreign author or a co-author trailer fails here rather than passing silently. +# The proof runs before the live history and the live event, so a check that had +# stopped rejecting a foreign author, a co-author trailer, or a generated footer fails +# here rather than passing silently. bash .agents/checks/attribution-test.sh step 'attribution' bash .agents/checks/attribution.sh +step 'attribution metadata' +# The pull request's own title and body, which no commit carries. A local run and a +# push have no pull request to read, and the check reports that rather than inventing +# a pass. +bash .agents/checks/attribution-metadata.sh + step 'tracked tree' tracked_after=$(git status --porcelain --untracked-files=no) if [[ ${tracked_after} != "${tracked_before}" ]]; then diff --git a/.agents/work.toml b/.agents/work.toml index b518f460..0a099f0b 100644 --- a/.agents/work.toml +++ b/.agents/work.toml @@ -1,24 +1,26 @@ version = 1 [[step]] -id = "ship-standalone-review-prompt" +id = "enforce-attribution-messages" status = "complete" blocked_by = [] user_problem = """ -A human who wants a review of a whole tree or of one diff has to improvise the request. A whole-tree review can silently collapse into an empty diff review. +The complete-history gate rejects foreign authors and Co-Authored-By lines, but other generated attribution footers can pass. Pull-request titles and bodies rely only on guidance. """ change = """ -Ship one compact standalone review prompt as a built-in pack reference asset, with the discoverability and tests it needs. +Use one structural attribution policy for commit messages and pull-request metadata. Reject attribution trailers and known generated footer forms without banning ordinary discussion of agent tools. """ acceptance = [ - "A fresh default scaffold writes `.agents/user-prompts/review.md` and no other new asset.", - "The prompt makes the current-tree and diff targets exclusive and unambiguous.", - "It requires criteria, resolved refs, a clean-or-dirty status statement, read-only work and a direct response.", - "It requires severity and reproducible evidence and allows a concise clean result.", - "It forbids edits and every persisted review-state family the reset removed.", - "Shipped guidance says when to use the kickoff prompt and when to use this one.", - "Tests pin the asset, its limits and that contract.", + "The exact raw author allowlist, complete-history scan, shallow-history refusal and every Co-Authored-By rejection remain unchanged.", + "One deterministic classifier rejects structural attribution trailers and known generated footer forms for Claude, GPT or Codex, Gemini and Copilot.", + "The same classifier checks reachable commit messages and pull-request titles and bodies.", + "Pull-request metadata is parsed from the GitHub event file without shell evaluation, and a missing or malformed event fails when the check expects a pull request.", + "A pull-request title or body edit triggers the required quality workflow again.", + "Normal technical discussion of model names and attribution detection remains valid when it is not itself an attribution line.", + "Failures name the commit or pull-request field and the broken rule without echoing untrusted message content.", + "Scratch cases prove Claude, GPT or Codex, Gemini and Copilot forms, safe mentions, event parsing, push behaviour and the existing history rules.", + "The shared just ci gate, protected quality job, existing reset checks and complete-history attribution check pass.", ] why_next = """ -The comparison already chose this surface. Shipping it closes the gap without a command, provider or new state. +The repository history is correct, but deterministic prevention must cover the message surfaces that current guidance alone cannot enforce. """ diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 2ac117ff..cac7cf1d 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -2,6 +2,11 @@ name: ci on: pull_request: + # The first three are the default set, which naming any type replaces. `edited` + # is the addition: it fires on a title or body change, and those are what + # `.agents/checks/attribution-metadata.sh` reads, so an edit is checked again + # rather than riding on the run that the opening commit passed. + types: [opened, synchronize, reopened, edited] push: branches: [main] @@ -43,4 +48,10 @@ jobs: # push. The merge commit has that head as a parent, so the fetched # history already contains it. ATTRIBUTION_TARGET: ${{ github.event.pull_request.head.sha || github.sha }} + # The metadata check reads the pull request's title and body out of the + # event file itself. Only the event's name and location are passed in, both + # of them GitHub's own values; interpolating the title or the body here + # would run a stranger's text as part of this script. + GITHUB_EVENT_NAME: ${{ github.event_name }} + GITHUB_EVENT_PATH: ${{ github.event_path }} run: nix develop --command just ci diff --git a/flake.nix b/flake.nix index 7b08cbd9..e37e6260 100644 --- a/flake.nix +++ b/flake.nix @@ -94,6 +94,9 @@ # CI gate (`.agents/checks/ci-gate.sh`) pkgs.actionlint + # `.agents/checks/attribution-metadata.sh` parses the GitHub event JSON + # with it, rather than letting the shell near a stranger's text. + pkgs.jq ]; env = {