diff --git a/.github/workflows/validate.yml b/.github/workflows/validate.yml new file mode 100644 index 0000000..f24a86e --- /dev/null +++ b/.github/workflows/validate.yml @@ -0,0 +1,17 @@ +name: Validate rules + +on: + push: + branches: [main] + pull_request: + +jobs: + validate: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: "3.12" + - run: pip install pyyaml + - run: python3 scripts/validate_rules.py diff --git a/LICENSE b/LICENSE index 28570d8..3287f4b 100644 --- a/LICENSE +++ b/LICENSE @@ -1,6 +1,6 @@ MIT License -Copyright (c) 2025 CoderBuds +Copyright (c) 2025 Coderbuds Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal diff --git a/README.md b/README.md index f0f2535..fdfc1cb 100644 --- a/README.md +++ b/README.md @@ -2,7 +2,7 @@ > Portable, maintainable AI code detection rules for identifying AI-generated pull requests and commits. Detect Claude Code, GitHub Copilot, Cursor, and other AI coding assistants. -[![Accuracy](https://img.shields.io/badge/accuracy-98%25-brightgreen)](https://github.com/coderbuds/ai-detector) +[![Validate rules](https://github.com/coderbuds/ai-detector/actions/workflows/validate.yml/badge.svg)](https://github.com/coderbuds/ai-detector/actions/workflows/validate.yml) [![License](https://img.shields.io/badge/license-MIT-blue.svg)](LICENSE.md) [![Stars](https://img.shields.io/github/stars/coderbuds/ai-detector?style=social)](https://github.com/coderbuds/ai-detector/stargazers) @@ -12,7 +12,7 @@ Over 46% of code on GitHub is now AI-assisted (GitHub Octoverse 2024). Engineering teams need transparency into which pull requests use AI coding tools. -We built these detection rules for [CoderBuds](https://coderbuds.com) and decided to **open-source them** because: +We built these detection rules for [Coderbuds](https://coderbuds.com) and decided to **open-source them** because: - āœ… **Transparency builds trust** - Developers deserve to know how AI detection works - āœ… **Community contributions** - Help us keep rules updated as AI tools evolve @@ -58,17 +58,18 @@ explicit_markers: ## šŸ¤– Supported AI Tools -| Tool | Provider | Detection Method | Accuracy | -|------|----------|------------------|----------| -| **Claude Code** | Anthropic | Footer, co-author, bot email | 100% | -| **GitHub Copilot** | Microsoft | Bot commits, co-author | 100% | -| **Cursor** | Anysphere | Footer, link, markers | 96% | -| **Devin** | Cognition AI | Bot author, footer | 100% | -| **WindSurf** | Codeium | Footer, attribution | 100% | -| **OpenAI Codex** | OpenAI | Branch patterns, markers | 100% | -| **Aider** | Open Source | Commit patterns | 90% | -| **v0.dev** | Vercel | Markers, comments | 95% | -| **Replit AI** | Replit | Bot author, markers | 100% | +| Tool | Provider | What it leaves behind | +|------|----------|------------------------| +| **Claude Code** | Anthropic | Description footer, `Co-Authored-By: Claude` trailer, `claude/` branches | +| **GitHub Copilot** | Microsoft | Bot commit authors, co-author trailer, footer | +| **Cursor** | Anysphere | Footer and links, `CURSOR_SUMMARY` comment, `cursor/` branches | +| **Jules** | Google | "PR created automatically by Jules" signature, task link, `jules-` branches | +| **Devin** | Cognition AI | Bot commit authors, footer, links | +| **OpenAI Codex** | OpenAI | `codex/` branches, `[codex]` titles, Codex task links, `codex` label | +| **WindSurf** | Codeium | Footer link, HTML comment, `windsurf/` branches | +| **Aider** | Open Source | `aider: ` commit prefix, aider.chat links, `aider/` branches | +| **v0.dev** | Vercel | Footer and HTML comments | +| **Replit AI** | Replit | Bot commit author, footer link, HTML comment, `replit-ai` label | **Missing a tool?** [Submit a PR](https://github.com/coderbuds/ai-detector/pulls) or [open an issue](https://github.com/coderbuds/ai-detector/issues). @@ -114,7 +115,7 @@ function detectAI(string $prDescription, array $commits): ?array // Check bot authors in commits foreach ($rule['explicit_markers']['bot_authors'] ?? [] as $bot) { foreach ($commits as $commit) { - if (str_contains($commit['author']['email'], $bot['pattern'])) { + if (isset($bot['email']) && $commit['author']['email'] === $bot['email']) { return [ 'tool' => $rule['tool']['name'], 'confidence' => $bot['confidence'], @@ -161,7 +162,7 @@ def detect_ai(pr_description, commits): # Check bot authors for bot in rule.get('explicit_markers', {}).get('bot_authors', []): for commit in commits: - if bot['pattern'] in commit['author']['email']: + if bot.get('email') and commit['author']['email'] == bot['email']: return { 'tool': rule['tool']['name'], 'confidence': bot['confidence'], @@ -206,7 +207,7 @@ function detectAI(prDescription, commits) { // Check bot authors for (const bot of rule.explicit_markers?.bot_authors || []) { for (const commit of commits) { - if (commit.author.email.includes(bot.pattern)) { + if (bot.email && commit.author.email === bot.email) { return { tool: rule.tool.name, confidence: bot.confidence, @@ -234,7 +235,8 @@ The YAML rules check for these marker types: | `bot_authors` | Bot emails and usernames | `github-copilot[bot]`, `noreply@anthropic.com` | | `html_comments` | Special HTML comments | `` | | `labels` | PR labels | `codex`, `ai-generated` | -| `branch_patterns` | Branch naming conventions | `codex/feature`, `cursor-refactor` | +| `branch_patterns` | Branch naming conventions | `codex/feature`, `claude/fix-login` | +| `text_patterns` | Title or description text (`location: title` or `description`) | `[codex] Fix publishing` | --- @@ -248,7 +250,7 @@ The YAML rules check for these marker types: - Branch names following AI tool patterns - PR labels indicating AI usage -**Accuracy: 98-100%** - When explicit markers exist, detection is certain. +A marker at confidence 100 is the tool signing its own work. Lower confidences ā€” branch names, title prefixes ā€” are strong conventions, not signatures: a person can name a branch `claude/ā€¦` too. ### āŒ **Doesn't Detect (Without Additional Analysis)** @@ -256,8 +258,11 @@ The YAML rules check for these marker types: - ChatGPT code copied manually - AI-assisted refactoring without attribution - Code quality or "AI-like" patterns +- Agents that sign nothing. **Codex CLI run locally leaves no marker at all**, and neither does any agent whose footer a person deletes -**For behavioral analysis** (analyzing code patterns), see [CoderBuds Platform](https://coderbuds.com). +**No match does not mean a person wrote it.** On Coderbuds' own production data, pull requests with no marker were routinely agent-written ā€” including every one Codex CLI produced. Treat "no marker found" as unknown, never as human. The only reliable source for those is the agent saying so itself; Coderbuds asks coding agents to report their authorship over MCP for exactly this reason. + +**For behavioral analysis** (analyzing code patterns), see [Coderbuds Platform](https://coderbuds.com). --- @@ -270,7 +275,7 @@ This package is **100% free and open source** (MIT License). Use it for: - āœ… Local development workflows - āœ… Custom integrations -**[CoderBuds Platform](https://coderbuds.com)** (paid service) adds: +**[Coderbuds Platform](https://coderbuds.com)** (paid service) adds: - šŸ“Š Team-level analytics over time - šŸ“ˆ AI adoption trends and insights @@ -279,7 +284,7 @@ This package is **100% free and open source** (MIT License). Use it for: - šŸ¢ Enterprise features (SSO, audit logs) - šŸ“ Custom reporting and exports -**Analogy:** This package is like Sentry's SDK (free). CoderBuds is like Sentry's hosted platform (paid). +**Analogy:** This package is like Sentry's SDK (free). Coderbuds is like Sentry's hosted platform (paid). --- @@ -313,18 +318,21 @@ explicit_markers: description: "Tool footer in PR description" bot_authors: - - pattern: 'your-tool[bot]' - location: commit_author + - username: 'your-tool[bot]' confidence: 100 description: "Your Tool bot author" ``` -4. **Test against real PRs** - Verify accuracy -5. **Submit a PR** with test results +`bot_authors` entries match on `username`, `email`, `email_pattern` or `name_pattern`; `labels` entries on `name`. Any other key is silently ignored by the matcher, so the validator rejects it. + +4. **Add fixture cases** to `fixtures/cases.yml` ā€” a pull request your rule must catch, and one it must not +5. **Run the validator:** `pip install pyyaml && python3 scripts/validate_rules.py` +6. **Submit a PR** ā€” CI runs the same check ### Contribution Guidelines - Include at least 3 example PRs showing the pattern +- Match signatures, not mentions: a rule that fires on a tool's *name* also fires on every changelog and dependency bump that mentions it - Document confidence levels (100 = definitive, 80+ = high, 60+ = medium) - Add test cases if possible - Update this README's tool table @@ -334,7 +342,7 @@ explicit_markers: ## šŸ“š Documentation - **[Blog Post: Why We Open-Sourced This](https://coderbuds.com/blog/open-source-ai-code-detection-yaml-rules)** -- **[CoderBuds Platform](https://coderbuds.com)** - Team analytics +- **[Coderbuds Platform](https://coderbuds.com)** - Team analytics - **[GitHub Discussions](https://github.com/coderbuds/ai-detector/discussions)** - Ask questions - **[Issues](https://github.com/coderbuds/ai-detector/issues)** - Report bugs @@ -350,7 +358,7 @@ See [LICENSE.md](LICENSE.md) for details. ## šŸ™ Credits -Created by **[CoderBuds](https://coderbuds.com)** - AI adoption analytics for engineering teams. +Created by **[Coderbuds](https://coderbuds.com)** - AI adoption analytics for engineering teams. **Built with transparency in mind.** Developers deserve to know how AI detection works. @@ -362,7 +370,7 @@ Created by **[CoderBuds](https://coderbuds.com)** - AI adoption analytics for en - **[Try the Live Detector](https://coderbuds.com/blog/open-source-ai-code-detection-yaml-rules#try-it)** - Paste any GitHub PR URL - **[Full Blog Post](https://coderbuds.com/blog/open-source-ai-code-detection-yaml-rules)** - Why we open-sourced this -- **[CoderBuds Platform](https://coderbuds.com)** - Team AI adoption analytics +- **[Coderbuds Platform](https://coderbuds.com)** - Team AI adoption analytics - **[GitHub](https://github.com/coderbuds/ai-detector)** - Source code - **[Issues](https://github.com/coderbuds/ai-detector/issues)** - Bug reports - **[Discussions](https://github.com/coderbuds/ai-detector/discussions)** - Community @@ -371,4 +379,4 @@ Created by **[CoderBuds](https://coderbuds.com)** - AI adoption analytics for en **Have questions?** Open a [GitHub Discussion](https://github.com/coderbuds/ai-detector/discussions) or [tweet at us](https://twitter.com/coderbuds). -**Want team insights?** [Start tracking with CoderBuds](https://coderbuds.com/register) (30-day free trial, no credit card required). +**Want team insights?** [Start tracking with Coderbuds](https://coderbuds.com/register) (30-day free trial, no credit card required). diff --git a/fixtures/cases.yml b/fixtures/cases.yml new file mode 100644 index 0000000..13ee8ac --- /dev/null +++ b/fixtures/cases.yml @@ -0,0 +1,124 @@ +# Pull request shapes seen in production, and who they must be attributed to. +# `expect: null` means no tool ā€” the rules must stay quiet. Titles, branches +# and descriptions are real shapes with identifying details removed. + +- name: Dependabot bump of @openai/codex is not a Codex pull request + expect: null + pr: + title: "build(deps): bump @openai/codex from 0.117.0 to 0.118.0" + branch: dependabot/npm_and_yarn/openai/codex-0.118.0 + description: "Bumps [@openai/codex](https://github.com/openai/codex/tree/HEAD/codex-cli) from 0.117.0 to 0.118.0." + +- name: Mentioning a .codex/ config path is not a Codex pull request + expect: null + pr: + title: Configure root MCP servers + branch: feature/mcp-config + description: "Adds `.codex/config.toml` alongside `.cursor/mcp.json` so every agent loads the same servers." + +- name: Codex cloud branch + expect: openai-codex + pr: + title: Validate generated PR actor + branch: codex/validate-generated-pr-actor + description: "## Summary\n- validate the actor" + +- name: Codex cloud title prefix with the branch unknown + expect: openai-codex + pr: + title: "[codex] Fix trusted publishing npm upgrade step" + description: "" + +- name: Codex cloud task link + expect: openai-codex + pr: + title: "fix(client): use getJWTToken() for dashboard requests" + description: "Uses the SDK helper.\n\n------\n[Codex Task](https://chatgpt.com/codex/cloud/tasks/task_e_0123456789abcdef0123456789abcdef)" + +- name: A pull request about Codex setup is not a Codex pull request + expect: null + pr: + title: "docs: add Codex MCP setup" + branch: docs/codex-mcp + description: "The setup page did not include a Codex-specific path." + +- name: Jules signature + expect: jules + pr: + title: "Palette: Add Escape key dismissal to modals" + branch: palette/escape-dismissal-1498727143055922862 + description: "Adds keyboard dismissal.\n\n---\n*PR created automatically by Jules for task [1498727143055922862](https://jules.google.com/task/1498727143055922862) started by @someone*" + +- name: Jules task branch alone + expect: jules + pr: + title: "Sentinel: Path parameter UUID validation" + branch: jules-4486629921808545700-9ef0dd4e + description: "Validates UUID path parameters." + +- name: Claude Code web branch + expect: claude-code + pr: + title: "fix(PLT-1400): set dedicated ip_pool on sends" + branch: claude/sparkpost-ip-pool-update-ab4c44 + description: "Sets the pool." + +- name: Claude co-author trailer in lower case + expect: claude-code + pr: + title: Add bulk delete script + branch: bulk-delete + description: "" + commits: + - message: "Add bulk delete script\n\nCo-authored-by: Claude " + +- name: Claude Code footer + expect: claude-code + pr: + title: Show agent action stages + branch: agent-stages + description: "## Summary\n...\n\nšŸ¤– Generated with [Claude Code](https://claude.com/claude-code)" + +- name: Cursor background agent branch + expect: cursor + pr: + title: "fix(CLP-2112): prevent crash on loops with no end date" + branch: cursor/fix-loop-no-end-date-crash-1edb + description: "Guards the loop." + +- name: The word raider is not Aider + expect: null + pr: + title: "Fix raider: boss spawn timing" + branch: fix/raider-spawn + description: "The raider: boss spawned twice. Braided timers fixed." + +- name: Aider commit prefix + expect: aider + pr: + title: Add tests for parser + branch: parser-tests + description: "" + commits: + - message: "aider: Added tests for the parser" + +- name: Aider branch + expect: aider + pr: + title: Add parser tests + branch: aider/parser-tests + description: "" + +- name: A changelog naming WindSurf is not a WindSurf pull request + expect: null + pr: + title: "build(deps): bump laravel/boost from 2.1 to 2.2" + branch: dependabot/composer/laravel/boost-2.2 + description: "Add WindSurf and Kiro guideline files by @contributor" + +- name: A plain human pull request + expect: null + pr: + title: Fix pagination on the repositories page + branch: fix/repositories-pagination + description: "## Summary\n- off-by-one in the page count\n\n## Testing\n- added a feature test" diff --git a/rules/aider.yml b/rules/aider.yml index 1207f34..d7163ab 100644 --- a/rules/aider.yml +++ b/rules/aider.yml @@ -6,8 +6,8 @@ tool: explicit_markers: commit_footers: - - pattern: 'aider:' - regex: false + - pattern: '(?m)^aider: ' + regex: true confidence: 90 description: "Aider commit message prefix" @@ -16,15 +16,9 @@ explicit_markers: confidence: 100 description: "Aider website reference" - text_patterns: - - pattern: 'aider' - location: description - case_insensitive: true - confidence: 60 - description: "Description mentions aider" - branch_patterns: - pattern: '^aider[-/]' + regex: true case_insensitive: true confidence: 80 description: "Branch name starts with aider-" diff --git a/rules/claude-code.yml b/rules/claude-code.yml index fe5de0a..4eb513d 100644 --- a/rules/claude-code.yml +++ b/rules/claude-code.yml @@ -1,6 +1,6 @@ # Claude Code / Anthropic AI Detection Rules # Transcribed from CheckExplicitAIAttributionAction -# Last updated: 2025-12-30 +# Last updated: 2026-09-26 tool: id: claude-code @@ -11,6 +11,9 @@ tool: - claude-3-5-sonnet-20240620 - claude-sonnet-4-5 - claude-opus-4-5 + - claude-sonnet-5 + - claude-opus-5 + - claude-opus-5-5 # Explicit markers provide definitive detection (100% confidence) explicit_markers: @@ -30,14 +33,23 @@ explicit_markers: co_author_attributions: - pattern: 'Co-Authored-By:\s*Claude.*noreply@anthropic\.com' regex: true + case_insensitive: true confidence: 100 description: "Claude co-author attribution" - pattern: 'Co-Authored-By:\s*Claude Sonnet' regex: true + case_insensitive: true confidence: 100 description: "Claude Sonnet co-author" + # Claude Code on the web and desktop worktrees open branches as claude/ + branch_patterns: + - pattern: '^claude/' + regex: true + confidence: 90 + description: "Branch opened by Claude Code (claude/ā€¦)" + # Commit author emails and names bot_authors: - email: noreply@anthropic.com diff --git a/rules/cursor.yml b/rules/cursor.yml index c4defc4..797ad40 100644 --- a/rules/cursor.yml +++ b/rules/cursor.yml @@ -1,6 +1,6 @@ # Cursor Detection Rules # Transcribed from CheckExplicitAIAttributionAction + analysis -# Last updated: 2026-02-21 +# Last updated: 2026-09-26 tool: id: cursor @@ -65,6 +65,13 @@ explicit_markers: location: description description: "Cursor dashboard link in description" + # Background agents open branches as cursor/ + branch_patterns: + - pattern: '^cursor/' + regex: true + confidence: 90 + description: "Branch opened by a Cursor background agent (cursor/ā€¦)" + # Co-author attributions co_author_attributions: - pattern: 'Co-Authored-By:.*Cursor' diff --git a/rules/jules.yml b/rules/jules.yml new file mode 100644 index 0000000..e25bdb0 --- /dev/null +++ b/rules/jules.yml @@ -0,0 +1,33 @@ +# Jules (Google) Detection Rules +# Last updated: 2026-09-26 +# +# Jules opens pull requests itself and signs the description. Seen on 172 +# pull requests in one Coderbuds-tracked repository; none were attributed +# before this file existed. + +tool: + id: jules + name: Jules + provider: Google + website: https://jules.google + +explicit_markers: + # PR description signature + commit_footers: + - pattern: 'PR created automatically by Jules for task' + regex: false + case_insensitive: true + confidence: 100 + description: "Jules pull request signature" + + - pattern: 'jules\.google\.com/task/' + regex: true + confidence: 100 + description: "Link to the Jules task that opened the PR" + + # Branch naming: jules-- + branch_patterns: + - pattern: '^jules-\d{6,}' + regex: true + confidence: 90 + description: "Branch named after a Jules task id" diff --git a/rules/openai-codex.yml b/rules/openai-codex.yml index 0617a62..3257cc5 100644 --- a/rules/openai-codex.yml +++ b/rules/openai-codex.yml @@ -1,6 +1,6 @@ # OpenAI Codex Detection Rules # Transcribed from CheckExplicitAIAttributionAction -# Last updated: 2025-12-30 +# Last updated: 2026-09-26 tool: id: openai-codex @@ -35,28 +35,23 @@ explicit_markers: confidence: 85 description: "Branch contains /codex/" - # Title/description mentions + # Codex cloud titles its pull requests "[codex] ā€¦" text_patterns: - - pattern: 'codex/' - location: title + - pattern: '^\[codex\]' + regex: true case_insensitive: true - confidence: 70 - description: "Title mentions codex/" - - - pattern: 'codex-' location: title - case_insensitive: true - confidence: 70 - description: "Title mentions codex-" + confidence: 90 + description: "Title starts with [codex]" - - pattern: 'codex/' - location: description - case_insensitive: true - confidence: 60 - description: "Description mentions codex/" + # Link back to the Codex cloud task that produced the change + commit_footers: + - pattern: 'chatgpt\.com/codex/(cloud/)?tasks/' + regex: true + confidence: 100 + description: "Link to the Codex task that opened the PR" - - pattern: 'codex-' - location: description - case_insensitive: true - confidence: 60 - description: "Description mentions codex-" +# Deliberately absent: bare mentions of "codex/" or "codex-" in a title or +# description. On Coderbuds' production data they matched Dependabot bumps of +# @openai/codex and `.codex/` config paths ā€” never an agent's own pull request. +# Codex CLI run locally leaves no marker at all; nothing in this file can see it. diff --git a/rules/replit-ai.yml b/rules/replit-ai.yml index 061574d..17ca682 100644 --- a/rules/replit-ai.yml +++ b/rules/replit-ai.yml @@ -6,22 +6,15 @@ tool: explicit_markers: bot_authors: - - pattern: 'replit[bot]' - location: commit_author + - username: 'replit[bot]' confidence: 100 description: "Replit bot commit author" - - pattern: 'ai@replit.com' - location: commit_author_email + - email: 'ai@replit.com' confidence: 100 description: "Replit AI bot email" commit_footers: - - pattern: 'Replit AI' - regex: false - confidence: 95 - description: "Replit AI mention in PR description" - - pattern: 'replit.com/ai' regex: false confidence: 100 @@ -34,7 +27,6 @@ explicit_markers: description: "Replit AI HTML comment" labels: - - pattern: 'replit-ai' - case_insensitive: true + - name: 'replit-ai' confidence: 90 description: "Replit AI PR label" diff --git a/rules/v0-dev.yml b/rules/v0-dev.yml index f531dd3..9861ee2 100644 --- a/rules/v0-dev.yml +++ b/rules/v0-dev.yml @@ -26,10 +26,3 @@ explicit_markers: regex: false confidence: 100 description: "v0 by Vercel comment marker" - - text_patterns: - - pattern: 'v0\.dev' - location: description - case_insensitive: false - confidence: 80 - description: "Description mentions v0.dev" diff --git a/rules/windsurf.yml b/rules/windsurf.yml index 66632ab..4f4a543 100644 --- a/rules/windsurf.yml +++ b/rules/windsurf.yml @@ -6,11 +6,6 @@ tool: explicit_markers: commit_footers: - - pattern: 'WindSurf' - regex: false - confidence: 95 - description: "WindSurf mention in PR description" - - pattern: 'codeium.com/windsurf' regex: false confidence: 100 @@ -24,6 +19,7 @@ explicit_markers: branch_patterns: - pattern: '^windsurf[-/]' + regex: true case_insensitive: true confidence: 80 description: "Branch name starts with windsurf-" diff --git a/scripts/validate_rules.py b/scripts/validate_rules.py new file mode 100644 index 0000000..1da57f2 --- /dev/null +++ b/scripts/validate_rules.py @@ -0,0 +1,204 @@ +#!/usr/bin/env python3 +"""Validate the detection rules and run them against fixture pull requests. + +Two kinds of failure this exists to catch, both of which shipped before: + +1. Rules that can never match. A key the matcher does not read (`pattern` + under `bot_authors`, `pattern` under `labels`) or a regex written without + `regex: true` is silently compared as literal text, forever. +2. Rules that match the wrong thing. Every fixture in fixtures/cases.yml is a + pull request shape seen in production, with the tool it must ā€” or must + not ā€” be attributed to. + +The reference matcher mirrors Coderbuds' CheckYamlRulesAction: rule files are +checked in filename order and the first tool with any matching marker wins. +Patterns are PCRE in production; everything here also compiles under Python's +`re`, and the validator rejects `#` because production uses it as the regex +delimiter. + +Usage: python3 scripts/validate_rules.py +""" + +import re +import sys +from pathlib import Path + +import yaml + +ROOT = Path(__file__).resolve().parent.parent +RULES_DIR = ROOT / "rules" +CASES_FILE = ROOT / "fixtures" / "cases.yml" + +PATTERN_CATEGORIES = { + "commit_footers", + "co_author_attributions", + "html_comments", + "text_patterns", + "branch_patterns", +} +PATTERN_KEYS = {"pattern", "regex", "case_insensitive", "confidence", "description"} +LABEL_KEYS = {"name", "confidence", "description"} +BOT_MATCH_KEYS = {"username", "email", "email_pattern", "name_pattern"} +BOT_KEYS = BOT_MATCH_KEYS | {"regex", "case_insensitive", "confidence", "description"} +TOOL_KEYS = {"id", "name", "provider", "website", "variants"} +REGEX_HINT = re.compile(r"(^\^)|(\\[sdwbS.\[\]()])|(\[[^\]]+\])|(\.\*)") + + +def validate(path: Path, data: dict) -> list[str]: + errors = [] + where = path.name + + tool = data.get("tool") or {} + if not tool.get("id") or not tool.get("name"): + errors.append(f"{where}: tool.id and tool.name are required") + for key in set(tool) - TOOL_KEYS: + errors.append(f"{where}: unknown tool key `{key}`") + + for category, markers in (data.get("explicit_markers") or {}).items(): + if category in PATTERN_CATEGORIES: + allowed, required = PATTERN_KEYS | ({"location"} if category == "text_patterns" else set()), {"pattern"} + elif category == "labels": + allowed, required = LABEL_KEYS, {"name"} + elif category == "bot_authors": + allowed, required = BOT_KEYS, None + else: + errors.append(f"{where}: unknown marker category `{category}` is never checked") + continue + + for index, marker in enumerate(markers or []): + at = f"{where} {category}[{index}]" + + for key in set(marker) - allowed: + errors.append(f"{at}: key `{key}` is never read by the matcher") + + if required and not required <= set(marker): + errors.append(f"{at}: missing {sorted(required - set(marker))}") + + if category == "bot_authors" and not BOT_MATCH_KEYS & set(marker): + errors.append(f"{at}: needs one of {sorted(BOT_MATCH_KEYS)}") + + if category == "text_patterns" and marker.get("location", "description") not in ("title", "description"): + errors.append(f"{at}: location must be title or description") + + confidence = marker.get("confidence") + if not isinstance(confidence, int) or not 0 < confidence <= 100: + errors.append(f"{at}: confidence must be an integer from 1 to 100") + + for key in ("pattern", "email_pattern", "name_pattern"): + pattern = marker.get(key) + if pattern is None: + continue + + is_regex = key != "pattern" or marker.get("regex", False) + + if is_regex: + if "#" in pattern: + errors.append(f"{at}: `#` is the production regex delimiter and cannot appear in a pattern") + try: + re.compile(pattern) + except re.error as error: + errors.append(f"{at}: invalid regex {pattern!r}: {error}") + elif REGEX_HINT.search(pattern): + errors.append(f"{at}: {pattern!r} looks like a regex but has no `regex: true`, so it is matched as literal text") + + return errors + + +def text_matches(text: str | None, marker: dict, key: str = "pattern", force_regex: bool = False) -> bool: + pattern = marker.get(key) + + if not text or not pattern: + return False + + insensitive = marker.get("case_insensitive", False) + + if force_regex or marker.get("regex", False): + return re.search(pattern, text, re.IGNORECASE if insensitive else 0) is not None + + return pattern.lower() in text.lower() if insensitive else pattern in text + + +def bot_matches(commits: list[dict], marker: dict) -> bool: + for commit in commits: + author = commit.get("author") or {} + username = (author.get("username") or "").lower() + email = (author.get("email") or "").lower() + name = author.get("name") or "" + + if username and "username" in marker and marker["username"].lower() in username: + return True + if "email" in marker and marker["email"].lower() == email: + return True + if "email_pattern" in marker and text_matches(email, marker, "email_pattern", force_regex=True): + return True + if "name_pattern" in marker and text_matches(name, marker, "name_pattern", force_regex=True): + return True + + return False + + +def detect(rules: list[dict], pr: dict) -> str | None: + title = pr.get("title", "") + description = pr.get("description", "") + commits = pr.get("commits", []) + messages = [commit.get("message", "") for commit in commits] + labels = [label.lower() for label in pr.get("labels", [])] + branch = pr.get("branch") + + if not branch: + bracketed = re.match(r"^\[([^\]]+)\]", title) + branch = bracketed.group(1) if bracketed else None + + for rule in rules: + for category, markers in (rule.get("explicit_markers") or {}).items(): + for marker in markers or []: + if category in ("commit_footers", "co_author_attributions", "html_comments"): + hit = any(text_matches(text, marker) for text in [description, *messages]) + elif category == "text_patterns": + hit = text_matches(title if marker.get("location") == "title" else description, marker) + elif category == "branch_patterns": + hit = text_matches(branch, marker) + elif category == "labels": + hit = bool(marker.get("name")) and marker["name"].lower() in labels + elif category == "bot_authors": + hit = bot_matches(commits, marker) + else: + hit = False + + if hit: + return rule["tool"]["id"] + + return None + + +def main() -> int: + errors = [] + rules = [] + + for path in sorted(RULES_DIR.glob("*.yml")): + data = yaml.safe_load(path.read_text()) + errors.extend(validate(path, data)) + rules.append(data) + + ids = [rule["tool"]["id"] for rule in rules if rule.get("tool")] + for duplicate in {tool_id for tool_id in ids if ids.count(tool_id) > 1}: + errors.append(f"duplicate tool.id `{duplicate}`") + + cases = yaml.safe_load(CASES_FILE.read_text()) + for case in cases: + detected = detect(rules, case["pr"]) + if detected != case["expect"]: + errors.append(f"case `{case['name']}`: expected {case['expect']!r}, detected {detected!r}") + + for error in errors: + print(f"āœ— {error}") + + if errors: + return 1 + + print(f"āœ“ {len(rules)} rule files valid, {len(cases)} fixture cases pass") + return 0 + + +if __name__ == "__main__": + sys.exit(main())