diff --git a/.github/ISSUE_TEMPLATE/paper.yml b/.github/ISSUE_TEMPLATE/paper.yml index 705ad5b..6f7d909 100644 --- a/.github/ISSUE_TEMPLATE/paper.yml +++ b/.github/ISSUE_TEMPLATE/paper.yml @@ -34,6 +34,47 @@ body: - Historical OpenAI Codex model validations: required: true + - type: dropdown + id: conference + attributes: + label: Conference or source + description: Choose the accepted conference series. Use arXiv when no listed acceptance is verified. + options: + - arXiv + - AAAI + - ASE + - FSE + - ICLR + - ICML + - ICSE + - ISSTA + - NeurIPS + - Other + validations: + required: true + - type: input + id: venue + attributes: + label: Exact venue or track + description: For example, ISSTA Research Papers, ICLR, or arXiv. + validations: + required: true + - type: dropdown + id: domains + attributes: + label: Research domains + multiple: true + options: + - Software engineering + - Security + - Systems and performance + - Machine learning + - Scientific computing + - Formal methods + - Web and UI + - Documents + validations: + required: true - type: dropdown id: classification attributes: diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 040e4f2..f15e1c8 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -31,17 +31,20 @@ same_budget: unknown 1. Add one entry to `data/papers.yaml`. 2. Use a stable identifier such as `short-title-year`. 3. Record authors, identifiers, system name, artifact status, and publication date from primary sources. -4. Choose the narrowest correct evidence class and comparison scope. -5. Record where the supporting result appears in the paper. -6. Run `make build` to regenerate README stats, both JSON exports, views, and paper dossiers. -7. Run `make check` and `make site-check` before opening a pull request. +4. Assign one standardized `conference`, preserve the exact `venue`, and select every applicable `domain`. +5. Choose the narrowest correct evidence class and comparison scope. +6. Record where the supporting result appears in the paper. +7. Run `make build` to regenerate README stats, both JSON exports, views, and paper dossiers. +8. Run `make check` and `make site-check` before opening a pull request. -Do not manually edit content between the `CATALOG:STATS:START` and `CATALOG:STATS:END` markers in either README. Files under `papers/` and `views/`, plus `data/papers.json` and `website/data/catalog.json`, are also generated. +Do not manually edit content between either pair of `CATALOG:STATS` or `CATALOG:COVERAGE` markers in the READMEs. Files under `papers/` and `views/`, plus `data/papers.json` and `website/data/catalog.json`, are also generated. ## Required source checks - Confirm the paper title and authors against an official paper page or PDF. - Distinguish arXiv publication dates from final venue publication. +- Use the standardized conference series only after acceptance is verified; otherwise use `arXiv` and keep the exact source in `venue`. +- Select domains from `docs/taxonomy.md`; multiple domains are encouraged when the evidence genuinely crosses areas. - Confirm that an artifact repository is author-maintained before marking it `official`. - Record the exact CLI version and model when the paper or artifact reports them. - Identify whether the result is product-level, component-level, a configuration ablation, benchmark-only, or historical-model evidence. @@ -70,6 +73,7 @@ Future products must be added to both `data/schema.json` and `docs/taxonomy.md` - [ ] All quantitative claims match the cited primary source. - [ ] Model, version, budget, and tool details are recorded or marked unknown. - [ ] Author, identifier, publication, and artifact fields come from primary sources. +- [ ] Conference, exact venue, and domains follow the catalog taxonomy. - [ ] The evidence location and comparison scope are recorded. - [ ] Comparison caveats are explicit. - [ ] `make check` passes. diff --git a/README.md b/README.md index 430dcde..6fb3594 100644 --- a/README.md +++ b/README.md @@ -18,21 +18,33 @@

Validate catalog - MIT License - PRs welcome + MIT License + PRs welcome Data YAML

- papers: 19 - direct comparisons: 13 - official artifacts: 11 - venues: 7 + papers: 32 + direct comparisons: 19 + official artifacts: 18 + domains: 8 + conference groups: 8 reviewed: 2026-08-21

+## Coverage at a glance + + +

+ Research domains
+ Software Engineering · 21 · Security · 5 · Systems & Performance · 5 · Machine Learning · 1 · Scientific Computing · 2 · Formal Methods · 2 · Web & UI · 3 · Documents · 1

+ Conferences / sources
+ ASE · 1 · FSE · 1 · ICLR · 4 · ICML · 1 · ICSE · 2 · ISSTA · 3 · NeurIPS · 2 · arXiv · 18 +

+ + This repository is the open data and maintenance layer behind the **web-first research catalog**. It covers papers that evaluate, analyze, or outperform **Claude Code** and **Codex CLI** as complete products—not papers that merely use a Claude or GPT-family model. ## Start with the website @@ -40,7 +52,7 @@ This repository is the open data and maintenance layer behind the **web-first re The [interactive catalog](https://micromilo.github.io/awesome-claude-code-codex-papers/) is the fastest way to: - search systems, tasks, methods, authors, and reported results; -- filter by product, evidence class, venue, method, and comparison fairness; +- choose a research domain or conference, then refine by product, evidence class, method, and comparison fairness; - inspect baseline models, versions, budgets, evidence locations, and caveats; - switch between English and Chinese without reading giant Markdown tables. @@ -48,7 +60,7 @@ The [interactive catalog](https://micromilo.github.io/awesome-claude-code-codex- This is not a leaderboard. Product versions, backbone models, budgets, tools, and task domains often differ. Every entry separates the paper's reported result from our comparison controls and caveats. Missing details stay explicitly `unknown`. -For raw or generated research material, use the [paper dossiers](papers/README.md), [method view](views/by-method.md), [fair-comparison view](views/fair-comparisons.md), or [machine-readable JSON](data/papers.json). +For raw or generated research material, use the [paper dossiers](papers/README.md), [domain view](views/by-domain.md), [conference view](views/by-conference.md), [fair-comparison view](views/fair-comparisons.md), or [machine-readable JSON](data/papers.json). ## Contributing diff --git a/README.zh-CN.md b/README.zh-CN.md index 489416f..ccfba4d 100644 --- a/README.zh-CN.md +++ b/README.zh-CN.md @@ -18,21 +18,33 @@

目录校验 - MIT License - 欢迎 PR + MIT License + 欢迎 PR YAML 数据

- papers: 19 - direct comparisons: 13 - official artifacts: 11 - venues: 7 + papers: 32 + direct comparisons: 19 + official artifacts: 18 + domains: 8 + conference groups: 8 reviewed: 2026-08-21

+## 收录范围一览 + + +

+ 研究领域
+ 软件工程 · 21 · 安全 · 5 · 系统与性能 · 5 · 机器学习 · 1 · 科学计算 · 2 · 形式化方法 · 2 · Web 与 UI · 3 · 文档 · 1

+ 会议 / 来源
+ ASE · 1 · FSE · 1 · ICLR · 4 · ICML · 1 · ICSE · 2 · ISSTA · 3 · NeurIPS · 2 · arXiv · 18 +

+ + 这个仓库是**网页优先论文目录**背后的开放数据与维护层。只收录把 **Claude Code** 和 **Codex CLI** 当作完整产品进行评测、分析或超越的论文，而不是只要用了 Claude 或 GPT 系列模型就算。 ## 先看网站 @@ -40,7 +52,7 @@ [可交互论文目录](https://micromilo.github.io/awesome-claude-code-codex-papers/)适合直接： - 搜索系统、任务、方法、作者和论文结果； -- 按产品、证据类型、会议、方法和实验公平性筛选； +- 先选择研究领域或会议，再按产品、证据类型、方法和实验公平性细筛； - 查看 baseline 模型、版本、预算、证据位置和限制； - 在中英文之间切换，不再阅读超长 Markdown 表格。 @@ -48,7 +60,7 @@ 这不是排行榜。产品版本、底层模型、预算、工具权限和任务领域经常不同。每条记录都会分开呈现论文报告的结果、实验控制和限制；论文没写清的内容会保留为 `unknown`。 -需要原始或生成资料时，可以查看[论文证据页](papers/README.md)、[按方法浏览](views/by-method.md)、[公平对比视图](views/fair-comparisons.md)或[机器可读 JSON](data/papers.json)。 +需要原始或生成资料时，可以查看[论文证据页](papers/README.md)、[按领域浏览](views/by-domain.md)、[按会议浏览](views/by-conference.md)、[公平对比视图](views/fair-comparisons.md)或[机器可读 JSON](data/papers.json)。 ## 参与贡献 diff --git a/assets/hero.png b/assets/hero.png index 1f72519..3451977 100644 Binary files a/assets/hero.png and b/assets/hero.png differ diff --git a/data/papers.json b/data/papers.json index 534b870..f26fd37 100644 --- a/data/papers.json +++ b/data/papers.json @@ -12,7 +12,11 @@ "Mayur Naik" ], "year": 2026, + "conference": "ICLR", "venue": "ICLR", + "domains": [ + "security" + ], "publication_status": "main", "classification": "direct", "system": "QLCoder", @@ -72,7 +76,11 @@ "Mao Yang" ], "year": 2026, + "conference": "ICLR", "venue": "ICLR", + "domains": [ + "software-engineering" + ], "publication_status": "main", "classification": "direct", "system": "ZeroRepo", @@ -138,7 +146,11 @@ "Daniel E. Ho" ], "year": 2026, + "conference": "ICLR", "venue": "ICLR", + "domains": [ + "security" + ], "publication_status": "main", "classification": "direct", "system": "ARTEMIS", @@ -205,7 +217,11 @@ "George Karypis" ], "year": 2025, + "conference": "NeurIPS", "venue": "NeurIPS", + "domains": [ + "machine-learning" + ], "publication_status": "main", "classification": "direct", "system": "MLZero", @@ -268,7 +284,11 @@ "Sujian Li" ], "year": 2026, + "conference": "ICML", "venue": "ICML", + "domains": [ + "documents" + ], "publication_status": "main", "classification": "direct", "system": "FormAct", @@ -323,7 +343,11 @@ "Yu-Gang Jiang" ], "year": 2026, + "conference": "arXiv", "venue": "arXiv", + "domains": [ + "software-engineering" + ], "publication_status": "preprint", "classification": "direct", "system": "Agentic Harness Engineering", @@ -375,7 +399,12 @@ "Yanlin Wang" ], "year": 2026, + "conference": "arXiv", "venue": "arXiv", + "domains": [ + "software-engineering", + "web-ui" + ], "publication_status": "preprint", "classification": "direct", "system": "WebDesignIter", @@ -430,7 +459,11 @@ "Michael Pradel" ], "year": 2026, + "conference": "arXiv", "venue": "arXiv", + "domains": [ + "software-engineering" + ], "publication_status": "preprint", "classification": "direct", "system": "Prefactory", @@ -480,7 +513,12 @@ "Nghi D. Q. Bui" ], "year": 2026, + "conference": "arXiv", "venue": "arXiv", + "domains": [ + "software-engineering", + "systems-performance" + ], "publication_status": "preprint", "classification": "direct", "system": "RustPrint", @@ -531,7 +569,12 @@ "Ling Li" ], "year": 2026, + "conference": "arXiv", "venue": "arXiv", + "domains": [ + "scientific-computing", + "systems-performance" + ], "publication_status": "preprint", "classification": "direct", "system": "RepoOMP", @@ -583,7 +626,11 @@ "Reyhaneh Jabbarvand" ], "year": 2026, + "conference": "arXiv", "venue": "arXiv", + "domains": [ + "software-engineering" + ], "publication_status": "preprint", "classification": "direct", "system": "icat-agent", @@ -635,7 +682,11 @@ "Swarat Chaudhuri" ], "year": 2026, + "conference": "arXiv", "venue": "arXiv", + "domains": [ + "software-engineering" + ], "publication_status": "preprint", "classification": "direct", "system": "Co-Coder", @@ -688,7 +739,11 @@ "Christoph Treude" ], "year": 2026, + "conference": "ICSE", "venue": "ICSE JAWs", + "domains": [ + "software-engineering" + ], "publication_status": "workshop", "classification": "direct", "system": "AGENTS.md", @@ -743,7 +798,11 @@ "Yue Lin" ], "year": 2026, + "conference": "arXiv", "venue": "arXiv", + "domains": [ + "software-engineering" + ], "publication_status": "preprint", "classification": "related", "system": "CodeGrep", @@ -791,7 +850,11 @@ "Ali Mesbah" ], "year": 2026, + "conference": "arXiv", "venue": "arXiv", + "domains": [ + "software-engineering" + ], "publication_status": "preprint", "classification": "related", "system": "SCATE", @@ -920,7 +983,12 @@ "Ludwig Schmidt" ], "year": 2026, + "conference": "ICLR", "venue": "ICLR", + "domains": [ + "software-engineering", + "systems-performance" + ], "publication_status": "main", "classification": "evaluation", "system": "Terminal-Bench 2.0", @@ -1004,7 +1072,11 @@ "Percy Liang" ], "year": 2025, + "conference": "NeurIPS", "venue": "NeurIPS", + "domains": [ + "security" + ], "publication_status": "dataset-benchmark", "classification": "evaluation", "system": "BountyBench", @@ -1060,7 +1132,11 @@ "Song Wang" ], "year": 2026, + "conference": "FSE", "venue": "FSE Industry Track", + "domains": [ + "software-engineering" + ], "publication_status": "conference", "classification": "evaluation", "system": "Bug taxonomy", @@ -1121,7 +1197,11 @@ "Jianfeng Gao" ], "year": 2022, + "conference": "ICSE", "venue": "AST at ICSE", + "domains": [ + "software-engineering" + ], "publication_status": "workshop", "classification": "historical", "system": "TiCoder", @@ -1159,6 +1239,752 @@ "source_location": "MBPP and HumanEval evaluation section", "caveats": "This is the historical OpenAI Codex model, not the modern Codex CLI product." } + }, + { + "id": "evodev-2026", + "title": "Towards Iterative End-to-End Software Development: A Feature-Driven Multi-Agent Framework", + "authors": [ + "Junwei Liu", + "Chen Xu", + "Chong Wang", + "Tong Bai", + "Weitong Chen", + "Kaseng Wong", + "Yiling Lou", + "Xin Peng" + ], + "year": 2026, + "conference": "ISSTA", + "venue": "ISSTA", + "domains": [ + "software-engineering", + "web-ui" + ], + "publication_status": "main", + "classification": "direct", + "system": "EvoDev", + "paper_url": "https://conf.researchr.org/details/issta-2026/issta-2026-research-papers/207/Towards-Iterative-End-to-End-Software-Development-A-Feature-Driven-Multi-Agent-Frame", + "artifact_status": "not-found", + "arxiv_id": "2511.02399", + "published_at": "2025-11-04", + "products": [ + { + "product": "claude-code", + "role": "baseline", + "model": "Claude 4 Sonnet", + "version": "not-reported" + } + ], + "task": { + "summary": "Iterative end-to-end Android application development", + "benchmark": "APPDev, 15 Android applications with 8 to 26 functional requirements" + }, + "method": { + "summary": "Feature-driven multi-agent development with a dependency DAG, layered business/design/code context, iterative implementation, and build repair", + "tags": [ + "structured-state", + "dependency-aware-planning", + "multi-agent", + "memory", + "test-feedback" + ] + }, + "evidence": { + "result": "With Claude 4 Sonnet, EvoDev reaches 100% build success and 3.57 function completeness versus 73.3% and 2.27 for Claude Code, a 57.3% relative completeness gain.", + "same_model": "yes", + "same_budget": "no", + "strength": "high", + "claim_type": "mixed", + "comparison_scope": "product-level", + "source_location": "ISSTA 2026 official abstract; Tables 3 and 7", + "caveats": "APPDev contains 15 Android apps and uses manual quality scoring; EvoDev takes about twice Claude Code's wall-clock time in the Claude 4 Sonnet comparison." + } + }, + { + "id": "toolleak-2026", + "title": "Red-Teaming Coding Agents from a Tool-Invocation Perspective: An Empirical Security Assessment", + "authors": [ + "Yuchong Xie", + "Mingyu Luo", + "Zesen Liu", + "Zhixiang Zhang", + "Kaikai Zhang", + "Yu Liu", + "Ci Tao", + "Changhui Wang", + "Zongjie Li", + "Ping Chen", + "Shuai Wang", + "Dongdong She" + ], + "year": 2026, + "conference": "ISSTA", + "venue": "ISSTA", + "domains": [ + "security" + ], + "publication_status": "main", + "classification": "evaluation", + "system": "ToolLeak", + "paper_url": "https://conf.researchr.org/details/issta-2026/issta-2026-research-papers/176/Red-Teaming-Coding-Agents-from-a-Tool-Invocation-Perspective-An-Empirical-Security-A", + "artifact_status": "not-found", + "arxiv_id": "2509.05755", + "published_at": "2025-09-06", + "products": [ + { + "product": "claude-code", + "role": "evaluated", + "model": "multiple", + "version": "multiple" + } + ], + "task": { + "summary": "Prompt exfiltration and tool-invocation hijacking in production coding agents", + "benchmark": "Six real-world coding agents and 25 agent-model configurations" + }, + "method": { + "summary": "Tool-call schema pressure for prompt leakage followed by a two-channel injection through tool descriptions and returns", + "tags": [ + "adversarial-testing", + "observability" + ] + }, + "evidence": { + "result": "ToolLeak obtains the best pseudo-recall on 18 of 25 real agent-model pairs; the follow-on attack hijacks all six studied agents, including a Claude Code RCE case study.", + "same_model": "unknown", + "same_budget": "unknown", + "strength": "high", + "claim_type": "diagnostic", + "comparison_scope": "benchmark-only", + "source_location": "ISSTA 2026 official abstract; Tables 3 to 5 and Sections 6 to 7", + "caveats": "Claude Code is an attack target rather than a baseline to beat; results depend on product and backend versions that may change quickly." + } + }, + { + "id": "execution-ablation-2026", + "title": "To Run or Not to Run: Analyzing the Cost-Effectiveness of Code Execution in LLM-Based Program Repair", + "authors": [ + "Zhihao Lin", + "Junhua Zhu", + "Mingyi Zhou", + "Xin Wang", + "Zhensu Sun", + "Renyu Yang", + "David Lo", + "Li Li" + ], + "year": 2026, + "conference": "ISSTA", + "venue": "ISSTA", + "domains": [ + "software-engineering", + "systems-performance" + ], + "publication_status": "main", + "classification": "related", + "system": "Execution-policy ablation", + "paper_url": "https://arxiv.org/abs/2606.26978", + "artifact_url": "https://github.com/mathieu0905/To_Run_Or_Not_To_Run", + "artifact_status": "official", + "arxiv_id": "2606.26978", + "published_at": "2026-06-25", + "products": [ + { + "product": "claude-code", + "role": "evaluated", + "model": "Claude Sonnet 4.5", + "version": "1.0.16" + }, + { + "product": "codex-cli", + "role": "evaluated", + "model": "GPT-5.2 xhigh", + "version": "0.1.2025062301" + } + ], + "task": { + "summary": "Repository-level program repair with and without local code execution", + "benchmark": "200 SWE-bench Lite and Verified instances; 3,000 controlled runs" + }, + "method": { + "summary": "Paired five-arm execution-policy ablation spanning prohibited, quota-limited, budget-guided, and unrestricted execution", + "tags": [ + "test-feedback", + "verifier-loop", + "context-reduction" + ] + }, + "evidence": { + "result": "Disabling execution changes resolve rate by only 1 to 3 points for Claude Code and Codex in the main cells, while saving 56 to 62% of tokens and 48 to 54% of wall-clock time for Claude Code.", + "same_model": "yes", + "same_budget": "unknown", + "strength": "high", + "claim_type": "mixed", + "comparison_scope": "configuration-ablation", + "source_location": "ISSTA 2026 paper, Tables 2 to 4 and Sections 4.2 to 4.3", + "caveats": "The prompt-level prohibition was not perfectly obeyed by Claude Code, though the paper separately measures and hard-checks unintended execution." + } + }, + { + "id": "llm2ltac-2026", + "title": "Mining Tactics for Automated Theorem Proving", + "authors": [ + "Jian Fang", + "Yixun Yao", + "Yingfei Xiong" + ], + "year": 2026, + "conference": "ASE", + "venue": "ASE", + "domains": [ + "formal-methods" + ], + "publication_status": "main", + "classification": "direct", + "system": "LLM2Ltac", + "paper_url": "https://conf.researchr.org/details/ase-2026/ase-2026-research-track/232/Mining-Tactics-for-Automated-Theorem-Proving", + "artifact_status": "not-found", + "products": [ + { + "product": "claude-code", + "role": "baseline", + "model": "not-reported", + "version": "not-reported" + } + ], + "task": { + "summary": "Automated theorem proving in Rocq projects", + "benchmark": "6,199 theorems across CompCert, Coq-Art, Ext-Lib, and VFA" + }, + "method": { + "summary": "LLM-mined reusable symbolic tactics, validity and generalization checks, and integration into CoqHammer", + "tags": [ + "deterministic-search", + "verifier-loop", + "static-analysis" + ] + }, + "evidence": { + "result": "Mined tactics let CoqHammer prove 23.87% more theorems; integrating the improved prover with Claude Code increases the overall number proved by 9.90%.", + "same_model": "unknown", + "same_budget": "unknown", + "strength": "high", + "claim_type": "quality", + "comparison_scope": "configuration-ablation", + "source_location": "ASE 2026 official abstract", + "caveats": "The public conference record does not report the Claude Code version, backbone model, absolute theorem counts for the product ablation, or budget parity." + } + }, + { + "id": "agentradio-2026", + "title": "AgentRadio: Passive Awareness for Long-Horizon Multi-Agent Collaboration", + "authors": [ + "Xinxing Ren", + "Qianbo Zang", + "Ziyan Wang", + "Caelum Forder", + "Suman Deb", + "Peter Carroll", + "Zekun Guo" + ], + "year": 2026, + "conference": "arXiv", + "venue": "arXiv", + "domains": [ + "software-engineering" + ], + "publication_status": "preprint", + "classification": "direct", + "system": "AgentRadio", + "paper_url": "https://arxiv.org/abs/2607.28430", + "artifact_url": "https://github.com/Coral-Protocol/AgentRadio", + "artifact_status": "official", + "arxiv_id": "2607.28430", + "published_at": "2026-07-30", + "products": [ + { + "product": "claude-code", + "role": "baseline", + "model": "Claude Opus 4.6", + "version": "not-reported" + } + ], + "task": { + "summary": "Long-horizon repository question answering", + "benchmark": "SWE-Atlas QnA, 124 tasks and 1,306 rubrics" + }, + "method": { + "summary": "Four Claude Code agents coordinated through asynchronous threads, messages, passive mention watching, and a five-phase negotiation protocol", + "tags": [ + "multi-agent", + "coordination", + "parallelism" + ] + }, + "evidence": { + "result": "AgentRadio reaches 62.1% task accuracy versus 32.3% for single-agent Claude Code; a near-cost six-run baseline reaches 37.9%, and single-agent Opus 4.8 reaches 57.2%.", + "same_model": "yes", + "same_budget": "no", + "strength": "high", + "claim_type": "mixed", + "comparison_scope": "product-level", + "source_location": "Figure 1 and Tables 1 to 2", + "caveats": "The four-agent method costs about six times a single Opus 4.6 run, although the paper includes a near-cost resampling baseline." + } + }, + { + "id": "context-files-ablation-2026", + "title": "Do Context Files Help Coding Agents? A Two-Agent Ablation Study on Real Repositories", + "authors": [ + "Prakhar Khatri" + ], + "year": 2026, + "conference": "arXiv", + "venue": "arXiv", + "domains": [ + "software-engineering" + ], + "publication_status": "preprint", + "classification": "related", + "system": "Context-file ablation", + "paper_url": "https://arxiv.org/abs/2607.27250", + "artifact_url": "https://github.com/codeprakhar25/context-files-coding-agents", + "artifact_status": "official", + "arxiv_id": "2607.27250", + "published_at": "2026-07-28", + "products": [ + { + "product": "claude-code", + "role": "evaluated", + "model": "Claude Sonnet 4.6", + "version": "not-reported" + }, + { + "product": "codex-cli", + "role": "evaluated", + "model": "GPT-5.5", + "version": "not-reported" + } + ], + "task": { + "summary": "Repository repair under different persistent-context strategies", + "benchmark": "17 real tasks from three Python repositories; 288 evaluated runs" + }, + "method": { + "summary": "Within-task comparison of no context, always-on AGENTS.md, and selectively retrieved repository wiki context with gold tests and equivalence testing", + "tags": [ + "repository-instructions", + "retrieval" + ] + }, + "evidence": { + "result": "Claude Code pass rates are 53.3%, 55.6%, and 55.6%; Codex rates are 58.8%, 56.9%, and 52.9%, with no detectable strategy effect and bounded null effects of 10 to 15 points.", + "same_model": "yes", + "same_budget": "yes", + "strength": "high", + "claim_type": "diagnostic", + "comparison_scope": "configuration-ablation", + "source_location": "Table 1 and Sections 3.4 to 4.4", + "caveats": "Only three Python repositories are studied; Claude and Codex receive injected context through different channels, and two selective-context corpora are larger than AGENTS.md." + } + }, + { + "id": "guardrails-guidance-2026", + "title": "Guardrails Beat Guidance: A Large-Scale Study of Rules, Skills, and Persistent Configuration for Coding Agents", + "authors": [ + "Xing Zhang", + "Guanghui Wang", + "Yanwei Cui", + "Wei Qiu", + "Ziyuan Li", + "Bing Zhu", + "Peiyang He" + ], + "year": 2026, + "conference": "arXiv", + "venue": "arXiv", + "domains": [ + "software-engineering" + ], + "publication_status": "preprint", + "classification": "direct", + "system": "Guardrails Beat Guidance", + "paper_url": "https://arxiv.org/abs/2604.11088", + "artifact_status": "not-found", + "arxiv_id": "2604.11088", + "published_at": "2026-04-13", + "products": [ + { + "product": "claude-code", + "role": "baseline", + "model": "Claude Opus 4.6", + "version": "not-reported" + } + ], + "task": { + "summary": "Repository repair under persistent rule-file configurations", + "benchmark": "SWE-bench Verified; more than 5,000 Claude Code runs" + }, + "method": { + "summary": "Paired rule-file experiments over source, count, polarity, type, and composition, backed by a corpus of 25,532 rules", + "tags": [ + "repository-instructions" + ] + }, + "evidence": { + "result": "Random and expert-curated rule files both reach 63.8% pass rate versus 50.0% with no rules; only negative constraints are individually beneficial in the reported ablation.", + "same_model": "yes", + "same_budget": "unknown", + "strength": "high", + "claim_type": "quality", + "comparison_scope": "configuration-ablation", + "source_location": "Figure 1 and Tables 1 to 5", + "caveats": "Results are limited to Claude Code with Opus 4.6 on a discriminative Python subset; several headline pairwise differences do not reach conventional significance." + } + }, + { + "id": "lcm-2026", + "title": "LCM: Lossless Context Management", + "authors": [ + "Clint Ehrlich", + "Theodore Blackman" + ], + "year": 2026, + "conference": "arXiv", + "venue": "arXiv", + "domains": [ + "systems-performance" + ], + "publication_status": "preprint", + "classification": "direct", + "system": "Volt with LCM", + "paper_url": "https://arxiv.org/abs/2605.04050", + "artifact_url": "https://github.com/Martian-Engineering/volt", + "artifact_status": "official", + "arxiv_id": "2605.04050", + "published_at": "2026-02-14", + "products": [ + { + "product": "claude-code", + "role": "baseline", + "model": "Claude Opus 4.6", + "version": "2.1.4" + } + ], + "task": { + "summary": "Long-context reasoning and aggregation", + "benchmark": "OOLONG trec_coarse, 8K to 1M-token contexts" + }, + "method": { + "summary": "Lossless message storage, hierarchical summary DAGs, deterministic recursive compression, and engine-managed parallel map operators", + "tags": [ + "memory", + "context-reduction", + "parallelism" + ] + }, + "evidence": { + "result": "Volt averages 74.8 versus 70.3 for Claude Code and wins at every tested context length from 32K to 1M tokens, with gaps up to 12.6 points.", + "same_model": "yes", + "same_budget": "unknown", + "strength": "high", + "claim_type": "quality", + "comparison_scope": "product-level", + "source_location": "Section 4.2 and Figures 6 to 7", + "caveats": "OOLONG may be contaminated in Opus 4.6, and the paper does not provide a matched end-to-end cost comparison for LCM's auxiliary calls." + } + }, + { + "id": "brief-product-context-2026", + "title": "Context-Augmented Code Generation: How Product Context Improves AI Coding Agent Decision Compliance by 49%", + "authors": [ + "Drew Dillon", + "Kasyap Varanasi" + ], + "year": 2026, + "conference": "arXiv", + "venue": "arXiv", + "domains": [ + "software-engineering" + ], + "publication_status": "preprint", + "classification": "direct", + "system": "Brief", + "paper_url": "https://arxiv.org/abs/2605.08112", + "artifact_url": "https://github.com/brief-hq/dcbench", + "artifact_status": "official", + "arxiv_id": "2605.08112", + "published_at": "2026-04-27", + "products": [ + { + "product": "claude-code", + "role": "baseline", + "model": "Claude Sonnet 4.6", + "version": "not-reported" + } + ], + "task": { + "summary": "Product-decision-compliant feature implementation", + "benchmark": "DCBench, eight tasks with 41 weighted decision points and 48 runs" + }, + "method": { + "summary": "Retrieval over recorded product decisions, personas, customer signals, and competitive context during specification and implementation", + "tags": [ + "retrieval", + "structured-state", + "test-feedback" + ] + }, + "evidence": { + "result": "Decision compliance rises from 19/41 (46%) for Claude Code to 39/41 (95%) with Brief; cost per merge-ready task falls 68% despite 28% higher total spend.", + "same_model": "no", + "same_budget": "no", + "strength": "high", + "claim_type": "mixed", + "comparison_scope": "product-level", + "source_location": "Tables 2 to 5", + "caveats": "The augmented arm adds Opus planning, a longer timeout, generated specifications, tests, and retrieval, so the 49-point gain is not attributable to product context alone." + } + }, + { + "id": "swe-bench-science-2026", + "title": "SWE-bench Science: Can Coding Agents Resolve Engineering Tasks in Science?", + "authors": [ + "Zhipeng Xu", + "Jiahao Lu", + "Yining Zheng", + "Yuxin Wang", + "Xipeng Qiu" + ], + "year": 2026, + "conference": "arXiv", + "venue": "arXiv", + "domains": [ + "software-engineering", + "scientific-computing" + ], + "publication_status": "preprint", + "classification": "evaluation", + "system": "SWE-bench Science", + "paper_url": "https://arxiv.org/abs/2608.19799", + "artifact_url": "https://github.com/OpenMOSS/SWE-bench-Science", + "artifact_status": "official", + "arxiv_id": "2608.19799", + "published_at": "2026-08-20", + "products": [ + { + "product": "claude-code", + "role": "evaluated", + "model": "Claude Opus 5 and DeepSeek V4 variants", + "version": "not-reported" + }, + { + "product": "codex-cli", + "role": "evaluated", + "model": "GPT-5.6, GLM 5.2, Nex N2, and Qwen 3.5 variants", + "version": "not-reported" + } + ], + "task": { + "summary": "Repository repair in scientific software", + "benchmark": "119 tasks, 98 repositories, and 20 scientific domains" + }, + "method": { + "summary": "Three-paradigm benchmark with private tests, a scientific failure taxonomy, and paired ablations of explicit scientific guidance", + "tags": [ + "benchmark-design", + "observability" + ] + }, + "evidence": { + "result": "The best configuration, Claude Code with Opus 5 at max effort, reaches 47.90% Pass@1; no evaluated agent crosses 50%, and scientific guidance helps some models but hurts others.", + "same_model": "unknown", + "same_budget": "unknown", + "strength": "high", + "claim_type": "diagnostic", + "comparison_scope": "benchmark-only", + "source_location": "Tables 2 to 3 and Figures 5 to 7", + "caveats": "Results span different models, harnesses, and reasoning settings; the paper is a benchmark and failure analysis, not a method that beats the products." + } + }, + { + "id": "swe-bench-mobile-2026", + "title": "SWE-Bench Mobile: Can Large Language Model Agents Develop Industry-Level Mobile Applications?", + "authors": [ + "Muxin Tian", + "Zhe Wang", + "Blair Yang", + "Zhenwei Tang", + "Kunlun Zhu", + "Honghua Dong", + "Hanchen Li", + "Xinni Xie", + "Guangjing Wang", + "Jiaxuan You" + ], + "year": 2026, + "conference": "arXiv", + "venue": "arXiv", + "domains": [ + "software-engineering", + "web-ui" + ], + "publication_status": "preprint", + "classification": "evaluation", + "system": "SWE-Bench Mobile", + "paper_url": "https://arxiv.org/abs/2602.09540", + "artifact_status": "not-found", + "arxiv_id": "2602.09540", + "published_at": "2026-02-10", + "products": [ + { + "product": "claude-code", + "role": "evaluated", + "model": "multiple", + "version": "2.1.37" + }, + { + "product": "codex-cli", + "role": "evaluated", + "model": "multiple", + "version": "0.77.0" + } + ], + "task": { + "summary": "Industry-scale iOS feature development from PRDs and Figma designs", + "benchmark": "50 tasks over a 500,000-line Swift and Objective-C production codebase" + }, + "method": { + "summary": "Multimodal mobile benchmark with private tests, cross-agent model controls, cost analysis, prompt ablations, and error taxonomy", + "tags": [ + "benchmark-design", + "visual-review", + "test-feedback" + ] + }, + "evidence": { + "result": "Across 22 configurations, the best task success is 12%; the same Opus 4.5 model varies by up to 6x across agents, and a defensive-programming prompt raises Claude Code test pass from 19.3% to 26.7%.", + "same_model": "unknown", + "same_budget": "unknown", + "strength": "high", + "claim_type": "diagnostic", + "comparison_scope": "benchmark-only", + "source_location": "Figures 4 to 7 and Tables 3 to 6", + "caveats": "The public artifact URL reported in the PDF was unavailable during review; product results vary across models and include MCP-assisted visual access." + } + }, + { + "id": "sec-bench-pro-2026", + "title": "SEC-bench Pro: Can Language Models Solve Long-Horizon Software Security Tasks?", + "authors": [ + "Hwiwon Lee", + "Jiawei Liu", + "Dongjun Kim", + "Wubing Xia", + "Ziqi Zhang", + "Chunqiu Steven Xia", + "Lingming Zhang" + ], + "year": 2026, + "conference": "arXiv", + "venue": "arXiv", + "domains": [ + "security" + ], + "publication_status": "preprint", + "classification": "evaluation", + "system": "SEC-bench Pro", + "paper_url": "https://arxiv.org/abs/2605.26548", + "artifact_url": "https://github.com/SEC-bench/SEC-bench-Pro", + "artifact_status": "official", + "arxiv_id": "2605.26548", + "published_at": "2026-05-26", + "products": [ + { + "product": "claude-code", + "role": "evaluated", + "model": "Claude Opus 4.6", + "version": "not-reported" + }, + { + "product": "codex-cli", + "role": "evaluated", + "model": "GPT-5.4 and GPT-5.5", + "version": "not-reported" + } + ], + "task": { + "summary": "Long-horizon proof-of-concept synthesis for disclosed vulnerabilities", + "benchmark": "344 validated V8, SpiderMonkey, and Linux kernel vulnerabilities" + }, + "method": { + "summary": "Self-evolving reconstruction pipeline, three-image execution oracle, and LLM judge for vulnerability-specific PoC attribution", + "tags": [ + "benchmark-design", + "verifier-loop" + ] + }, + "evidence": { + "result": "Codex with GPT-5.5 solves 58% of all instances; Claude Code with Opus 4.6 times out often but solves most completed attempts, while GLM-5 solves 13 of 344.", + "same_model": "unknown", + "same_budget": "unknown", + "strength": "high", + "claim_type": "diagnostic", + "comparison_scope": "benchmark-only", + "source_location": "Abstract and main agent-results tables", + "caveats": "The configurations use different models and completion behavior; this is comparative capability measurement rather than a proposed system beating a product baseline." + } + }, + { + "id": "aria-2026", + "title": "Harnessing Code Agents for Automatic Software Verification", + "authors": [ + "Shuangxiang Kan", + "Shuanglong Kan", + "Sebastian Ertel" + ], + "year": 2026, + "conference": "arXiv", + "venue": "arXiv", + "domains": [ + "formal-methods" + ], + "publication_status": "preprint", + "classification": "related", + "system": "Aria", + "paper_url": "https://arxiv.org/abs/2607.06341", + "artifact_status": "not-found", + "arxiv_id": "2607.06341", + "published_at": "2026-07-07", + "products": [ + { + "product": "claude-code", + "role": "host", + "model": "Claude Opus 4.7", + "version": "SDK, not-reported" + } + ], + "task": { + "summary": "Fully automatic proof construction for verified software", + "benchmark": "4,257 Iris lemmas, 217 Rust-library lemmas, 318 reglang theorems, and 72 Lean-port lemmas" + }, + "method": { + "summary": "A reusable harness that lets the code agent choose whole-proof strategy while prover kernels enforce soundness, completeness, retries, and termination", + "tags": [ + "verifier-loop", + "test-feedback" + ] + }, + "evidence": { + "result": "Aria proves all 4,257 targeted Iris lemmas, all 217 Rust-library lemmas, all 318 reglang theorems, and 72 not-yet-ported Lean lemmas; prior reglang provers manage about one in eight.", + "same_model": "no", + "same_budget": "no", + "strength": "high", + "claim_type": "quality", + "comparison_scope": "component-level", + "source_location": "Abstract, Tables 1 and 3 to 4, and Sections 4 to 6", + "caveats": "Claude Code is the host harness, not a separately measured baseline; comparisons to prior provers mix models, systems, and compute budgets." + } } ] } diff --git a/data/papers.yaml b/data/papers.yaml index 0ee48c9..bbd2359 100644 --- a/data/papers.yaml +++ b/data/papers.yaml @@ -5,7 +5,9 @@ papers: title: "QLCoder: A Query Synthesizer For Static Analysis of Security Vulnerabilities" authors: [Claire Wang, Ziyang Li, Saikat Dutta, Mayur Naik] year: 2026 + conference: ICLR venue: ICLR + domains: [security] publication_status: main classification: direct system: QLCoder @@ -39,7 +41,9 @@ papers: title: "RPG: A Repository Planning Graph for Unified and Scalable Codebase Generation" authors: [Jane Luo, Xin Zhang, Steven Liu, Jie Wu, Jianfeng Liu, Yiming Huang, Yangyu Huang, Chengyu Yin, Ying Xin, Yuefeng Zhan, Hao Sun, Qi Chen, Scarlett Li, Mao Yang] year: 2026 + conference: ICLR venue: ICLR + domains: [software-engineering] publication_status: main classification: direct system: ZeroRepo @@ -77,7 +81,9 @@ papers: title: "Comparing AI Agents to Cybersecurity Professionals in Real-World Penetration Testing" authors: [Justin W. Lin, Eliot Krzysztof Jones, Donovan Julian Jasper, Ethan Jun-shen Ho, Anna Wu, Arnold Tianyi Yang, Neil Perry, Andy Zou, Matt Fredrikson, J. Zico Kolter, Percy Liang, Dan Boneh, Daniel E. Ho] year: 2026 + conference: ICLR venue: ICLR + domains: [security] publication_status: main classification: direct system: ARTEMIS @@ -115,7 +121,9 @@ papers: title: "MLZero: A Multi-Agent System for End-to-end Machine Learning Automation" authors: [Haoyang Fang, Boran Han, Nick Erickson, Xiyuan Zhang, Su Zhou, Anirudh Dagar, Jiani Zhang, Ali Caner Turkmen, Cuixiong Hu, Huzefa Rangwala, Ying Nian Wu, Bernie Wang, George Karypis] year: 2025 + conference: NeurIPS venue: NeurIPS + domains: [machine-learning] publication_status: main classification: direct system: MLZero @@ -149,7 +157,9 @@ papers: title: "FormAct: Agentic Source Editing for Rich-Format Document Generation" authors: [Eugene J. Yu, Xingxing Zhang, Yuan Xia, Tao Ge, Xun Wang, FNU Kartik, Vishwas Suryanarayanan, Cheng Yang, Amanda Jiang, Jiayu Ding, Xiangyu Wong, Tengchao Lv, Lei Cui, Si-Qing Chen, Furu Wei, Sujian Li] year: 2026 + conference: ICML venue: ICML + domains: [documents] publication_status: main classification: direct system: FormAct @@ -180,7 +190,9 @@ papers: title: "Agentic Harness Engineering: Observability-Driven Automatic Evolution of Coding-Agent Harnesses" authors: [Jiahang Lin, Shichun Liu, Chengjun Pan, Lizhi Lin, Shihan Dou, Zhiheng Xi, Xuanjing Huang, Hang Yan, Zhenhua Han, Tao Gui, Yu-Gang Jiang] year: 2026 + conference: arXiv venue: arXiv + domains: [software-engineering] publication_status: preprint classification: direct system: Agentic Harness Engineering @@ -214,7 +226,9 @@ papers: title: "WebDesignIter: Co-Evolving Design Knowledge for Repository-Level Front-End Code Generation" authors: [Zheng Pei, Mingwei Liu, Zhenxi Chen, Zihao Wang, Yanlin Wang] year: 2026 + conference: arXiv venue: arXiv + domains: [software-engineering, web-ui] publication_status: preprint classification: direct system: WebDesignIter @@ -252,7 +266,9 @@ papers: title: "Prefactory: Automated Discovery and Application of Library-Adoption Refactorings" authors: [Islem Bouzenia, Michael Pradel] year: 2026 + conference: arXiv venue: arXiv + domains: [software-engineering] publication_status: preprint classification: direct system: Prefactory @@ -285,7 +301,9 @@ papers: title: "RustPrint: Documentation-Guided Agentic Codebase Migration from C to Rust" authors: [Minh Le-Anh, Anh Nguyen Hoang, Bach Le, Nghi D. Q. Bui] year: 2026 + conference: arXiv venue: arXiv + domains: [software-engineering, systems-performance] publication_status: preprint classification: direct system: RustPrint @@ -318,7 +336,9 @@ papers: title: "RepoOMP: Repository-Aware Hotspot OpenMP Parallelization via Dependency-Aware Context Reduction" authors: [Yongjie Qian, Ke Gao, Zhibin Zhang, Shaohui Peng, Ling Li] year: 2026 + conference: arXiv venue: arXiv + domains: [scientific-computing, systems-performance] publication_status: preprint classification: direct system: RepoOMP @@ -352,7 +372,9 @@ papers: title: "Unlocking Model Potentials Through Adaptive Multi-Agent Scaffolding for Efficient Issue Resolution" authors: [Yang Chen, Aliya Ahmad, Yiheng Zhou, Reyhaneh Jabbarvand] year: 2026 + conference: arXiv venue: arXiv + domains: [software-engineering] publication_status: preprint classification: direct system: icat-agent @@ -385,7 +407,9 @@ papers: title: "When Parallelism Pays Off: Cohesion-Aware Task Partitioning for Multi-Agent Coding" authors: [Xu Yang, Lunyiu Nie, Ethan Chandra, Stanislav Gannutin, Fangru Lin, Swarat Chaudhuri] year: 2026 + conference: arXiv venue: arXiv + domains: [software-engineering] publication_status: preprint classification: direct system: Co-Coder @@ -419,7 +443,9 @@ papers: title: "On the Impact of AGENTS.md Files on the Efficiency of AI Coding Agents" authors: [Jai Lal Lulla, Seyedmoein Mohsenimofidi, Matthias Galster, Jie M. Zhang, Sebastian Baltes, Christoph Treude] year: 2026 + conference: ICSE venue: ICSE JAWs + domains: [software-engineering] publication_status: workshop classification: direct system: AGENTS.md @@ -456,7 +482,9 @@ papers: title: "CodeGrep: An RL-Trained Retrieval Agent for LLM Coding Agents" authors: [Wuya Chen, Yihao Yang, Yang Cao, Yue Lin] year: 2026 + conference: arXiv venue: arXiv + domains: [software-engineering] publication_status: preprint classification: related system: CodeGrep @@ -489,7 +517,9 @@ papers: title: "SCATE: Learning to Supervise Coding Agents for Cost-Effective Test Generation" authors: [Sijia Gu, Noor Nashid, Ali Mesbah] year: 2026 + conference: arXiv venue: arXiv + domains: [software-engineering] publication_status: preprint classification: related system: SCATE @@ -522,7 +552,9 @@ papers: title: "Terminal-Bench 2.0: Advancing Agentic Terminal Intelligence through Better Evaluation and Verification" authors: [Mike A. Merrill, Alexander G. Shaw, Nicholas Carlini, Boxuan Li, Harsh Raj, Ivan Bercovich, Lin Shi, Jeong Yeon Shin, Thomas Walshe, E. Kelly Buchanan, Junhong Shen, Guanghao Ye, Haowei Lin, Jason Poulos, Maoyu Wang, Marianna Nezhurina, Jenia Jitsev, Di Lu, Orfeas Menis Mastromichalakis, Zhiwei Xu, Zizhao Chen, Yue Liu, Robert Zhang, Leon Liangyu Chen, Anurag Kashyap, Jan-Lucas Uslu, Jeffrey Li, Jianbo Wu, Minghao Yan, Song Bian, Vedang Sharma, Ke Sun, Steven Dillmann, Akshay Anand, Andrew Lanpouthakoun, Bardia Koopah, Changran Hu, Etash Guha, Gabriel H. S. Dreiman, Jiacheng Zhu, Karl Krauth, Li Zhong, Niklas Muennighoff, Robert Amanfu, Shangyin Tan, Shreyas Pimpalgaonkar, Tushar Aggarwal, Xiangning Lin, Xin Lan, Xuandong Zhao, Yiqing Liang, Yuanli Wang, Zilong Wang, Changzhi Zhou, David Heineman, Hange Liu, Harsh Trivedi, John Yang, Junhong Lin, Manish Shetty, Michael Yang, Nabil Omi, Negin Raoof, Shanda Li, Terry Yue Zhuo, Wuwei Lin, Yiwei Dai, Yuxin Wang, Wenhao Chai, Shang Zhou, Dariush Wahdany, Ziyu She, Jiaming Hu, Zhikang Dong, Yuxuan Zhu, Sasha Cui, Ahson Saiyed, Arinbjörn Kolbeinsson, Jesse Hu, Christopher Michael Rytting, Ryan Marten, Yixin Wang, Alex Dimakis, Andy Konwinski, Ludwig Schmidt] year: 2026 + conference: ICLR venue: ICLR + domains: [software-engineering, systems-performance] publication_status: main classification: evaluation system: Terminal-Bench 2.0 @@ -560,7 +592,9 @@ papers: title: "BountyBench: Dollar Impact of AI Agent Attackers and Defenders on Real-World Cybersecurity Systems" authors: [Andy K. Zhang, Joey Ji, Celeste Menders, Riya Dulepet, Thomas Qin, Ron Y. Wang, Junrong Wu, Kyleen Liao, Jiliang Li, Jinghan Hu, Sara Hong, Nardos Demilew, Shivatmica Murgai, Jason Tran, Nishka Kacheria, Ethan Ho, Denis Liu, Lauren McLane, Olivia Bruvik, Dai-Rong Han, Seungwoo Kim, Akhil Vyas, Cuiyuanxiu Chen, Ryan Li, Weiran Xu, Jonathan Z. Ye, Prerit Choudhary, Siddharth M. Bhatia, Vikram Sivashankar, Yuxuan Bao, Dawn Song, Dan Boneh, Daniel E. Ho, Percy Liang] year: 2025 + conference: NeurIPS venue: NeurIPS + domains: [security] publication_status: dataset-benchmark classification: evaluation system: BountyBench @@ -598,7 +632,9 @@ papers: title: "Engineering Pitfalls in AI Coding Tools: An Empirical Study of Bugs in Claude Code, Codex, and Gemini CLI" authors: [Ruixin Zhang, Wuyang Dai, Hung Viet Pham, Gias Uddin, Jinqiu Yang, Song Wang] year: 2026 + conference: FSE venue: FSE Industry Track + domains: [software-engineering] publication_status: conference classification: evaluation system: Bug taxonomy @@ -636,7 +672,9 @@ papers: title: "Interactive Code Generation via Test-Driven User-Intent Formalization" authors: [Shuvendu K. Lahiri, Sarah Fakhoury, Aaditya Naik, Georgios Sakkas, Saikat Chakraborty, Madanlal Musuvathi, Piali Choudhury, Curtis von Veh, Jeevana Priya Inala, Chenglong Wang, Jianfeng Gao] year: 2022 + conference: ICSE venue: AST at ICSE + domains: [software-engineering] publication_status: workshop classification: historical system: TiCoder @@ -665,3 +703,483 @@ papers: comparison_scope: historical-model source_location: MBPP and HumanEval evaluation section caveats: This is the historical OpenAI Codex model, not the modern Codex CLI product. + + - id: evodev-2026 + title: "Towards Iterative End-to-End Software Development: A Feature-Driven Multi-Agent Framework" + authors: [Junwei Liu, Chen Xu, Chong Wang, Tong Bai, Weitong Chen, Kaseng Wong, Yiling Lou, Xin Peng] + year: 2026 + conference: ISSTA + venue: ISSTA + domains: [software-engineering, web-ui] + publication_status: main + classification: direct + system: EvoDev + paper_url: https://conf.researchr.org/details/issta-2026/issta-2026-research-papers/207/Towards-Iterative-End-to-End-Software-Development-A-Feature-Driven-Multi-Agent-Frame + artifact_status: not-found + arxiv_id: "2511.02399" + published_at: "2025-11-04" + products: + - product: claude-code + role: baseline + model: Claude 4 Sonnet + version: not-reported + task: + summary: Iterative end-to-end Android application development + benchmark: APPDev, 15 Android applications with 8 to 26 functional requirements + method: + summary: Feature-driven multi-agent development with a dependency DAG, layered business/design/code context, iterative implementation, and build repair + tags: [structured-state, dependency-aware-planning, multi-agent, memory, test-feedback] + evidence: + result: "With Claude 4 Sonnet, EvoDev reaches 100% build success and 3.57 function completeness versus 73.3% and 2.27 for Claude Code, a 57.3% relative completeness gain." + same_model: "yes" + same_budget: "no" + strength: high + claim_type: mixed + comparison_scope: product-level + source_location: ISSTA 2026 official abstract; Tables 3 and 7 + caveats: APPDev contains 15 Android apps and uses manual quality scoring; EvoDev takes about twice Claude Code's wall-clock time in the Claude 4 Sonnet comparison. + + - id: toolleak-2026 + title: "Red-Teaming Coding Agents from a Tool-Invocation Perspective: An Empirical Security Assessment" + authors: [Yuchong Xie, Mingyu Luo, Zesen Liu, Zhixiang Zhang, Kaikai Zhang, Yu Liu, Ci Tao, Changhui Wang, Zongjie Li, Ping Chen, Shuai Wang, Dongdong She] + year: 2026 + conference: ISSTA + venue: ISSTA + domains: [security] + publication_status: main + classification: evaluation + system: ToolLeak + paper_url: https://conf.researchr.org/details/issta-2026/issta-2026-research-papers/176/Red-Teaming-Coding-Agents-from-a-Tool-Invocation-Perspective-An-Empirical-Security-A + artifact_status: not-found + arxiv_id: "2509.05755" + published_at: "2025-09-06" + products: + - product: claude-code + role: evaluated + model: multiple + version: multiple + task: + summary: Prompt exfiltration and tool-invocation hijacking in production coding agents + benchmark: Six real-world coding agents and 25 agent-model configurations + method: + summary: Tool-call schema pressure for prompt leakage followed by a two-channel injection through tool descriptions and returns + tags: [adversarial-testing, observability] + evidence: + result: "ToolLeak obtains the best pseudo-recall on 18 of 25 real agent-model pairs; the follow-on attack hijacks all six studied agents, including a Claude Code RCE case study." + same_model: unknown + same_budget: unknown + strength: high + claim_type: diagnostic + comparison_scope: benchmark-only + source_location: ISSTA 2026 official abstract; Tables 3 to 5 and Sections 6 to 7 + caveats: Claude Code is an attack target rather than a baseline to beat; results depend on product and backend versions that may change quickly. + + - id: execution-ablation-2026 + title: "To Run or Not to Run: Analyzing the Cost-Effectiveness of Code Execution in LLM-Based Program Repair" + authors: [Zhihao Lin, Junhua Zhu, Mingyi Zhou, Xin Wang, Zhensu Sun, Renyu Yang, David Lo, Li Li] + year: 2026 + conference: ISSTA + venue: ISSTA + domains: [software-engineering, systems-performance] + publication_status: main + classification: related + system: Execution-policy ablation + paper_url: https://arxiv.org/abs/2606.26978 + artifact_url: https://github.com/mathieu0905/To_Run_Or_Not_To_Run + artifact_status: official + arxiv_id: "2606.26978" + published_at: "2026-06-25" + products: + - product: claude-code + role: evaluated + model: Claude Sonnet 4.5 + version: 1.0.16 + - product: codex-cli + role: evaluated + model: GPT-5.2 xhigh + version: 0.1.2025062301 + task: + summary: Repository-level program repair with and without local code execution + benchmark: 200 SWE-bench Lite and Verified instances; 3,000 controlled runs + method: + summary: Paired five-arm execution-policy ablation spanning prohibited, quota-limited, budget-guided, and unrestricted execution + tags: [test-feedback, verifier-loop, context-reduction] + evidence: + result: "Disabling execution changes resolve rate by only 1 to 3 points for Claude Code and Codex in the main cells, while saving 56 to 62% of tokens and 48 to 54% of wall-clock time for Claude Code." + same_model: "yes" + same_budget: unknown + strength: high + claim_type: mixed + comparison_scope: configuration-ablation + source_location: ISSTA 2026 paper, Tables 2 to 4 and Sections 4.2 to 4.3 + caveats: The prompt-level prohibition was not perfectly obeyed by Claude Code, though the paper separately measures and hard-checks unintended execution. + + - id: llm2ltac-2026 + title: "Mining Tactics for Automated Theorem Proving" + authors: [Jian Fang, Yixun Yao, Yingfei Xiong] + year: 2026 + conference: ASE + venue: ASE + domains: [formal-methods] + publication_status: main + classification: direct + system: LLM2Ltac + paper_url: https://conf.researchr.org/details/ase-2026/ase-2026-research-track/232/Mining-Tactics-for-Automated-Theorem-Proving + artifact_status: not-found + products: + - product: claude-code + role: baseline + model: not-reported + version: not-reported + task: + summary: Automated theorem proving in Rocq projects + benchmark: 6,199 theorems across CompCert, Coq-Art, Ext-Lib, and VFA + method: + summary: LLM-mined reusable symbolic tactics, validity and generalization checks, and integration into CoqHammer + tags: [deterministic-search, verifier-loop, static-analysis] + evidence: + result: "Mined tactics let CoqHammer prove 23.87% more theorems; integrating the improved prover with Claude Code increases the overall number proved by 9.90%." + same_model: unknown + same_budget: unknown + strength: high + claim_type: quality + comparison_scope: configuration-ablation + source_location: ASE 2026 official abstract + caveats: The public conference record does not report the Claude Code version, backbone model, absolute theorem counts for the product ablation, or budget parity. + + - id: agentradio-2026 + title: "AgentRadio: Passive Awareness for Long-Horizon Multi-Agent Collaboration" + authors: [Xinxing Ren, Qianbo Zang, Ziyan Wang, Caelum Forder, Suman Deb, Peter Carroll, Zekun Guo] + year: 2026 + conference: arXiv + venue: arXiv + domains: [software-engineering] + publication_status: preprint + classification: direct + system: AgentRadio + paper_url: https://arxiv.org/abs/2607.28430 + artifact_url: https://github.com/Coral-Protocol/AgentRadio + artifact_status: official + arxiv_id: "2607.28430" + published_at: "2026-07-30" + products: + - product: claude-code + role: baseline + model: Claude Opus 4.6 + version: not-reported + task: + summary: Long-horizon repository question answering + benchmark: SWE-Atlas QnA, 124 tasks and 1,306 rubrics + method: + summary: Four Claude Code agents coordinated through asynchronous threads, messages, passive mention watching, and a five-phase negotiation protocol + tags: [multi-agent, coordination, parallelism] + evidence: + result: "AgentRadio reaches 62.1% task accuracy versus 32.3% for single-agent Claude Code; a near-cost six-run baseline reaches 37.9%, and single-agent Opus 4.8 reaches 57.2%." + same_model: "yes" + same_budget: "no" + strength: high + claim_type: mixed + comparison_scope: product-level + source_location: Figure 1 and Tables 1 to 2 + caveats: The four-agent method costs about six times a single Opus 4.6 run, although the paper includes a near-cost resampling baseline. + + - id: context-files-ablation-2026 + title: "Do Context Files Help Coding Agents? A Two-Agent Ablation Study on Real Repositories" + authors: [Prakhar Khatri] + year: 2026 + conference: arXiv + venue: arXiv + domains: [software-engineering] + publication_status: preprint + classification: related + system: Context-file ablation + paper_url: https://arxiv.org/abs/2607.27250 + artifact_url: https://github.com/codeprakhar25/context-files-coding-agents + artifact_status: official + arxiv_id: "2607.27250" + published_at: "2026-07-28" + products: + - product: claude-code + role: evaluated + model: Claude Sonnet 4.6 + version: not-reported + - product: codex-cli + role: evaluated + model: GPT-5.5 + version: not-reported + task: + summary: Repository repair under different persistent-context strategies + benchmark: 17 real tasks from three Python repositories; 288 evaluated runs + method: + summary: Within-task comparison of no context, always-on AGENTS.md, and selectively retrieved repository wiki context with gold tests and equivalence testing + tags: [repository-instructions, retrieval] + evidence: + result: "Claude Code pass rates are 53.3%, 55.6%, and 55.6%; Codex rates are 58.8%, 56.9%, and 52.9%, with no detectable strategy effect and bounded null effects of 10 to 15 points." + same_model: "yes" + same_budget: "yes" + strength: high + claim_type: diagnostic + comparison_scope: configuration-ablation + source_location: Table 1 and Sections 3.4 to 4.4 + caveats: Only three Python repositories are studied; Claude and Codex receive injected context through different channels, and two selective-context corpora are larger than AGENTS.md. + + - id: guardrails-guidance-2026 + title: "Guardrails Beat Guidance: A Large-Scale Study of Rules, Skills, and Persistent Configuration for Coding Agents" + authors: [Xing Zhang, Guanghui Wang, Yanwei Cui, Wei Qiu, Ziyuan Li, Bing Zhu, Peiyang He] + year: 2026 + conference: arXiv + venue: arXiv + domains: [software-engineering] + publication_status: preprint + classification: direct + system: Guardrails Beat Guidance + paper_url: https://arxiv.org/abs/2604.11088 + artifact_status: not-found + arxiv_id: "2604.11088" + published_at: "2026-04-13" + products: + - product: claude-code + role: baseline + model: Claude Opus 4.6 + version: not-reported + task: + summary: Repository repair under persistent rule-file configurations + benchmark: SWE-bench Verified; more than 5,000 Claude Code runs + method: + summary: Paired rule-file experiments over source, count, polarity, type, and composition, backed by a corpus of 25,532 rules + tags: [repository-instructions] + evidence: + result: "Random and expert-curated rule files both reach 63.8% pass rate versus 50.0% with no rules; only negative constraints are individually beneficial in the reported ablation." + same_model: "yes" + same_budget: unknown + strength: high + claim_type: quality + comparison_scope: configuration-ablation + source_location: Figure 1 and Tables 1 to 5 + caveats: Results are limited to Claude Code with Opus 4.6 on a discriminative Python subset; several headline pairwise differences do not reach conventional significance. + + - id: lcm-2026 + title: "LCM: Lossless Context Management" + authors: [Clint Ehrlich, Theodore Blackman] + year: 2026 + conference: arXiv + venue: arXiv + domains: [systems-performance] + publication_status: preprint + classification: direct + system: Volt with LCM + paper_url: https://arxiv.org/abs/2605.04050 + artifact_url: https://github.com/Martian-Engineering/volt + artifact_status: official + arxiv_id: "2605.04050" + published_at: "2026-02-14" + products: + - product: claude-code + role: baseline + model: Claude Opus 4.6 + version: 2.1.4 + task: + summary: Long-context reasoning and aggregation + benchmark: OOLONG trec_coarse, 8K to 1M-token contexts + method: + summary: Lossless message storage, hierarchical summary DAGs, deterministic recursive compression, and engine-managed parallel map operators + tags: [memory, context-reduction, parallelism] + evidence: + result: "Volt averages 74.8 versus 70.3 for Claude Code and wins at every tested context length from 32K to 1M tokens, with gaps up to 12.6 points." + same_model: "yes" + same_budget: unknown + strength: high + claim_type: quality + comparison_scope: product-level + source_location: Section 4.2 and Figures 6 to 7 + caveats: OOLONG may be contaminated in Opus 4.6, and the paper does not provide a matched end-to-end cost comparison for LCM's auxiliary calls. + + - id: brief-product-context-2026 + title: "Context-Augmented Code Generation: How Product Context Improves AI Coding Agent Decision Compliance by 49%" + authors: [Drew Dillon, Kasyap Varanasi] + year: 2026 + conference: arXiv + venue: arXiv + domains: [software-engineering] + publication_status: preprint + classification: direct + system: Brief + paper_url: https://arxiv.org/abs/2605.08112 + artifact_url: https://github.com/brief-hq/dcbench + artifact_status: official + arxiv_id: "2605.08112" + published_at: "2026-04-27" + products: + - product: claude-code + role: baseline + model: Claude Sonnet 4.6 + version: not-reported + task: + summary: Product-decision-compliant feature implementation + benchmark: DCBench, eight tasks with 41 weighted decision points and 48 runs + method: + summary: Retrieval over recorded product decisions, personas, customer signals, and competitive context during specification and implementation + tags: [retrieval, structured-state, test-feedback] + evidence: + result: "Decision compliance rises from 19/41 (46%) for Claude Code to 39/41 (95%) with Brief; cost per merge-ready task falls 68% despite 28% higher total spend." + same_model: "no" + same_budget: "no" + strength: high + claim_type: mixed + comparison_scope: product-level + source_location: Tables 2 to 5 + caveats: The augmented arm adds Opus planning, a longer timeout, generated specifications, tests, and retrieval, so the 49-point gain is not attributable to product context alone. + + - id: swe-bench-science-2026 + title: "SWE-bench Science: Can Coding Agents Resolve Engineering Tasks in Science?" + authors: [Zhipeng Xu, Jiahao Lu, Yining Zheng, Yuxin Wang, Xipeng Qiu] + year: 2026 + conference: arXiv + venue: arXiv + domains: [software-engineering, scientific-computing] + publication_status: preprint + classification: evaluation + system: SWE-bench Science + paper_url: https://arxiv.org/abs/2608.19799 + artifact_url: https://github.com/OpenMOSS/SWE-bench-Science + artifact_status: official + arxiv_id: "2608.19799" + published_at: "2026-08-20" + products: + - product: claude-code + role: evaluated + model: Claude Opus 5 and DeepSeek V4 variants + version: not-reported + - product: codex-cli + role: evaluated + model: GPT-5.6, GLM 5.2, Nex N2, and Qwen 3.5 variants + version: not-reported + task: + summary: Repository repair in scientific software + benchmark: 119 tasks, 98 repositories, and 20 scientific domains + method: + summary: Three-paradigm benchmark with private tests, a scientific failure taxonomy, and paired ablations of explicit scientific guidance + tags: [benchmark-design, observability] + evidence: + result: "The best configuration, Claude Code with Opus 5 at max effort, reaches 47.90% Pass@1; no evaluated agent crosses 50%, and scientific guidance helps some models but hurts others." + same_model: unknown + same_budget: unknown + strength: high + claim_type: diagnostic + comparison_scope: benchmark-only + source_location: Tables 2 to 3 and Figures 5 to 7 + caveats: Results span different models, harnesses, and reasoning settings; the paper is a benchmark and failure analysis, not a method that beats the products. + + - id: swe-bench-mobile-2026 + title: "SWE-Bench Mobile: Can Large Language Model Agents Develop Industry-Level Mobile Applications?" + authors: [Muxin Tian, Zhe Wang, Blair Yang, Zhenwei Tang, Kunlun Zhu, Honghua Dong, Hanchen Li, Xinni Xie, Guangjing Wang, Jiaxuan You] + year: 2026 + conference: arXiv + venue: arXiv + domains: [software-engineering, web-ui] + publication_status: preprint + classification: evaluation + system: SWE-Bench Mobile + paper_url: https://arxiv.org/abs/2602.09540 + artifact_status: not-found + arxiv_id: "2602.09540" + published_at: "2026-02-10" + products: + - product: claude-code + role: evaluated + model: multiple + version: 2.1.37 + - product: codex-cli + role: evaluated + model: multiple + version: 0.77.0 + task: + summary: Industry-scale iOS feature development from PRDs and Figma designs + benchmark: 50 tasks over a 500,000-line Swift and Objective-C production codebase + method: + summary: Multimodal mobile benchmark with private tests, cross-agent model controls, cost analysis, prompt ablations, and error taxonomy + tags: [benchmark-design, visual-review, test-feedback] + evidence: + result: "Across 22 configurations, the best task success is 12%; the same Opus 4.5 model varies by up to 6x across agents, and a defensive-programming prompt raises Claude Code test pass from 19.3% to 26.7%." + same_model: unknown + same_budget: unknown + strength: high + claim_type: diagnostic + comparison_scope: benchmark-only + source_location: Figures 4 to 7 and Tables 3 to 6 + caveats: The public artifact URL reported in the PDF was unavailable during review; product results vary across models and include MCP-assisted visual access. + + - id: sec-bench-pro-2026 + title: "SEC-bench Pro: Can Language Models Solve Long-Horizon Software Security Tasks?" + authors: [Hwiwon Lee, Jiawei Liu, Dongjun Kim, Wubing Xia, Ziqi Zhang, Chunqiu Steven Xia, Lingming Zhang] + year: 2026 + conference: arXiv + venue: arXiv + domains: [security] + publication_status: preprint + classification: evaluation + system: SEC-bench Pro + paper_url: https://arxiv.org/abs/2605.26548 + artifact_url: https://github.com/SEC-bench/SEC-bench-Pro + artifact_status: official + arxiv_id: "2605.26548" + published_at: "2026-05-26" + products: + - product: claude-code + role: evaluated + model: Claude Opus 4.6 + version: not-reported + - product: codex-cli + role: evaluated + model: GPT-5.4 and GPT-5.5 + version: not-reported + task: + summary: Long-horizon proof-of-concept synthesis for disclosed vulnerabilities + benchmark: 344 validated V8, SpiderMonkey, and Linux kernel vulnerabilities + method: + summary: Self-evolving reconstruction pipeline, three-image execution oracle, and LLM judge for vulnerability-specific PoC attribution + tags: [benchmark-design, verifier-loop] + evidence: + result: "Codex with GPT-5.5 solves 58% of all instances; Claude Code with Opus 4.6 times out often but solves most completed attempts, while GLM-5 solves 13 of 344." + same_model: unknown + same_budget: unknown + strength: high + claim_type: diagnostic + comparison_scope: benchmark-only + source_location: Abstract and main agent-results tables + caveats: The configurations use different models and completion behavior; this is comparative capability measurement rather than a proposed system beating a product baseline. + + - id: aria-2026 + title: "Harnessing Code Agents for Automatic Software Verification" + authors: [Shuangxiang Kan, Shuanglong Kan, Sebastian Ertel] + year: 2026 + conference: arXiv + venue: arXiv + domains: [formal-methods] + publication_status: preprint + classification: related + system: Aria + paper_url: https://arxiv.org/abs/2607.06341 + artifact_status: not-found + arxiv_id: "2607.06341" + published_at: "2026-07-07" + products: + - product: claude-code + role: host + model: Claude Opus 4.7 + version: SDK, not-reported + task: + summary: Fully automatic proof construction for verified software + benchmark: 4,257 Iris lemmas, 217 Rust-library lemmas, 318 reglang theorems, and 72 Lean-port lemmas + method: + summary: A reusable harness that lets the code agent choose whole-proof strategy while prover kernels enforce soundness, completeness, retries, and termination + tags: [verifier-loop, test-feedback] + evidence: + result: "Aria proves all 4,257 targeted Iris lemmas, all 217 Rust-library lemmas, all 318 reglang theorems, and 72 not-yet-ported Lean lemmas; prior reglang provers manage about one in eight." + same_model: "no" + same_budget: "no" + strength: high + claim_type: quality + comparison_scope: component-level + source_location: Abstract, Tables 1 and 3 to 4, and Sections 4 to 6 + caveats: Claude Code is the host harness, not a separately measured baseline; comparisons to prior provers mix models, systems, and compute budgets. diff --git a/data/schema.json b/data/schema.json index c9c9ff9..c8f008c 100644 --- a/data/schema.json +++ b/data/schema.json @@ -51,7 +51,9 @@ "title", "authors", "year", + "conference", "venue", + "domains", "publication_status", "classification", "system", @@ -85,10 +87,41 @@ "minimum": 2020, "maximum": 2100 }, + "conference": { + "enum": [ + "AAAI", + "ASE", + "FSE", + "ICLR", + "ICML", + "ICSE", + "ISSTA", + "NeurIPS", + "arXiv", + "Other" + ] + }, "venue": { "type": "string", "minLength": 2 }, + "domains": { + "type": "array", + "minItems": 1, + "uniqueItems": true, + "items": { + "enum": [ + "software-engineering", + "security", + "systems-performance", + "machine-learning", + "scientific-computing", + "formal-methods", + "web-ui", + "documents" + ] + } + }, "publication_status": { "enum": [ "main", @@ -185,7 +218,10 @@ "observability", "repository-instructions", "deterministic-search", - "visual-review" + "visual-review", + "coordination", + "adversarial-testing", + "benchmark-design" ] } } diff --git a/docs/taxonomy.md b/docs/taxonomy.md index bd8b440..d0bbd16 100644 --- a/docs/taxonomy.md +++ b/docs/taxonomy.md @@ -8,6 +8,23 @@ The product field describes the evaluated harness, not merely the model provider. A paper using a Claude API model inside a custom scaffold is not automatically a Claude Code paper. +## Domains + +Domains describe the task or evidence area studied by a paper. They are multi-valued: a mobile-agent benchmark can be both `software-engineering` and `web-ui`. + +- `software-engineering`: repository understanding, code generation, repair, testing, maintenance, and development workflows. +- `security`: vulnerability discovery, exploitation, red teaming, secure repair, or agent security. +- `systems-performance`: systems work, runtime behavior, resource efficiency, or performance engineering. +- `machine-learning`: model training, architecture search, or ML research automation. +- `scientific-computing`: scientific software, simulation, numerical computing, or research code. +- `formal-methods`: theorem proving, proof assistants, program verification, or formal reasoning. +- `web-ui`: web, mobile, frontend, visual fidelity, or interaction implementation. +- `documents`: document generation, conversion, layout, or structured office artifacts. + +## Conference and venue + +`conference` is the standardized series used by filters: `AAAI`, `ASE`, `FSE`, `ICLR`, `ICML`, `ICSE`, `ISSTA`, `NeurIPS`, `arXiv`, or `Other`. `venue` preserves the exact proceedings, track, workshop, or preprint label reported by the source. A paper that has not been accepted by a listed conference remains `arXiv`, even when its subject fits that community. + ## Evidence classes ### Direct comparison @@ -46,6 +63,9 @@ The paper studies the pre-CLI OpenAI Codex model. These entries are retained for - `repository-instructions`: repository-scoped instruction files such as AGENTS.md. - `deterministic-search`: non-LLM rules or analyses narrow candidates before generation. - `visual-review`: rendered output is inspected and used as feedback. +- `coordination`: explicit communication, synchronization, delegation, or handoff between agents. +- `adversarial-testing`: attacks, red-team probes, or hostile inputs are used to expose failures. +- `benchmark-design`: dataset construction, executable oracles, contamination controls, or evaluation protocol design is the central contribution. ## Comparison controls diff --git a/papers/README.md b/papers/README.md index 138d308..aa0dc01 100644 --- a/papers/README.md +++ b/papers/README.md @@ -6,24 +6,37 @@ Every catalog entry has a generated evidence dossier. The YAML catalog remains the source of truth. -| System | Paper | Venue | Evidence class | Product | Artifact | -|---|---|---|---|---|---| -| [AGENTS.md](agents-md-impact-2026.md) | On the Impact of AGENTS.md Files on the Efficiency of AI Coding Agents | ICSE JAWs 2026 | Direct comparison | Claude Code
Codex CLI | Not found during the latest review | -| [ARTEMIS](artemis-2026.md) | Comparing AI Agents to Cybersecurity Professionals in Real-World Penetration Testing | ICLR 2026 | Direct comparison | Claude Code (Claude Sonnet 4)
Codex CLI (GPT-5) | [Official artifact](https://github.com/Stanford-Trinity/ARTEMIS) | -| [Agentic Harness Engineering](agentic-harness-engineering-2026.md) | Agentic Harness Engineering: Observability-Driven Automatic Evolution of Coding-Agent Harnesses | arXiv 2026 | Direct comparison | Codex CLI | [Official artifact](https://github.com/china-qijizhifeng/agentic-harness-engineering) | -| [Bug taxonomy](engineering-pitfalls-2026.md) | Engineering Pitfalls in AI Coding Tools: An Empirical Study of Bugs in Claude Code, Codex, and Gemini CLI | FSE Industry Track 2026 | Evaluation only | Claude Code
Codex CLI | Not found during the latest review | -| [Co-Coder](co-coder-2026.md) | When Parallelism Pays Off: Cohesion-Aware Task Partitioning for Multi-Agent Coding | arXiv 2026 | Direct comparison | Claude Code (Agent Teams) | [Official artifact](https://github.com/Flitternie/CoCoder) | -| [CodeGrep](codegrep-2026.md) | CodeGrep: An RL-Trained Retrieval Agent for LLM Coding Agents | arXiv 2026 | Related method | Claude Code (Claude-Code-like agent) | Not found during the latest review | -| [FormAct](formact-2026.md) | FormAct: Agentic Source Editing for Rich-Format Document Generation | ICML 2026 | Direct comparison | Codex CLI | Not found during the latest review | -| [Prefactory](prefactory-2026.md) | Prefactory: Automated Discovery and Application of Library-Adoption Refactorings | arXiv 2026 | Direct comparison | Codex CLI | Not found during the latest review | -| [QLCoder](qlcoder-2026.md) | QLCoder: A Query Synthesizer For Static Analysis of Security Vulnerabilities | ICLR 2026 | Direct comparison | Claude Code (Claude Sonnet 4, 1.0.120) | [Official artifact](https://github.com/neuralprogram/qlcoder) | -| [RepoOMP](repoomp-2026.md) | RepoOMP: Repository-Aware Hotspot OpenMP Parallelization via Dependency-Aware Context Reduction | arXiv 2026 | Direct comparison | Claude Code | [Official artifact](https://github.com/Qlalq/RepoOMP_Simplified) | -| [RustPrint](rustprint-2026.md) | RustPrint: Documentation-Guided Agentic Codebase Migration from C to Rust | arXiv 2026 | Direct comparison | Claude Code | Not found during the latest review | -| [SCATE](scate-2026.md) | SCATE: Learning to Supervise Coding Agents for Cost-Effective Test Generation | arXiv 2026 | Related method | Claude Code | Not found during the latest review | -| [Terminal-Bench 2.0](terminal-bench-2-2026.md) | Terminal-Bench 2.0: Advancing Agentic Terminal Intelligence through Better Evaluation and Verification | ICLR 2026 | Evaluation only | Claude Code (Claude Opus 4.5)
Codex CLI (GPT-5.2) | [Official artifact](https://github.com/harbor-framework/terminal-bench-2) | -| [WebDesignIter](webdesigniter-2026.md) | WebDesignIter: Co-Evolving Design Knowledge for Repository-Level Front-End Code Generation | arXiv 2026 | Direct comparison | Claude Code
Codex CLI | [Official artifact](https://github.com/SYSUSELab/WebDesignIter) | -| [ZeroRepo](rpg-zerorepo-2026.md) | RPG: A Repository Planning Graph for Unified and Scalable Codebase Generation | ICLR 2026 | Direct comparison | Claude Code (Claude 4 Sonnet)
Codex CLI (o3-pro) | [Official artifact](https://github.com/microsoft/RPG-ZeroRepo) | -| [icat-agent](icat-agent-2026.md) | Unlocking Model Potentials Through Adaptive Multi-Agent Scaffolding for Efficient Issue Resolution | arXiv 2026 | Direct comparison | Claude Code | Not found during the latest review | -| [BountyBench](bountybench-2025.md) | BountyBench: Dollar Impact of AI Agent Attackers and Defenders on Real-World Cybersecurity Systems | NeurIPS 2025 | Evaluation only | Claude Code
Codex CLI (o3-high and o4-mini) | [Official artifact](https://github.com/bountybench/bountybench) | -| [MLZero](mlzero-2025.md) | MLZero: A Multi-Agent System for End-to-end Machine Learning Automation | NeurIPS 2025 | Direct comparison | Codex CLI (GPT-4.1 and o4-mini) | [Official artifact](https://github.com/autogluon/autogluon-assistant) | -| [TiCoder](ticoder-2022.md) | Interactive Code Generation via Test-Driven User-Intent Formalization | AST at ICSE 2022 | Historical model | OpenAI Codex model (OpenAI Codex) | [Official artifact](https://github.com/microsoft/TiCoder) | +| System | Paper | Conference / venue | Domains | Evidence class | Product | Artifact | +|---|---|---|---|---|---|---| +| [AGENTS.md](agents-md-impact-2026.md) | On the Impact of AGENTS.md Files on the Efficiency of AI Coding Agents | ICSE · ICSE JAWs 2026 | Software Engineering | Direct comparison | Claude Code
Codex CLI | Not found during the latest review | +| [ARTEMIS](artemis-2026.md) | Comparing AI Agents to Cybersecurity Professionals in Real-World Penetration Testing | ICLR · ICLR 2026 | Security | Direct comparison | Claude Code (Claude Sonnet 4)
Codex CLI (GPT-5) | [Official artifact](https://github.com/Stanford-Trinity/ARTEMIS) | +| [AgentRadio](agentradio-2026.md) | AgentRadio: Passive Awareness for Long-Horizon Multi-Agent Collaboration | arXiv · arXiv 2026 | Software Engineering | Direct comparison | Claude Code (Claude Opus 4.6) | [Official artifact](https://github.com/Coral-Protocol/AgentRadio) | +| [Agentic Harness Engineering](agentic-harness-engineering-2026.md) | Agentic Harness Engineering: Observability-Driven Automatic Evolution of Coding-Agent Harnesses | arXiv · arXiv 2026 | Software Engineering | Direct comparison | Codex CLI | [Official artifact](https://github.com/china-qijizhifeng/agentic-harness-engineering) | +| [Aria](aria-2026.md) | Harnessing Code Agents for Automatic Software Verification | arXiv · arXiv 2026 | Formal Methods | Related method | Claude Code (Claude Opus 4.7, SDK, not-reported) | Not found during the latest review | +| [Brief](brief-product-context-2026.md) | Context-Augmented Code Generation: How Product Context Improves AI Coding Agent Decision Compliance by 49% | arXiv · arXiv 2026 | Software Engineering | Direct comparison | Claude Code (Claude Sonnet 4.6) | [Official artifact](https://github.com/brief-hq/dcbench) | +| [Bug taxonomy](engineering-pitfalls-2026.md) | Engineering Pitfalls in AI Coding Tools: An Empirical Study of Bugs in Claude Code, Codex, and Gemini CLI | FSE · FSE Industry Track 2026 | Software Engineering | Evaluation only | Claude Code
Codex CLI | Not found during the latest review | +| [Co-Coder](co-coder-2026.md) | When Parallelism Pays Off: Cohesion-Aware Task Partitioning for Multi-Agent Coding | arXiv · arXiv 2026 | Software Engineering | Direct comparison | Claude Code (Agent Teams) | [Official artifact](https://github.com/Flitternie/CoCoder) | +| [CodeGrep](codegrep-2026.md) | CodeGrep: An RL-Trained Retrieval Agent for LLM Coding Agents | arXiv · arXiv 2026 | Software Engineering | Related method | Claude Code (Claude-Code-like agent) | Not found during the latest review | +| [Context-file ablation](context-files-ablation-2026.md) | Do Context Files Help Coding Agents? A Two-Agent Ablation Study on Real Repositories | arXiv · arXiv 2026 | Software Engineering | Related method | Claude Code (Claude Sonnet 4.6)
Codex CLI (GPT-5.5) | [Official artifact](https://github.com/codeprakhar25/context-files-coding-agents) | +| [EvoDev](evodev-2026.md) | Towards Iterative End-to-End Software Development: A Feature-Driven Multi-Agent Framework | ISSTA · ISSTA 2026 | Software Engineering
Web & UI | Direct comparison | Claude Code (Claude 4 Sonnet) | Not found during the latest review | +| [Execution-policy ablation](execution-ablation-2026.md) | To Run or Not to Run: Analyzing the Cost-Effectiveness of Code Execution in LLM-Based Program Repair | ISSTA · ISSTA 2026 | Software Engineering
Systems & Performance | Related method | Claude Code (Claude Sonnet 4.5, 1.0.16)
Codex CLI (GPT-5.2 xhigh, 0.1.2025062301) | [Official artifact](https://github.com/mathieu0905/To_Run_Or_Not_To_Run) | +| [FormAct](formact-2026.md) | FormAct: Agentic Source Editing for Rich-Format Document Generation | ICML · ICML 2026 | Documents | Direct comparison | Codex CLI | Not found during the latest review | +| [Guardrails Beat Guidance](guardrails-guidance-2026.md) | Guardrails Beat Guidance: A Large-Scale Study of Rules, Skills, and Persistent Configuration for Coding Agents | arXiv · arXiv 2026 | Software Engineering | Direct comparison | Claude Code (Claude Opus 4.6) | Not found during the latest review | +| [LLM2Ltac](llm2ltac-2026.md) | Mining Tactics for Automated Theorem Proving | ASE · ASE 2026 | Formal Methods | Direct comparison | Claude Code | Not found during the latest review | +| [Prefactory](prefactory-2026.md) | Prefactory: Automated Discovery and Application of Library-Adoption Refactorings | arXiv · arXiv 2026 | Software Engineering | Direct comparison | Codex CLI | Not found during the latest review | +| [QLCoder](qlcoder-2026.md) | QLCoder: A Query Synthesizer For Static Analysis of Security Vulnerabilities | ICLR · ICLR 2026 | Security | Direct comparison | Claude Code (Claude Sonnet 4, 1.0.120) | [Official artifact](https://github.com/neuralprogram/qlcoder) | +| [RepoOMP](repoomp-2026.md) | RepoOMP: Repository-Aware Hotspot OpenMP Parallelization via Dependency-Aware Context Reduction | arXiv · arXiv 2026 | Scientific Computing
Systems & Performance | Direct comparison | Claude Code | [Official artifact](https://github.com/Qlalq/RepoOMP_Simplified) | +| [RustPrint](rustprint-2026.md) | RustPrint: Documentation-Guided Agentic Codebase Migration from C to Rust | arXiv · arXiv 2026 | Software Engineering
Systems & Performance | Direct comparison | Claude Code | Not found during the latest review | +| [SCATE](scate-2026.md) | SCATE: Learning to Supervise Coding Agents for Cost-Effective Test Generation | arXiv · arXiv 2026 | Software Engineering | Related method | Claude Code | Not found during the latest review | +| [SEC-bench Pro](sec-bench-pro-2026.md) | SEC-bench Pro: Can Language Models Solve Long-Horizon Software Security Tasks? | arXiv · arXiv 2026 | Security | Evaluation only | Claude Code (Claude Opus 4.6)
Codex CLI (GPT-5.4 and GPT-5.5) | [Official artifact](https://github.com/SEC-bench/SEC-bench-Pro) | +| [SWE-Bench Mobile](swe-bench-mobile-2026.md) | SWE-Bench Mobile: Can Large Language Model Agents Develop Industry-Level Mobile Applications? | arXiv · arXiv 2026 | Software Engineering
Web & UI | Evaluation only | Claude Code (multiple, 2.1.37)
Codex CLI (multiple, 0.77.0) | Not found during the latest review | +| [SWE-bench Science](swe-bench-science-2026.md) | SWE-bench Science: Can Coding Agents Resolve Engineering Tasks in Science? | arXiv · arXiv 2026 | Software Engineering
Scientific Computing | Evaluation only | Claude Code (Claude Opus 5 and DeepSeek V4 variants)
Codex CLI (GPT-5.6, GLM 5.2, Nex N2, and Qwen 3.5 variants) | [Official artifact](https://github.com/OpenMOSS/SWE-bench-Science) | +| [Terminal-Bench 2.0](terminal-bench-2-2026.md) | Terminal-Bench 2.0: Advancing Agentic Terminal Intelligence through Better Evaluation and Verification | ICLR · ICLR 2026 | Software Engineering
Systems & Performance | Evaluation only | Claude Code (Claude Opus 4.5)
Codex CLI (GPT-5.2) | [Official artifact](https://github.com/harbor-framework/terminal-bench-2) | +| [ToolLeak](toolleak-2026.md) | Red-Teaming Coding Agents from a Tool-Invocation Perspective: An Empirical Security Assessment | ISSTA · ISSTA 2026 | Security | Evaluation only | Claude Code (multiple) | Not found during the latest review | +| [Volt with LCM](lcm-2026.md) | LCM: Lossless Context Management | arXiv · arXiv 2026 | Systems & Performance | Direct comparison | Claude Code (Claude Opus 4.6, 2.1.4) | [Official artifact](https://github.com/Martian-Engineering/volt) | +| [WebDesignIter](webdesigniter-2026.md) | WebDesignIter: Co-Evolving Design Knowledge for Repository-Level Front-End Code Generation | arXiv · arXiv 2026 | Software Engineering
Web & UI | Direct comparison | Claude Code
Codex CLI | [Official artifact](https://github.com/SYSUSELab/WebDesignIter) | +| [ZeroRepo](rpg-zerorepo-2026.md) | RPG: A Repository Planning Graph for Unified and Scalable Codebase Generation | ICLR · ICLR 2026 | Software Engineering | Direct comparison | Claude Code (Claude 4 Sonnet)
Codex CLI (o3-pro) | [Official artifact](https://github.com/microsoft/RPG-ZeroRepo) | +| [icat-agent](icat-agent-2026.md) | Unlocking Model Potentials Through Adaptive Multi-Agent Scaffolding for Efficient Issue Resolution | arXiv · arXiv 2026 | Software Engineering | Direct comparison | Claude Code | Not found during the latest review | +| [BountyBench](bountybench-2025.md) | BountyBench: Dollar Impact of AI Agent Attackers and Defenders on Real-World Cybersecurity Systems | NeurIPS · NeurIPS 2025 | Security | Evaluation only | Claude Code
Codex CLI (o3-high and o4-mini) | [Official artifact](https://github.com/bountybench/bountybench) | +| [MLZero](mlzero-2025.md) | MLZero: A Multi-Agent System for End-to-end Machine Learning Automation | NeurIPS · NeurIPS 2025 | Machine Learning | Direct comparison | Codex CLI (GPT-4.1 and o4-mini) | [Official artifact](https://github.com/autogluon/autogluon-assistant) | +| [TiCoder](ticoder-2022.md) | Interactive Code Generation via Test-Driven User-Intent Formalization | ICSE · AST at ICSE 2022 | Software Engineering | Historical model | OpenAI Codex model (OpenAI Codex) | [Official artifact](https://github.com/microsoft/TiCoder) | diff --git a/papers/agentic-harness-engineering-2026.md b/papers/agentic-harness-engineering-2026.md index 7773547..fe9633c 100644 --- a/papers/agentic-harness-engineering-2026.md +++ b/papers/agentic-harness-engineering-2026.md @@ -9,7 +9,9 @@ | Field | Value | |---|---| | Authors | Jiahang Lin, Shichun Liu, Chengjun Pan, Lizhi Lin, Shihan Dou, Zhiheng Xi, Xuanjing Huang, Hang Yan, Zhenhua Han, Tao Gui, Yu-Gang Jiang | +| Conference | arXiv | | Venue | arXiv 2026 (preprint) | +| Domains | Software Engineering | | Evidence class | Direct comparison | | First published | 2026-04-28 | | Identifiers | [arXiv:2604.25850](https://arxiv.org/abs/2604.25850) | diff --git a/papers/agentradio-2026.md b/papers/agentradio-2026.md new file mode 100644 index 0000000..2dcbf92 --- /dev/null +++ b/papers/agentradio-2026.md @@ -0,0 +1,62 @@ + + +[← Paper index](README.md) · [Home](../README.md) + +# AgentRadio + +## AgentRadio: Passive Awareness for Long-Horizon Multi-Agent Collaboration + +| Field | Value | +|---|---| +| Authors | Xinxing Ren, Qianbo Zang, Ziyan Wang, Caelum Forder, Suman Deb, Peter Carroll, Zekun Guo | +| Conference | arXiv | +| Venue | arXiv 2026 (preprint) | +| Domains | Software Engineering | +| Evidence class | Direct comparison | +| First published | 2026-07-30 | +| Identifiers | [arXiv:2607.28430](https://arxiv.org/abs/2607.28430) | +| Artifact | [Official artifact](https://github.com/Coral-Protocol/AgentRadio) | + +## Product configuration + +| Product | Role | Model | Product version | +|---|---|---|---| +| Claude Code | baseline | Claude Opus 4.6 | not-reported | + +## Task + +**Task.** Long-horizon repository question answering + +**Benchmark or scale.** SWE-Atlas QnA, 124 tasks and 1,306 rubrics + +## Method + +Four Claude Code agents coordinated through asynchronous threads, messages, passive mention watching, and a five-phase negotiation protocol + +**Tags.** `multi-agent` `coordination` `parallelism` + +## Reported evidence + +AgentRadio reaches 62.1% task accuracy versus 32.3% for single-agent Claude Code; a near-cost six-run baseline reaches 37.9%, and single-agent Opus 4.8 reaches 57.2%. + +| Control | Recorded value | +|---|---| +| Same model | yes | +| Same budget | no | +| Evidence strength | high | +| Claim type | mixed | +| Comparison scope | product-level | +| Source location | Figure 1 and Tables 1 to 2 | + +## Caveat + +The four-agent method costs about six times a single Opus 4.6 run, although the paper includes a near-cost resampling baseline. + +## Primary links + +- [Paper](https://arxiv.org/abs/2607.28430) +- [Artifact](https://github.com/Coral-Protocol/AgentRadio) + +--- + +Catalog ID: `agentradio-2026` · Metadata last reviewed with catalog release. diff --git a/papers/agents-md-impact-2026.md b/papers/agents-md-impact-2026.md index fc9b515..7e3129e 100644 --- a/papers/agents-md-impact-2026.md +++ b/papers/agents-md-impact-2026.md @@ -9,7 +9,9 @@ | Field | Value | |---|---| | Authors | Jai Lal Lulla, Seyedmoein Mohsenimofidi, Matthias Galster, Jie M. Zhang, Sebastian Baltes, Christoph Treude | +| Conference | ICSE | | Venue | ICSE JAWs 2026 (workshop) | +| Domains | Software Engineering | | Evidence class | Direct comparison | | First published | 2026-01-28 | | Identifiers | [arXiv:2601.20404](https://arxiv.org/abs/2601.20404) | diff --git a/papers/aria-2026.md b/papers/aria-2026.md new file mode 100644 index 0000000..7dce3b3 --- /dev/null +++ b/papers/aria-2026.md @@ -0,0 +1,62 @@ + + +[← Paper index](README.md) · [Home](../README.md) + +# Aria + +## Harnessing Code Agents for Automatic Software Verification + +| Field | Value | +|---|---| +| Authors | Shuangxiang Kan, Shuanglong Kan, Sebastian Ertel | +| Conference | arXiv | +| Venue | arXiv 2026 (preprint) | +| Domains | Formal Methods | +| Evidence class | Related method | +| First published | 2026-07-07 | +| Identifiers | [arXiv:2607.06341](https://arxiv.org/abs/2607.06341) | +| Artifact | Not found during the latest review | + +## Product configuration + +| Product | Role | Model | Product version | +|---|---|---|---| +| Claude Code | host | Claude Opus 4.7 | SDK, not-reported | + +## Task + +**Task.** Fully automatic proof construction for verified software + +**Benchmark or scale.** 4,257 Iris lemmas, 217 Rust-library lemmas, 318 reglang theorems, and 72 Lean-port lemmas + +## Method + +A reusable harness that lets the code agent choose whole-proof strategy while prover kernels enforce soundness, completeness, retries, and termination + +**Tags.** `verifier-loop` `test-feedback` + +## Reported evidence + +Aria proves all 4,257 targeted Iris lemmas, all 217 Rust-library lemmas, all 318 reglang theorems, and 72 not-yet-ported Lean lemmas; prior reglang provers manage about one in eight. + +| Control | Recorded value | +|---|---| +| Same model | no | +| Same budget | no | +| Evidence strength | high | +| Claim type | quality | +| Comparison scope | component-level | +| Source location | Abstract, Tables 1 and 3 to 4, and Sections 4 to 6 | + +## Caveat + +Claude Code is the host harness, not a separately measured baseline; comparisons to prior provers mix models, systems, and compute budgets. + +## Primary links + +- [Paper](https://arxiv.org/abs/2607.06341) +- No official artifact was found during the latest review. + +--- + +Catalog ID: `aria-2026` · Metadata last reviewed with catalog release. diff --git a/papers/artemis-2026.md b/papers/artemis-2026.md index 4d227ea..e243854 100644 --- a/papers/artemis-2026.md +++ b/papers/artemis-2026.md @@ -9,7 +9,9 @@ | Field | Value | |---|---| | Authors | Justin W. Lin, Eliot Krzysztof Jones, Donovan Julian Jasper, Ethan Jun-shen Ho, Anna Wu, Arnold Tianyi Yang, Neil Perry, Andy Zou, Matt Fredrikson, J. Zico Kolter, Percy Liang, Dan Boneh, Daniel E. Ho | +| Conference | ICLR | | Venue | ICLR 2026 (main) | +| Domains | Security | | Evidence class | Direct comparison | | First published | 2025-12-10 | | Identifiers | [arXiv:2512.09882](https://arxiv.org/abs/2512.09882) | diff --git a/papers/bountybench-2025.md b/papers/bountybench-2025.md index 9405baf..7665ca8 100644 --- a/papers/bountybench-2025.md +++ b/papers/bountybench-2025.md @@ -9,7 +9,9 @@ | Field | Value | |---|---| | Authors | Andy K. Zhang, Joey Ji, Celeste Menders, Riya Dulepet, Thomas Qin, Ron Y. Wang, Junrong Wu, Kyleen Liao, Jiliang Li, Jinghan Hu, Sara Hong, Nardos Demilew, Shivatmica Murgai, Jason Tran, Nishka Kacheria, Ethan Ho, Denis Liu, Lauren McLane, Olivia Bruvik, Dai-Rong Han, Seungwoo Kim, Akhil Vyas, Cuiyuanxiu Chen, Ryan Li, Weiran Xu, Jonathan Z. Ye, Prerit Choudhary, Siddharth M. Bhatia, Vikram Sivashankar, Yuxuan Bao, Dawn Song, Dan Boneh, Daniel E. Ho, Percy Liang | +| Conference | NeurIPS | | Venue | NeurIPS 2025 (dataset-benchmark) | +| Domains | Security | | Evidence class | Evaluation only | | First published | 2025-05-21 | | Identifiers | [arXiv:2505.15216](https://arxiv.org/abs/2505.15216) | diff --git a/papers/brief-product-context-2026.md b/papers/brief-product-context-2026.md new file mode 100644 index 0000000..bec0694 --- /dev/null +++ b/papers/brief-product-context-2026.md @@ -0,0 +1,62 @@ + + +[← Paper index](README.md) · [Home](../README.md) + +# Brief + +## Context-Augmented Code Generation: How Product Context Improves AI Coding Agent Decision Compliance by 49% + +| Field | Value | +|---|---| +| Authors | Drew Dillon, Kasyap Varanasi | +| Conference | arXiv | +| Venue | arXiv 2026 (preprint) | +| Domains | Software Engineering | +| Evidence class | Direct comparison | +| First published | 2026-04-27 | +| Identifiers | [arXiv:2605.08112](https://arxiv.org/abs/2605.08112) | +| Artifact | [Official artifact](https://github.com/brief-hq/dcbench) | + +## Product configuration + +| Product | Role | Model | Product version | +|---|---|---|---| +| Claude Code | baseline | Claude Sonnet 4.6 | not-reported | + +## Task + +**Task.** Product-decision-compliant feature implementation + +**Benchmark or scale.** DCBench, eight tasks with 41 weighted decision points and 48 runs + +## Method + +Retrieval over recorded product decisions, personas, customer signals, and competitive context during specification and implementation + +**Tags.** `retrieval` `structured-state` `test-feedback` + +## Reported evidence + +Decision compliance rises from 19/41 (46%) for Claude Code to 39/41 (95%) with Brief; cost per merge-ready task falls 68% despite 28% higher total spend. + +| Control | Recorded value | +|---|---| +| Same model | no | +| Same budget | no | +| Evidence strength | high | +| Claim type | mixed | +| Comparison scope | product-level | +| Source location | Tables 2 to 5 | + +## Caveat + +The augmented arm adds Opus planning, a longer timeout, generated specifications, tests, and retrieval, so the 49-point gain is not attributable to product context alone. + +## Primary links + +- [Paper](https://arxiv.org/abs/2605.08112) +- [Artifact](https://github.com/brief-hq/dcbench) + +--- + +Catalog ID: `brief-product-context-2026` · Metadata last reviewed with catalog release. diff --git a/papers/co-coder-2026.md b/papers/co-coder-2026.md index 56bcafb..c9f7e76 100644 --- a/papers/co-coder-2026.md +++ b/papers/co-coder-2026.md @@ -9,7 +9,9 @@ | Field | Value | |---|---| | Authors | Xu Yang, Lunyiu Nie, Ethan Chandra, Stanislav Gannutin, Fangru Lin, Swarat Chaudhuri | +| Conference | arXiv | | Venue | arXiv 2026 (preprint) | +| Domains | Software Engineering | | Evidence class | Direct comparison | | First published | 2026-05-31 | | Identifiers | [arXiv:2606.00953](https://arxiv.org/abs/2606.00953) | diff --git a/papers/codegrep-2026.md b/papers/codegrep-2026.md index 65c2be3..e2fd538 100644 --- a/papers/codegrep-2026.md +++ b/papers/codegrep-2026.md @@ -9,7 +9,9 @@ | Field | Value | |---|---| | Authors | Wuya Chen, Yihao Yang, Yang Cao, Yue Lin | +| Conference | arXiv | | Venue | arXiv 2026 (preprint) | +| Domains | Software Engineering | | Evidence class | Related method | | First published | 2026-08-06 | | Identifiers | [arXiv:2608.05886](https://arxiv.org/abs/2608.05886) | diff --git a/papers/context-files-ablation-2026.md b/papers/context-files-ablation-2026.md new file mode 100644 index 0000000..a72b166 --- /dev/null +++ b/papers/context-files-ablation-2026.md @@ -0,0 +1,63 @@ + + +[← Paper index](README.md) · [Home](../README.md) + +# Context-file ablation + +## Do Context Files Help Coding Agents? A Two-Agent Ablation Study on Real Repositories + +| Field | Value | +|---|---| +| Authors | Prakhar Khatri | +| Conference | arXiv | +| Venue | arXiv 2026 (preprint) | +| Domains | Software Engineering | +| Evidence class | Related method | +| First published | 2026-07-28 | +| Identifiers | [arXiv:2607.27250](https://arxiv.org/abs/2607.27250) | +| Artifact | [Official artifact](https://github.com/codeprakhar25/context-files-coding-agents) | + +## Product configuration + +| Product | Role | Model | Product version | +|---|---|---|---| +| Claude Code | evaluated | Claude Sonnet 4.6 | not-reported | +| Codex CLI | evaluated | GPT-5.5 | not-reported | + +## Task + +**Task.** Repository repair under different persistent-context strategies + +**Benchmark or scale.** 17 real tasks from three Python repositories; 288 evaluated runs + +## Method + +Within-task comparison of no context, always-on AGENTS.md, and selectively retrieved repository wiki context with gold tests and equivalence testing + +**Tags.** `repository-instructions` `retrieval` + +## Reported evidence + +Claude Code pass rates are 53.3%, 55.6%, and 55.6%; Codex rates are 58.8%, 56.9%, and 52.9%, with no detectable strategy effect and bounded null effects of 10 to 15 points. + +| Control | Recorded value | +|---|---| +| Same model | yes | +| Same budget | yes | +| Evidence strength | high | +| Claim type | diagnostic | +| Comparison scope | configuration-ablation | +| Source location | Table 1 and Sections 3.4 to 4.4 | + +## Caveat + +Only three Python repositories are studied; Claude and Codex receive injected context through different channels, and two selective-context corpora are larger than AGENTS.md. + +## Primary links + +- [Paper](https://arxiv.org/abs/2607.27250) +- [Artifact](https://github.com/codeprakhar25/context-files-coding-agents) + +--- + +Catalog ID: `context-files-ablation-2026` · Metadata last reviewed with catalog release. diff --git a/papers/engineering-pitfalls-2026.md b/papers/engineering-pitfalls-2026.md index b32c63f..3ff887e 100644 --- a/papers/engineering-pitfalls-2026.md +++ b/papers/engineering-pitfalls-2026.md @@ -9,7 +9,9 @@ | Field | Value | |---|---| | Authors | Ruixin Zhang, Wuyang Dai, Hung Viet Pham, Gias Uddin, Jinqiu Yang, Song Wang | +| Conference | FSE | | Venue | FSE Industry Track 2026 (conference) | +| Domains | Software Engineering | | Evidence class | Evaluation only | | First published | 2026-03-21 | | Identifiers | [arXiv:2603.20847](https://arxiv.org/abs/2603.20847) · [DOI:10.1145/3803437.3805213](https://doi.org/10.1145/3803437.3805213) | diff --git a/papers/evodev-2026.md b/papers/evodev-2026.md new file mode 100644 index 0000000..a2b5b28 --- /dev/null +++ b/papers/evodev-2026.md @@ -0,0 +1,62 @@ + + +[← Paper index](README.md) · [Home](../README.md) + +# EvoDev + +## Towards Iterative End-to-End Software Development: A Feature-Driven Multi-Agent Framework + +| Field | Value | +|---|---| +| Authors | Junwei Liu, Chen Xu, Chong Wang, Tong Bai, Weitong Chen, Kaseng Wong, Yiling Lou, Xin Peng | +| Conference | ISSTA | +| Venue | ISSTA 2026 (main) | +| Domains | Software Engineering, Web & UI | +| Evidence class | Direct comparison | +| First published | 2025-11-04 | +| Identifiers | [arXiv:2511.02399](https://arxiv.org/abs/2511.02399) | +| Artifact | Not found during the latest review | + +## Product configuration + +| Product | Role | Model | Product version | +|---|---|---|---| +| Claude Code | baseline | Claude 4 Sonnet | not-reported | + +## Task + +**Task.** Iterative end-to-end Android application development + +**Benchmark or scale.** APPDev, 15 Android applications with 8 to 26 functional requirements + +## Method + +Feature-driven multi-agent development with a dependency DAG, layered business/design/code context, iterative implementation, and build repair + +**Tags.** `structured-state` `dependency-aware-planning` `multi-agent` `memory` `test-feedback` + +## Reported evidence + +With Claude 4 Sonnet, EvoDev reaches 100% build success and 3.57 function completeness versus 73.3% and 2.27 for Claude Code, a 57.3% relative completeness gain. + +| Control | Recorded value | +|---|---| +| Same model | yes | +| Same budget | no | +| Evidence strength | high | +| Claim type | mixed | +| Comparison scope | product-level | +| Source location | ISSTA 2026 official abstract; Tables 3 and 7 | + +## Caveat + +APPDev contains 15 Android apps and uses manual quality scoring; EvoDev takes about twice Claude Code's wall-clock time in the Claude 4 Sonnet comparison. + +## Primary links + +- [Paper](https://conf.researchr.org/details/issta-2026/issta-2026-research-papers/207/Towards-Iterative-End-to-End-Software-Development-A-Feature-Driven-Multi-Agent-Frame) +- No official artifact was found during the latest review. + +--- + +Catalog ID: `evodev-2026` · Metadata last reviewed with catalog release. diff --git a/papers/execution-ablation-2026.md b/papers/execution-ablation-2026.md new file mode 100644 index 0000000..04b9d44 --- /dev/null +++ b/papers/execution-ablation-2026.md @@ -0,0 +1,63 @@ + + +[← Paper index](README.md) · [Home](../README.md) + +# Execution-policy ablation + +## To Run or Not to Run: Analyzing the Cost-Effectiveness of Code Execution in LLM-Based Program Repair + +| Field | Value | +|---|---| +| Authors | Zhihao Lin, Junhua Zhu, Mingyi Zhou, Xin Wang, Zhensu Sun, Renyu Yang, David Lo, Li Li | +| Conference | ISSTA | +| Venue | ISSTA 2026 (main) | +| Domains | Software Engineering, Systems & Performance | +| Evidence class | Related method | +| First published | 2026-06-25 | +| Identifiers | [arXiv:2606.26978](https://arxiv.org/abs/2606.26978) | +| Artifact | [Official artifact](https://github.com/mathieu0905/To_Run_Or_Not_To_Run) | + +## Product configuration + +| Product | Role | Model | Product version | +|---|---|---|---| +| Claude Code | evaluated | Claude Sonnet 4.5 | 1.0.16 | +| Codex CLI | evaluated | GPT-5.2 xhigh | 0.1.2025062301 | + +## Task + +**Task.** Repository-level program repair with and without local code execution + +**Benchmark or scale.** 200 SWE-bench Lite and Verified instances; 3,000 controlled runs + +## Method + +Paired five-arm execution-policy ablation spanning prohibited, quota-limited, budget-guided, and unrestricted execution + +**Tags.** `test-feedback` `verifier-loop` `context-reduction` + +## Reported evidence + +Disabling execution changes resolve rate by only 1 to 3 points for Claude Code and Codex in the main cells, while saving 56 to 62% of tokens and 48 to 54% of wall-clock time for Claude Code. + +| Control | Recorded value | +|---|---| +| Same model | yes | +| Same budget | unknown | +| Evidence strength | high | +| Claim type | mixed | +| Comparison scope | configuration-ablation | +| Source location | ISSTA 2026 paper, Tables 2 to 4 and Sections 4.2 to 4.3 | + +## Caveat + +The prompt-level prohibition was not perfectly obeyed by Claude Code, though the paper separately measures and hard-checks unintended execution. + +## Primary links + +- [Paper](https://arxiv.org/abs/2606.26978) +- [Artifact](https://github.com/mathieu0905/To_Run_Or_Not_To_Run) + +--- + +Catalog ID: `execution-ablation-2026` · Metadata last reviewed with catalog release. diff --git a/papers/formact-2026.md b/papers/formact-2026.md index 8eb1e02..00e9cbf 100644 --- a/papers/formact-2026.md +++ b/papers/formact-2026.md @@ -9,7 +9,9 @@ | Field | Value | |---|---| | Authors | Eugene J. Yu, Xingxing Zhang, Yuan Xia, Tao Ge, Xun Wang, FNU Kartik, Vishwas Suryanarayanan, Cheng Yang, Amanda Jiang, Jiayu Ding, Xiangyu Wong, Tengchao Lv, Lei Cui, Si-Qing Chen, Furu Wei, Sujian Li | +| Conference | ICML | | Venue | ICML 2026 (main) | +| Domains | Documents | | Evidence class | Direct comparison | | First published | Not recorded | | Identifiers | Not recorded | diff --git a/papers/guardrails-guidance-2026.md b/papers/guardrails-guidance-2026.md new file mode 100644 index 0000000..bc8d263 --- /dev/null +++ b/papers/guardrails-guidance-2026.md @@ -0,0 +1,62 @@ + + +[← Paper index](README.md) · [Home](../README.md) + +# Guardrails Beat Guidance + +## Guardrails Beat Guidance: A Large-Scale Study of Rules, Skills, and Persistent Configuration for Coding Agents + +| Field | Value | +|---|---| +| Authors | Xing Zhang, Guanghui Wang, Yanwei Cui, Wei Qiu, Ziyuan Li, Bing Zhu, Peiyang He | +| Conference | arXiv | +| Venue | arXiv 2026 (preprint) | +| Domains | Software Engineering | +| Evidence class | Direct comparison | +| First published | 2026-04-13 | +| Identifiers | [arXiv:2604.11088](https://arxiv.org/abs/2604.11088) | +| Artifact | Not found during the latest review | + +## Product configuration + +| Product | Role | Model | Product version | +|---|---|---|---| +| Claude Code | baseline | Claude Opus 4.6 | not-reported | + +## Task + +**Task.** Repository repair under persistent rule-file configurations + +**Benchmark or scale.** SWE-bench Verified; more than 5,000 Claude Code runs + +## Method + +Paired rule-file experiments over source, count, polarity, type, and composition, backed by a corpus of 25,532 rules + +**Tags.** `repository-instructions` + +## Reported evidence + +Random and expert-curated rule files both reach 63.8% pass rate versus 50.0% with no rules; only negative constraints are individually beneficial in the reported ablation. + +| Control | Recorded value | +|---|---| +| Same model | yes | +| Same budget | unknown | +| Evidence strength | high | +| Claim type | quality | +| Comparison scope | configuration-ablation | +| Source location | Figure 1 and Tables 1 to 5 | + +## Caveat + +Results are limited to Claude Code with Opus 4.6 on a discriminative Python subset; several headline pairwise differences do not reach conventional significance. + +## Primary links + +- [Paper](https://arxiv.org/abs/2604.11088) +- No official artifact was found during the latest review. + +--- + +Catalog ID: `guardrails-guidance-2026` · Metadata last reviewed with catalog release. diff --git a/papers/icat-agent-2026.md b/papers/icat-agent-2026.md index 3fb790a..de9de33 100644 --- a/papers/icat-agent-2026.md +++ b/papers/icat-agent-2026.md @@ -9,7 +9,9 @@ | Field | Value | |---|---| | Authors | Yang Chen, Aliya Ahmad, Yiheng Zhou, Reyhaneh Jabbarvand | +| Conference | arXiv | | Venue | arXiv 2026 (preprint) | +| Domains | Software Engineering | | Evidence class | Direct comparison | | First published | 2026-06-24 | | Identifiers | [arXiv:2606.25514](https://arxiv.org/abs/2606.25514) | diff --git a/papers/lcm-2026.md b/papers/lcm-2026.md new file mode 100644 index 0000000..d4d5300 --- /dev/null +++ b/papers/lcm-2026.md @@ -0,0 +1,62 @@ + + +[← Paper index](README.md) · [Home](../README.md) + +# Volt with LCM + +## LCM: Lossless Context Management + +| Field | Value | +|---|---| +| Authors | Clint Ehrlich, Theodore Blackman | +| Conference | arXiv | +| Venue | arXiv 2026 (preprint) | +| Domains | Systems & Performance | +| Evidence class | Direct comparison | +| First published | 2026-02-14 | +| Identifiers | [arXiv:2605.04050](https://arxiv.org/abs/2605.04050) | +| Artifact | [Official artifact](https://github.com/Martian-Engineering/volt) | + +## Product configuration + +| Product | Role | Model | Product version | +|---|---|---|---| +| Claude Code | baseline | Claude Opus 4.6 | 2.1.4 | + +## Task + +**Task.** Long-context reasoning and aggregation + +**Benchmark or scale.** OOLONG trec_coarse, 8K to 1M-token contexts + +## Method + +Lossless message storage, hierarchical summary DAGs, deterministic recursive compression, and engine-managed parallel map operators + +**Tags.** `memory` `context-reduction` `parallelism` + +## Reported evidence + +Volt averages 74.8 versus 70.3 for Claude Code and wins at every tested context length from 32K to 1M tokens, with gaps up to 12.6 points. + +| Control | Recorded value | +|---|---| +| Same model | yes | +| Same budget | unknown | +| Evidence strength | high | +| Claim type | quality | +| Comparison scope | product-level | +| Source location | Section 4.2 and Figures 6 to 7 | + +## Caveat + +OOLONG may be contaminated in Opus 4.6, and the paper does not provide a matched end-to-end cost comparison for LCM's auxiliary calls. + +## Primary links + +- [Paper](https://arxiv.org/abs/2605.04050) +- [Artifact](https://github.com/Martian-Engineering/volt) + +--- + +Catalog ID: `lcm-2026` · Metadata last reviewed with catalog release. diff --git a/papers/llm2ltac-2026.md b/papers/llm2ltac-2026.md new file mode 100644 index 0000000..c310493 --- /dev/null +++ b/papers/llm2ltac-2026.md @@ -0,0 +1,62 @@ + + +[← Paper index](README.md) · [Home](../README.md) + +# LLM2Ltac + +## Mining Tactics for Automated Theorem Proving + +| Field | Value | +|---|---| +| Authors | Jian Fang, Yixun Yao, Yingfei Xiong | +| Conference | ASE | +| Venue | ASE 2026 (main) | +| Domains | Formal Methods | +| Evidence class | Direct comparison | +| First published | Not recorded | +| Identifiers | Not recorded | +| Artifact | Not found during the latest review | + +## Product configuration + +| Product | Role | Model | Product version | +|---|---|---|---| +| Claude Code | baseline | not-reported | not-reported | + +## Task + +**Task.** Automated theorem proving in Rocq projects + +**Benchmark or scale.** 6,199 theorems across CompCert, Coq-Art, Ext-Lib, and VFA + +## Method + +LLM-mined reusable symbolic tactics, validity and generalization checks, and integration into CoqHammer + +**Tags.** `deterministic-search` `verifier-loop` `static-analysis` + +## Reported evidence + +Mined tactics let CoqHammer prove 23.87% more theorems; integrating the improved prover with Claude Code increases the overall number proved by 9.90%. + +| Control | Recorded value | +|---|---| +| Same model | unknown | +| Same budget | unknown | +| Evidence strength | high | +| Claim type | quality | +| Comparison scope | configuration-ablation | +| Source location | ASE 2026 official abstract | + +## Caveat + +The public conference record does not report the Claude Code version, backbone model, absolute theorem counts for the product ablation, or budget parity. + +## Primary links + +- [Paper](https://conf.researchr.org/details/ase-2026/ase-2026-research-track/232/Mining-Tactics-for-Automated-Theorem-Proving) +- No official artifact was found during the latest review. + +--- + +Catalog ID: `llm2ltac-2026` · Metadata last reviewed with catalog release. diff --git a/papers/mlzero-2025.md b/papers/mlzero-2025.md index 7e3bc54..7198a5c 100644 --- a/papers/mlzero-2025.md +++ b/papers/mlzero-2025.md @@ -9,7 +9,9 @@ | Field | Value | |---|---| | Authors | Haoyang Fang, Boran Han, Nick Erickson, Xiyuan Zhang, Su Zhou, Anirudh Dagar, Jiani Zhang, Ali Caner Turkmen, Cuixiong Hu, Huzefa Rangwala, Ying Nian Wu, Bernie Wang, George Karypis | +| Conference | NeurIPS | | Venue | NeurIPS 2025 (main) | +| Domains | Machine Learning | | Evidence class | Direct comparison | | First published | 2025-05-20 | | Identifiers | [arXiv:2505.13941](https://arxiv.org/abs/2505.13941) | diff --git a/papers/prefactory-2026.md b/papers/prefactory-2026.md index 529b2c8..0c4c4b0 100644 --- a/papers/prefactory-2026.md +++ b/papers/prefactory-2026.md @@ -9,7 +9,9 @@ | Field | Value | |---|---| | Authors | Islem Bouzenia, Michael Pradel | +| Conference | arXiv | | Venue | arXiv 2026 (preprint) | +| Domains | Software Engineering | | Evidence class | Direct comparison | | First published | 2026-07-19 | | Identifiers | [arXiv:2607.17211](https://arxiv.org/abs/2607.17211) | diff --git a/papers/qlcoder-2026.md b/papers/qlcoder-2026.md index f8308d6..c76686d 100644 --- a/papers/qlcoder-2026.md +++ b/papers/qlcoder-2026.md @@ -9,7 +9,9 @@ | Field | Value | |---|---| | Authors | Claire Wang, Ziyang Li, Saikat Dutta, Mayur Naik | +| Conference | ICLR | | Venue | ICLR 2026 (main) | +| Domains | Security | | Evidence class | Direct comparison | | First published | 2025-11-11 | | Identifiers | [arXiv:2511.08462](https://arxiv.org/abs/2511.08462) | diff --git a/papers/repoomp-2026.md b/papers/repoomp-2026.md index d5ffd9d..b3d416c 100644 --- a/papers/repoomp-2026.md +++ b/papers/repoomp-2026.md @@ -9,7 +9,9 @@ | Field | Value | |---|---| | Authors | Yongjie Qian, Ke Gao, Zhibin Zhang, Shaohui Peng, Ling Li | +| Conference | arXiv | | Venue | arXiv 2026 (preprint) | +| Domains | Scientific Computing, Systems & Performance | | Evidence class | Direct comparison | | First published | 2026-08-06 | | Identifiers | [arXiv:2608.05855](https://arxiv.org/abs/2608.05855) | diff --git a/papers/rpg-zerorepo-2026.md b/papers/rpg-zerorepo-2026.md index 7307305..d93128b 100644 --- a/papers/rpg-zerorepo-2026.md +++ b/papers/rpg-zerorepo-2026.md @@ -9,7 +9,9 @@ | Field | Value | |---|---| | Authors | Jane Luo, Xin Zhang, Steven Liu, Jie Wu, Jianfeng Liu, Yiming Huang, Yangyu Huang, Chengyu Yin, Ying Xin, Yuefeng Zhan, Hao Sun, Qi Chen, Scarlett Li, Mao Yang | +| Conference | ICLR | | Venue | ICLR 2026 (main) | +| Domains | Software Engineering | | Evidence class | Direct comparison | | First published | 2025-09-19 | | Identifiers | [arXiv:2509.16198](https://arxiv.org/abs/2509.16198) | diff --git a/papers/rustprint-2026.md b/papers/rustprint-2026.md index debb26c..4b71e8c 100644 --- a/papers/rustprint-2026.md +++ b/papers/rustprint-2026.md @@ -9,7 +9,9 @@ | Field | Value | |---|---| | Authors | Minh Le-Anh, Anh Nguyen Hoang, Bach Le, Nghi D. Q. Bui | +| Conference | arXiv | | Venue | arXiv 2026 (preprint) | +| Domains | Software Engineering, Systems & Performance | | Evidence class | Direct comparison | | First published | 2026-05-14 | | Identifiers | [arXiv:2605.14634](https://arxiv.org/abs/2605.14634) | diff --git a/papers/scate-2026.md b/papers/scate-2026.md index 3d8f614..c2a0999 100644 --- a/papers/scate-2026.md +++ b/papers/scate-2026.md @@ -9,7 +9,9 @@ | Field | Value | |---|---| | Authors | Sijia Gu, Noor Nashid, Ali Mesbah | +| Conference | arXiv | | Venue | arXiv 2026 (preprint) | +| Domains | Software Engineering | | Evidence class | Related method | | First published | 2026-07-09 | | Identifiers | [arXiv:2607.08983](https://arxiv.org/abs/2607.08983) | diff --git a/papers/sec-bench-pro-2026.md b/papers/sec-bench-pro-2026.md new file mode 100644 index 0000000..ac2923d --- /dev/null +++ b/papers/sec-bench-pro-2026.md @@ -0,0 +1,63 @@ + + +[← Paper index](README.md) · [Home](../README.md) + +# SEC-bench Pro + +## SEC-bench Pro: Can Language Models Solve Long-Horizon Software Security Tasks? + +| Field | Value | +|---|---| +| Authors | Hwiwon Lee, Jiawei Liu, Dongjun Kim, Wubing Xia, Ziqi Zhang, Chunqiu Steven Xia, Lingming Zhang | +| Conference | arXiv | +| Venue | arXiv 2026 (preprint) | +| Domains | Security | +| Evidence class | Evaluation only | +| First published | 2026-05-26 | +| Identifiers | [arXiv:2605.26548](https://arxiv.org/abs/2605.26548) | +| Artifact | [Official artifact](https://github.com/SEC-bench/SEC-bench-Pro) | + +## Product configuration + +| Product | Role | Model | Product version | +|---|---|---|---| +| Claude Code | evaluated | Claude Opus 4.6 | not-reported | +| Codex CLI | evaluated | GPT-5.4 and GPT-5.5 | not-reported | + +## Task + +**Task.** Long-horizon proof-of-concept synthesis for disclosed vulnerabilities + +**Benchmark or scale.** 344 validated V8, SpiderMonkey, and Linux kernel vulnerabilities + +## Method + +Self-evolving reconstruction pipeline, three-image execution oracle, and LLM judge for vulnerability-specific PoC attribution + +**Tags.** `benchmark-design` `verifier-loop` + +## Reported evidence + +Codex with GPT-5.5 solves 58% of all instances; Claude Code with Opus 4.6 times out often but solves most completed attempts, while GLM-5 solves 13 of 344. + +| Control | Recorded value | +|---|---| +| Same model | unknown | +| Same budget | unknown | +| Evidence strength | high | +| Claim type | diagnostic | +| Comparison scope | benchmark-only | +| Source location | Abstract and main agent-results tables | + +## Caveat + +The configurations use different models and completion behavior; this is comparative capability measurement rather than a proposed system beating a product baseline. + +## Primary links + +- [Paper](https://arxiv.org/abs/2605.26548) +- [Artifact](https://github.com/SEC-bench/SEC-bench-Pro) + +--- + +Catalog ID: `sec-bench-pro-2026` · Metadata last reviewed with catalog release. diff --git a/papers/swe-bench-mobile-2026.md b/papers/swe-bench-mobile-2026.md new file mode 100644 index 0000000..9248fb0 --- /dev/null +++ b/papers/swe-bench-mobile-2026.md @@ -0,0 +1,63 @@ + + +[← Paper index](README.md) · [Home](../README.md) + +# SWE-Bench Mobile + +## SWE-Bench Mobile: Can Large Language Model Agents Develop Industry-Level Mobile Applications? + +| Field | Value | +|---|---| +| Authors | Muxin Tian, Zhe Wang, Blair Yang, Zhenwei Tang, Kunlun Zhu, Honghua Dong, Hanchen Li, Xinni Xie, Guangjing Wang, Jiaxuan You | +| Conference | arXiv | +| Venue | arXiv 2026 (preprint) | +| Domains | Software Engineering, Web & UI | +| Evidence class | Evaluation only | +| First published | 2026-02-10 | +| Identifiers | [arXiv:2602.09540](https://arxiv.org/abs/2602.09540) | +| Artifact | Not found during the latest review | + +## Product configuration + +| Product | Role | Model | Product version | +|---|---|---|---| +| Claude Code | evaluated | multiple | 2.1.37 | +| Codex CLI | evaluated | multiple | 0.77.0 | + +## Task + +**Task.** Industry-scale iOS feature development from PRDs and Figma designs + +**Benchmark or scale.** 50 tasks over a 500,000-line Swift and Objective-C production codebase + +## Method + +Multimodal mobile benchmark with private tests, cross-agent model controls, cost analysis, prompt ablations, and error taxonomy + +**Tags.** `benchmark-design` `visual-review` `test-feedback` + +## Reported evidence + +Across 22 configurations, the best task success is 12%; the same Opus 4.5 model varies by up to 6x across agents, and a defensive-programming prompt raises Claude Code test pass from 19.3% to 26.7%. + +| Control | Recorded value | +|---|---| +| Same model | unknown | +| Same budget | unknown | +| Evidence strength | high | +| Claim type | diagnostic | +| Comparison scope | benchmark-only | +| Source location | Figures 4 to 7 and Tables 3 to 6 | + +## Caveat + +The public artifact URL reported in the PDF was unavailable during review; product results vary across models and include MCP-assisted visual access. + +## Primary links + +- [Paper](https://arxiv.org/abs/2602.09540) +- No official artifact was found during the latest review. + +--- + +Catalog ID: `swe-bench-mobile-2026` · Metadata last reviewed with catalog release. diff --git a/papers/swe-bench-science-2026.md b/papers/swe-bench-science-2026.md new file mode 100644 index 0000000..ba520dd --- /dev/null +++ b/papers/swe-bench-science-2026.md @@ -0,0 +1,63 @@ + + +[← Paper index](README.md) · [Home](../README.md) + +# SWE-bench Science + +## SWE-bench Science: Can Coding Agents Resolve Engineering Tasks in Science? + +| Field | Value | +|---|---| +| Authors | Zhipeng Xu, Jiahao Lu, Yining Zheng, Yuxin Wang, Xipeng Qiu | +| Conference | arXiv | +| Venue | arXiv 2026 (preprint) | +| Domains | Software Engineering, Scientific Computing | +| Evidence class | Evaluation only | +| First published | 2026-08-20 | +| Identifiers | [arXiv:2608.19799](https://arxiv.org/abs/2608.19799) | +| Artifact | [Official artifact](https://github.com/OpenMOSS/SWE-bench-Science) | + +## Product configuration + +| Product | Role | Model | Product version | +|---|---|---|---| +| Claude Code | evaluated | Claude Opus 5 and DeepSeek V4 variants | not-reported | +| Codex CLI | evaluated | GPT-5.6, GLM 5.2, Nex N2, and Qwen 3.5 variants | not-reported | + +## Task + +**Task.** Repository repair in scientific software + +**Benchmark or scale.** 119 tasks, 98 repositories, and 20 scientific domains + +## Method + +Three-paradigm benchmark with private tests, a scientific failure taxonomy, and paired ablations of explicit scientific guidance + +**Tags.** `benchmark-design` `observability` + +## Reported evidence + +The best configuration, Claude Code with Opus 5 at max effort, reaches 47.90% Pass@1; no evaluated agent crosses 50%, and scientific guidance helps some models but hurts others. + +| Control | Recorded value | +|---|---| +| Same model | unknown | +| Same budget | unknown | +| Evidence strength | high | +| Claim type | diagnostic | +| Comparison scope | benchmark-only | +| Source location | Tables 2 to 3 and Figures 5 to 7 | + +## Caveat + +Results span different models, harnesses, and reasoning settings; the paper is a benchmark and failure analysis, not a method that beats the products. + +## Primary links + +- [Paper](https://arxiv.org/abs/2608.19799) +- [Artifact](https://github.com/OpenMOSS/SWE-bench-Science) + +--- + +Catalog ID: `swe-bench-science-2026` · Metadata last reviewed with catalog release. diff --git a/papers/terminal-bench-2-2026.md b/papers/terminal-bench-2-2026.md index 8e60e2f..bdb370e 100644 --- a/papers/terminal-bench-2-2026.md +++ b/papers/terminal-bench-2-2026.md @@ -9,7 +9,9 @@ | Field | Value | |---|---| | Authors | Mike A. Merrill, Alexander G. Shaw, Nicholas Carlini, Boxuan Li, Harsh Raj, Ivan Bercovich, Lin Shi, Jeong Yeon Shin, Thomas Walshe, E. Kelly Buchanan, Junhong Shen, Guanghao Ye, Haowei Lin, Jason Poulos, Maoyu Wang, Marianna Nezhurina, Jenia Jitsev, Di Lu, Orfeas Menis Mastromichalakis, Zhiwei Xu, Zizhao Chen, Yue Liu, Robert Zhang, Leon Liangyu Chen, Anurag Kashyap, Jan-Lucas Uslu, Jeffrey Li, Jianbo Wu, Minghao Yan, Song Bian, Vedang Sharma, Ke Sun, Steven Dillmann, Akshay Anand, Andrew Lanpouthakoun, Bardia Koopah, Changran Hu, Etash Guha, Gabriel H. S. Dreiman, Jiacheng Zhu, Karl Krauth, Li Zhong, Niklas Muennighoff, Robert Amanfu, Shangyin Tan, Shreyas Pimpalgaonkar, Tushar Aggarwal, Xiangning Lin, Xin Lan, Xuandong Zhao, Yiqing Liang, Yuanli Wang, Zilong Wang, Changzhi Zhou, David Heineman, Hange Liu, Harsh Trivedi, John Yang, Junhong Lin, Manish Shetty, Michael Yang, Nabil Omi, Negin Raoof, Shanda Li, Terry Yue Zhuo, Wuwei Lin, Yiwei Dai, Yuxin Wang, Wenhao Chai, Shang Zhou, Dariush Wahdany, Ziyu She, Jiaming Hu, Zhikang Dong, Yuxuan Zhu, Sasha Cui, Ahson Saiyed, Arinbjörn Kolbeinsson, Jesse Hu, Christopher Michael Rytting, Ryan Marten, Yixin Wang, Alex Dimakis, Andy Konwinski, Ludwig Schmidt | +| Conference | ICLR | | Venue | ICLR 2026 (main) | +| Domains | Software Engineering, Systems & Performance | | Evidence class | Evaluation only | | First published | 2026-01-17 | | Identifiers | [arXiv:2601.11868](https://arxiv.org/abs/2601.11868) | diff --git a/papers/ticoder-2022.md b/papers/ticoder-2022.md index 652d52e..8371517 100644 --- a/papers/ticoder-2022.md +++ b/papers/ticoder-2022.md @@ -9,7 +9,9 @@ | Field | Value | |---|---| | Authors | Shuvendu K. Lahiri, Sarah Fakhoury, Aaditya Naik, Georgios Sakkas, Saikat Chakraborty, Madanlal Musuvathi, Piali Choudhury, Curtis von Veh, Jeevana Priya Inala, Chenglong Wang, Jianfeng Gao | +| Conference | ICSE | | Venue | AST at ICSE 2022 (workshop) | +| Domains | Software Engineering | | Evidence class | Historical model | | First published | 2022-08-11 | | Identifiers | [arXiv:2208.05950](https://arxiv.org/abs/2208.05950) | diff --git a/papers/toolleak-2026.md b/papers/toolleak-2026.md new file mode 100644 index 0000000..db81277 --- /dev/null +++ b/papers/toolleak-2026.md @@ -0,0 +1,62 @@ + + +[← Paper index](README.md) · [Home](../README.md) + +# ToolLeak + +## Red-Teaming Coding Agents from a Tool-Invocation Perspective: An Empirical Security Assessment + +| Field | Value | +|---|---| +| Authors | Yuchong Xie, Mingyu Luo, Zesen Liu, Zhixiang Zhang, Kaikai Zhang, Yu Liu, Ci Tao, Changhui Wang, Zongjie Li, Ping Chen, Shuai Wang, Dongdong She | +| Conference | ISSTA | +| Venue | ISSTA 2026 (main) | +| Domains | Security | +| Evidence class | Evaluation only | +| First published | 2025-09-06 | +| Identifiers | [arXiv:2509.05755](https://arxiv.org/abs/2509.05755) | +| Artifact | Not found during the latest review | + +## Product configuration + +| Product | Role | Model | Product version | +|---|---|---|---| +| Claude Code | evaluated | multiple | multiple | + +## Task + +**Task.** Prompt exfiltration and tool-invocation hijacking in production coding agents + +**Benchmark or scale.** Six real-world coding agents and 25 agent-model configurations + +## Method + +Tool-call schema pressure for prompt leakage followed by a two-channel injection through tool descriptions and returns + +**Tags.** `adversarial-testing` `observability` + +## Reported evidence + +ToolLeak obtains the best pseudo-recall on 18 of 25 real agent-model pairs; the follow-on attack hijacks all six studied agents, including a Claude Code RCE case study. + +| Control | Recorded value | +|---|---| +| Same model | unknown | +| Same budget | unknown | +| Evidence strength | high | +| Claim type | diagnostic | +| Comparison scope | benchmark-only | +| Source location | ISSTA 2026 official abstract; Tables 3 to 5 and Sections 6 to 7 | + +## Caveat + +Claude Code is an attack target rather than a baseline to beat; results depend on product and backend versions that may change quickly. + +## Primary links + +- [Paper](https://conf.researchr.org/details/issta-2026/issta-2026-research-papers/176/Red-Teaming-Coding-Agents-from-a-Tool-Invocation-Perspective-An-Empirical-Security-A) +- No official artifact was found during the latest review. + +--- + +Catalog ID: `toolleak-2026` · Metadata last reviewed with catalog release. diff --git a/papers/webdesigniter-2026.md b/papers/webdesigniter-2026.md index 6467eb2..f29b030 100644 --- a/papers/webdesigniter-2026.md +++ b/papers/webdesigniter-2026.md @@ -9,7 +9,9 @@ | Field | Value | |---|---| | Authors | Zheng Pei, Mingwei Liu, Zhenxi Chen, Zihao Wang, Yanlin Wang | +| Conference | arXiv | | Venue | arXiv 2026 (preprint) | +| Domains | Software Engineering, Web & UI | | Evidence class | Direct comparison | | First published | 2026-07-12 | | Identifiers | [arXiv:2607.10621](https://arxiv.org/abs/2607.10621) | diff --git a/scripts/build_readme.py b/scripts/build_readme.py index 74ef549..1c8ce10 100644 --- a/scripts/build_readme.py +++ b/scripts/build_readme.py @@ -20,6 +20,45 @@ GENERATED_NOTICE = "" STATS_MARKERS = ("", "") +COVERAGE_MARKERS = ( + "", + "", +) + +DOMAIN_LABELS = { + "software-engineering": "Software Engineering", + "security": "Security", + "systems-performance": "Systems & Performance", + "machine-learning": "Machine Learning", + "scientific-computing": "Scientific Computing", + "formal-methods": "Formal Methods", + "web-ui": "Web & UI", + "documents": "Documents", +} + +DOMAIN_LABELS_ZH = { + "software-engineering": "软件工程", + "security": "安全", + "systems-performance": "系统与性能", + "machine-learning": "机器学习", + "scientific-computing": "科学计算", + "formal-methods": "形式化方法", + "web-ui": "Web 与 UI", + "documents": "文档", +} + +CONFERENCE_ORDER = [ + "AAAI", + "ASE", + "FSE", + "ICLR", + "ICML", + "ICSE", + "ISSTA", + "NeurIPS", + "arXiv", + "Other", +] PRODUCT_LABELS = { "claude-code": "Claude Code", @@ -153,13 +192,15 @@ def generate_stats(catalog: dict) -> str: papers = catalog["papers"] direct = sum(paper["classification"] == "direct" for paper in papers) artifacts = sum(paper["artifact_status"] == "official" for paper in papers) - venues = len({paper["venue"] for paper in papers}) + conferences = len({paper["conference"] for paper in papers}) + domains = len({domain for paper in papers for domain in paper["domains"]}) reviewed = catalog["reviewed_at"] badges = [ - ("papers", len(papers), "0f766e"), + ("papers", len(papers), "16616a"), ("direct comparisons", direct, "dc6b46"), ("official artifacts", artifacts, "2563eb"), - ("venues", venues, "7c3aed"), + ("domains", domains, "4bcbd5"), + ("conference groups", conferences, "7c3aed"), ("reviewed", reviewed, "475569"), ] images = [] @@ -173,6 +214,38 @@ def generate_stats(catalog: dict) -> str: return '

\n ' + "\n ".join(images) + "\n

" +def generate_coverage(catalog: dict, language: str) -> str: + papers = catalog["papers"] + domain_counts: dict[str, int] = defaultdict(int) + conference_counts: dict[str, int] = defaultdict(int) + for paper in papers: + conference_counts[paper["conference"]] += 1 + for domain in paper["domains"]: + domain_counts[domain] += 1 + + labels = DOMAIN_LABELS_ZH if language == "zh" else DOMAIN_LABELS + domain_items = " · ".join( + f"{labels[domain]} · {domain_counts[domain]}" + for domain in DOMAIN_LABELS + if domain_counts[domain] + ) + conference_items = " · ".join( + f"{conference} · {conference_counts[conference]}" + for conference in CONFERENCE_ORDER + if conference_counts[conference] + ) + domain_title = "研究领域" if language == "zh" else "Research domains" + conference_title = "会议 / 来源" if language == "zh" else "Conferences / sources" + return ( + '

\n' + f' {domain_title}
\n' + f" {domain_items}

\n" + f' {conference_title}
\n' + f" {conference_items}\n" + "

" + ) + + def replace_section(text: str, start: str, end: str, content: str) -> str: pattern = re.compile(re.escape(start) + r".*?" + re.escape(end), re.DOTALL) replacement = f"{start}\n{content}\n{end}" @@ -182,9 +255,10 @@ def replace_section(text: str, start: str, end: str, content: str) -> str: return updated -def render_marked_readme(path: Path, catalog: dict) -> str: +def render_marked_readme(path: Path, catalog: dict, language: str) -> str: text = path.read_text(encoding="utf-8") - return replace_section(text, *STATS_MARKERS, generate_stats(catalog)) + text = replace_section(text, *STATS_MARKERS, generate_stats(catalog)) + return replace_section(text, *COVERAGE_MARKERS, generate_coverage(catalog, language)) def short_authors(paper: dict) -> str: @@ -247,7 +321,9 @@ def render_paper_page(paper: dict) -> str: | Field | Value | |---|---| | Authors | {escape_cell(", ".join(paper["authors"]))} | +| Conference | {escape_cell(paper["conference"])} | | Venue | {escape_cell(paper["venue"])} {paper["year"]} ({paper["publication_status"]}) | +| Domains | {escape_cell(", ".join(DOMAIN_LABELS[domain] for domain in paper["domains"]))} | | Evidence class | {CLASS_LABELS[paper["classification"]]} | | First published | {published} | | Identifiers | {" · ".join(identifiers)} | @@ -308,7 +384,8 @@ def render_paper_index(catalog: dict) -> str: [ f"[{escape_cell(paper['system'])}]({paper['id']}.md)", escape_cell(paper["title"]), - f"{paper['venue']} {paper['year']}", + f"{paper['conference']} · {paper['venue']} {paper['year']}", + "
".join(DOMAIN_LABELS[domain] for domain in paper["domains"]), CLASS_LABELS[paper["classification"]], format_products(paper), artifact_label(paper), @@ -324,8 +401,8 @@ def render_paper_index(catalog: dict) -> str: Every catalog entry has a generated evidence dossier. The YAML catalog remains the source of truth. -| System | Paper | Venue | Evidence class | Product | Artifact | -|---|---|---|---|---|---| +| System | Paper | Conference / venue | Domains | Evidence class | Product | Artifact | +|---|---|---|---|---|---|---| {chr(10).join(rows)} """ @@ -342,9 +419,11 @@ def render_views_index() -> str: # Research views - [By product](by-product.md) — Claude Code and Codex CLI separately. +- [By domain](by-domain.md) — software engineering, security, systems, formal methods, and more. +- [By conference](by-conference.md) — standardized conference series plus arXiv-only papers. - [By method](by-method.md) — repository graphs, retrieval, verification, orchestration, and more. - [Comparison fairness](fair-comparisons.md) — same-model and same-budget controls. -- [By venue](by-venue.md) — conferences, workshops, and preprints. +- [By exact venue](by-venue.md) — tracks, workshops, and proceedings labels as reported. - [All paper dossiers](../papers/README.md) — one evidence page per paper. """ @@ -412,6 +491,73 @@ def render_by_method(catalog: dict) -> str: """ +def render_by_domain(catalog: dict) -> str: + grouped: dict[str, list[dict]] = defaultdict(list) + for paper in catalog["papers"]: + for domain in paper["domains"]: + grouped[domain].append(paper) + + sections = [] + for domain in DOMAIN_LABELS: + papers = grouped.get(domain, []) + if not papers: + continue + rows = [ + f"| {view_paper_link(paper)} | {paper['conference']} | {paper['year']} | " + f"{CLASS_LABELS[paper['classification']]} | {escape_cell(paper['task']['summary'])} |" + for paper in sorted(papers, key=lambda item: (-item["year"], item["system"])) + ] + sections.append( + f"## {DOMAIN_LABELS[domain]} ({len(papers)})\n\n" + "| System | Conference | Year | Evidence class | Task |\n" + "|---|---|---|---|---|\n" + "\n".join(rows) + ) + return f"""{GENERATED_NOTICE} + +[← Research views](README.md) · [Home](../README.md) + +# Papers by domain + +Domains describe the task or evidence area studied by each paper. +A paper may appear in more than one domain. + +{chr(10).join(sections)} +""" + + +def render_by_conference(catalog: dict) -> str: + grouped: dict[str, list[dict]] = defaultdict(list) + for paper in catalog["papers"]: + grouped[paper["conference"]].append(paper) + + sections = [] + for conference in CONFERENCE_ORDER: + papers = grouped.get(conference, []) + if not papers: + continue + rows = [ + f"| {view_paper_link(paper)} | {paper['venue']} | {paper['year']} | " + f"{paper['publication_status']} | {CLASS_LABELS[paper['classification']]} |" + for paper in sorted(papers, key=lambda item: (-item["year"], item["system"])) + ] + sections.append( + f"## {conference} ({len(papers)})\n\n" + "| System | Exact venue / track | Year | Status | Evidence class |\n" + "|---|---|---|---|---|\n" + "\n".join(rows) + ) + return f"""{GENERATED_NOTICE} + +[← Research views](README.md) · [Home](../README.md) + +# Papers by conference + +Conference is a standardized series label used for filtering. +Exact tracks and proceedings names remain in the venue field. + +{chr(10).join(sections)} +""" + + def render_fair_comparisons(catalog: dict) -> str: direct = [paper for paper in catalog["papers"] if paper["classification"] == "direct"] groups = [ @@ -503,13 +649,15 @@ def render_all() -> dict[Path, str]: catalog = load_catalog() catalog_json = json.dumps(catalog, indent=2, ensure_ascii=False) + "\n" outputs = { - README_PATH: render_marked_readme(README_PATH, catalog), - README_ZH_PATH: render_marked_readme(README_ZH_PATH, catalog), + README_PATH: render_marked_readme(README_PATH, catalog, "en"), + README_ZH_PATH: render_marked_readme(README_ZH_PATH, catalog, "zh"), ROOT / "data" / "papers.json": catalog_json, ROOT / "website" / "data" / "catalog.json": catalog_json, ROOT / "papers" / "README.md": render_paper_index(catalog), ROOT / "views" / "README.md": render_views_index(), ROOT / "views" / "by-product.md": render_by_product(catalog), + ROOT / "views" / "by-domain.md": render_by_domain(catalog), + ROOT / "views" / "by-conference.md": render_by_conference(catalog), ROOT / "views" / "by-method.md": render_by_method(catalog), ROOT / "views" / "fair-comparisons.md": render_fair_comparisons(catalog), ROOT / "views" / "by-venue.md": render_by_venue(catalog), diff --git a/tests/test_catalog.py b/tests/test_catalog.py index 5c98e5f..741d65f 100644 --- a/tests/test_catalog.py +++ b/tests/test_catalog.py @@ -5,6 +5,8 @@ from pathlib import Path from xml.etree import ElementTree +import yaml + from scripts import check_links from scripts.discover_candidates import parse_feed @@ -37,9 +39,22 @@ def test_readmes_are_concise_website_entry_points() -> None: text = (ROOT / filename).read_text(encoding="utf-8") assert website_url in text assert "CATALOG:DIRECT" not in text + assert "CATALOG:COVERAGE:START" in text + assert "views/by-domain.md" in text + assert "views/by-conference.md" in text assert len(text.splitlines()) < 100 +def test_every_paper_has_filterable_domain_and_conference() -> None: + catalog = yaml.safe_load((ROOT / "data" / "papers.yaml").read_text(encoding="utf-8")) + papers = catalog["papers"] + + assert len(papers) == 32 + assert sum(paper["year"] == 2026 for paper in papers) == 29 + assert all(paper["conference"] for paper in papers) + assert all(paper["domains"] for paper in papers) + + def test_link_scan_excludes_dependencies_and_build_outputs(tmp_path: Path, monkeypatch) -> None: (tmp_path / "README.md").write_text("# Project\n", encoding="utf-8") for directory in ("node_modules", "dist", ".next", ".wrangler"): diff --git a/views/README.md b/views/README.md index e0fb9d3..ea0b9e2 100644 --- a/views/README.md +++ b/views/README.md @@ -5,7 +5,9 @@ # Research views - [By product](by-product.md) — Claude Code and Codex CLI separately. +- [By domain](by-domain.md) — software engineering, security, systems, formal methods, and more. +- [By conference](by-conference.md) — standardized conference series plus arXiv-only papers. - [By method](by-method.md) — repository graphs, retrieval, verification, orchestration, and more. - [Comparison fairness](fair-comparisons.md) — same-model and same-budget controls. -- [By venue](by-venue.md) — conferences, workshops, and preprints. +- [By exact venue](by-venue.md) — tracks, workshops, and proceedings labels as reported. - [All paper dossiers](../papers/README.md) — one evidence page per paper. diff --git a/views/by-conference.md b/views/by-conference.md new file mode 100644 index 0000000..dd8fc3e --- /dev/null +++ b/views/by-conference.md @@ -0,0 +1,73 @@ + + +[← Research views](README.md) · [Home](../README.md) + +# Papers by conference + +Conference is a standardized series label used for filtering. +Exact tracks and proceedings names remain in the venue field. + +## ASE (1) + +| System | Exact venue / track | Year | Status | Evidence class | +|---|---|---|---|---| +| [LLM2Ltac](../papers/llm2ltac-2026.md) | ASE | 2026 | main | Direct comparison | +## FSE (1) + +| System | Exact venue / track | Year | Status | Evidence class | +|---|---|---|---|---| +| [Bug taxonomy](../papers/engineering-pitfalls-2026.md) | FSE Industry Track | 2026 | conference | Evaluation only | +## ICLR (4) + +| System | Exact venue / track | Year | Status | Evidence class | +|---|---|---|---|---| +| [ARTEMIS](../papers/artemis-2026.md) | ICLR | 2026 | main | Direct comparison | +| [QLCoder](../papers/qlcoder-2026.md) | ICLR | 2026 | main | Direct comparison | +| [Terminal-Bench 2.0](../papers/terminal-bench-2-2026.md) | ICLR | 2026 | main | Evaluation only | +| [ZeroRepo](../papers/rpg-zerorepo-2026.md) | ICLR | 2026 | main | Direct comparison | +## ICML (1) + +| System | Exact venue / track | Year | Status | Evidence class | +|---|---|---|---|---| +| [FormAct](../papers/formact-2026.md) | ICML | 2026 | main | Direct comparison | +## ICSE (2) + +| System | Exact venue / track | Year | Status | Evidence class | +|---|---|---|---|---| +| [AGENTS.md](../papers/agents-md-impact-2026.md) | ICSE JAWs | 2026 | workshop | Direct comparison | +| [TiCoder](../papers/ticoder-2022.md) | AST at ICSE | 2022 | workshop | Historical model | +## ISSTA (3) + +| System | Exact venue / track | Year | Status | Evidence class | +|---|---|---|---|---| +| [EvoDev](../papers/evodev-2026.md) | ISSTA | 2026 | main | Direct comparison | +| [Execution-policy ablation](../papers/execution-ablation-2026.md) | ISSTA | 2026 | main | Related method | +| [ToolLeak](../papers/toolleak-2026.md) | ISSTA | 2026 | main | Evaluation only | +## NeurIPS (2) + +| System | Exact venue / track | Year | Status | Evidence class | +|---|---|---|---|---| +| [BountyBench](../papers/bountybench-2025.md) | NeurIPS | 2025 | dataset-benchmark | Evaluation only | +| [MLZero](../papers/mlzero-2025.md) | NeurIPS | 2025 | main | Direct comparison | +## arXiv (18) + +| System | Exact venue / track | Year | Status | Evidence class | +|---|---|---|---|---| +| [AgentRadio](../papers/agentradio-2026.md) | arXiv | 2026 | preprint | Direct comparison | +| [Agentic Harness Engineering](../papers/agentic-harness-engineering-2026.md) | arXiv | 2026 | preprint | Direct comparison | +| [Aria](../papers/aria-2026.md) | arXiv | 2026 | preprint | Related method | +| [Brief](../papers/brief-product-context-2026.md) | arXiv | 2026 | preprint | Direct comparison | +| [Co-Coder](../papers/co-coder-2026.md) | arXiv | 2026 | preprint | Direct comparison | +| [CodeGrep](../papers/codegrep-2026.md) | arXiv | 2026 | preprint | Related method | +| [Context-file ablation](../papers/context-files-ablation-2026.md) | arXiv | 2026 | preprint | Related method | +| [Guardrails Beat Guidance](../papers/guardrails-guidance-2026.md) | arXiv | 2026 | preprint | Direct comparison | +| [Prefactory](../papers/prefactory-2026.md) | arXiv | 2026 | preprint | Direct comparison | +| [RepoOMP](../papers/repoomp-2026.md) | arXiv | 2026 | preprint | Direct comparison | +| [RustPrint](../papers/rustprint-2026.md) | arXiv | 2026 | preprint | Direct comparison | +| [SCATE](../papers/scate-2026.md) | arXiv | 2026 | preprint | Related method | +| [SEC-bench Pro](../papers/sec-bench-pro-2026.md) | arXiv | 2026 | preprint | Evaluation only | +| [SWE-Bench Mobile](../papers/swe-bench-mobile-2026.md) | arXiv | 2026 | preprint | Evaluation only | +| [SWE-bench Science](../papers/swe-bench-science-2026.md) | arXiv | 2026 | preprint | Evaluation only | +| [Volt with LCM](../papers/lcm-2026.md) | arXiv | 2026 | preprint | Direct comparison | +| [WebDesignIter](../papers/webdesigniter-2026.md) | arXiv | 2026 | preprint | Direct comparison | +| [icat-agent](../papers/icat-agent-2026.md) | arXiv | 2026 | preprint | Direct comparison | diff --git a/views/by-domain.md b/views/by-domain.md new file mode 100644 index 0000000..eac401e --- /dev/null +++ b/views/by-domain.md @@ -0,0 +1,81 @@ + + +[← Research views](README.md) · [Home](../README.md) + +# Papers by domain + +Domains describe the task or evidence area studied by each paper. +A paper may appear in more than one domain. + +## Software Engineering (21) + +| System | Conference | Year | Evidence class | Task | +|---|---|---|---|---| +| [AGENTS.md](../papers/agents-md-impact-2026.md) | ICSE | 2026 | Direct comparison | Real pull-request tasks with and without repository instructions | +| [AgentRadio](../papers/agentradio-2026.md) | arXiv | 2026 | Direct comparison | Long-horizon repository question answering | +| [Agentic Harness Engineering](../papers/agentic-harness-engineering-2026.md) | arXiv | 2026 | Direct comparison | Automatic evolution of coding-agent harnesses | +| [Brief](../papers/brief-product-context-2026.md) | arXiv | 2026 | Direct comparison | Product-decision-compliant feature implementation | +| [Bug taxonomy](../papers/engineering-pitfalls-2026.md) | FSE | 2026 | Evaluation only | Empirical analysis of product bug reports | +| [Co-Coder](../papers/co-coder-2026.md) | arXiv | 2026 | Direct comparison | Parallel multi-agent coding | +| [CodeGrep](../papers/codegrep-2026.md) | arXiv | 2026 | Related method | Repository retrieval for issue resolution | +| [Context-file ablation](../papers/context-files-ablation-2026.md) | arXiv | 2026 | Related method | Repository repair under different persistent-context strategies | +| [EvoDev](../papers/evodev-2026.md) | ISSTA | 2026 | Direct comparison | Iterative end-to-end Android application development | +| [Execution-policy ablation](../papers/execution-ablation-2026.md) | ISSTA | 2026 | Related method | Repository-level program repair with and without local code execution | +| [Guardrails Beat Guidance](../papers/guardrails-guidance-2026.md) | arXiv | 2026 | Direct comparison | Repository repair under persistent rule-file configurations | +| [Prefactory](../papers/prefactory-2026.md) | arXiv | 2026 | Direct comparison | Discovery and application of library-adoption refactorings | +| [RustPrint](../papers/rustprint-2026.md) | arXiv | 2026 | Direct comparison | Whole-codebase migration from C to Rust | +| [SCATE](../papers/scate-2026.md) | arXiv | 2026 | Related method | Automated test generation | +| [SWE-Bench Mobile](../papers/swe-bench-mobile-2026.md) | arXiv | 2026 | Evaluation only | Industry-scale iOS feature development from PRDs and Figma designs | +| [SWE-bench Science](../papers/swe-bench-science-2026.md) | arXiv | 2026 | Evaluation only | Repository repair in scientific software | +| [Terminal-Bench 2.0](../papers/terminal-bench-2-2026.md) | ICLR | 2026 | Evaluation only | General terminal-agent evaluation | +| [WebDesignIter](../papers/webdesigniter-2026.md) | arXiv | 2026 | Direct comparison | Repository-level front-end code generation | +| [ZeroRepo](../papers/rpg-zerorepo-2026.md) | ICLR | 2026 | Direct comparison | Repository-level codebase generation | +| [icat-agent](../papers/icat-agent-2026.md) | arXiv | 2026 | Direct comparison | Repository issue resolution | +| [TiCoder](../papers/ticoder-2022.md) | ICSE | 2022 | Historical model | Interactive function-level code generation | +## Security (5) + +| System | Conference | Year | Evidence class | Task | +|---|---|---|---|---| +| [ARTEMIS](../papers/artemis-2026.md) | ICLR | 2026 | Direct comparison | Long-horizon penetration testing in a live enterprise network | +| [QLCoder](../papers/qlcoder-2026.md) | ICLR | 2026 | Direct comparison | CodeQL query synthesis from CVE metadata | +| [SEC-bench Pro](../papers/sec-bench-pro-2026.md) | arXiv | 2026 | Evaluation only | Long-horizon proof-of-concept synthesis for disclosed vulnerabilities | +| [ToolLeak](../papers/toolleak-2026.md) | ISSTA | 2026 | Evaluation only | Prompt exfiltration and tool-invocation hijacking in production coding agents | +| [BountyBench](../papers/bountybench-2025.md) | NeurIPS | 2025 | Evaluation only | Vulnerability detection, exploitation, and patching | +## Systems & Performance (5) + +| System | Conference | Year | Evidence class | Task | +|---|---|---|---|---| +| [Execution-policy ablation](../papers/execution-ablation-2026.md) | ISSTA | 2026 | Related method | Repository-level program repair with and without local code execution | +| [RepoOMP](../papers/repoomp-2026.md) | arXiv | 2026 | Direct comparison | Repository-aware OpenMP hotspot parallelization | +| [RustPrint](../papers/rustprint-2026.md) | arXiv | 2026 | Direct comparison | Whole-codebase migration from C to Rust | +| [Terminal-Bench 2.0](../papers/terminal-bench-2-2026.md) | ICLR | 2026 | Evaluation only | General terminal-agent evaluation | +| [Volt with LCM](../papers/lcm-2026.md) | arXiv | 2026 | Direct comparison | Long-context reasoning and aggregation | +## Machine Learning (1) + +| System | Conference | Year | Evidence class | Task | +|---|---|---|---|---| +| [MLZero](../papers/mlzero-2025.md) | NeurIPS | 2025 | Direct comparison | End-to-end machine learning automation | +## Scientific Computing (2) + +| System | Conference | Year | Evidence class | Task | +|---|---|---|---|---| +| [RepoOMP](../papers/repoomp-2026.md) | arXiv | 2026 | Direct comparison | Repository-aware OpenMP hotspot parallelization | +| [SWE-bench Science](../papers/swe-bench-science-2026.md) | arXiv | 2026 | Evaluation only | Repository repair in scientific software | +## Formal Methods (2) + +| System | Conference | Year | Evidence class | Task | +|---|---|---|---|---| +| [Aria](../papers/aria-2026.md) | arXiv | 2026 | Related method | Fully automatic proof construction for verified software | +| [LLM2Ltac](../papers/llm2ltac-2026.md) | ASE | 2026 | Direct comparison | Automated theorem proving in Rocq projects | +## Web & UI (3) + +| System | Conference | Year | Evidence class | Task | +|---|---|---|---|---| +| [EvoDev](../papers/evodev-2026.md) | ISSTA | 2026 | Direct comparison | Iterative end-to-end Android application development | +| [SWE-Bench Mobile](../papers/swe-bench-mobile-2026.md) | arXiv | 2026 | Evaluation only | Industry-scale iOS feature development from PRDs and Figma designs | +| [WebDesignIter](../papers/webdesigniter-2026.md) | arXiv | 2026 | Direct comparison | Repository-level front-end code generation | +## Documents (1) + +| System | Conference | Year | Evidence class | Task | +|---|---|---|---|---| +| [FormAct](../papers/formact-2026.md) | ICML | 2026 | Direct comparison | Rich-format document generation and editing | diff --git a/views/by-method.md b/views/by-method.md index aa6b33d..186f2b8 100644 --- a/views/by-method.md +++ b/views/by-method.md @@ -6,137 +6,183 @@ Tags describe the intervention added around or instead of the production coding-agent baseline. -## `verifier-loop` (12) +## `verifier-loop` (16) | System | Venue | Method | |---|---|---| | [ARTEMIS](../papers/artemis-2026.md) | ICLR 2026 | Supervisor with dynamic expert agents, recursive task decomposition, parallel exploration, context management, triage, and reporting | | [Agentic Harness Engineering](../papers/agentic-harness-engineering-2026.md) | arXiv 2026 | Closed-loop harness evolution using component, trajectory, and decision observability with evidence-backed edits and rollback | +| [Aria](../papers/aria-2026.md) | arXiv 2026 | A reusable harness that lets the code agent choose whole-proof strategy while prover kernels enforce soundness, completeness, retries, and termination | +| [Execution-policy ablation](../papers/execution-ablation-2026.md) | ISSTA 2026 | Paired five-arm execution-policy ablation spanning prohibited, quota-limited, budget-guided, and unrestricted execution | | [FormAct](../papers/formact-2026.md) | ICML 2026 | HTML source editor, template retrieval, rendered-page review agent, iterative refinement, and edit-triggered context compression | +| [LLM2Ltac](../papers/llm2ltac-2026.md) | ASE 2026 | LLM-mined reusable symbolic tactics, validity and generalization checks, and integration into CoqHammer | | [Prefactory](../papers/prefactory-2026.md) | arXiv 2026 | LLM-synthesized lexical and structural search heuristics, deterministic candidate ranking, targeted refactoring, and differential tests | | [QLCoder](../papers/qlcoder-2026.md) | ICLR 2026 | CVE-grounded retrieval, AST guidance, CodeQL LSP tools, and iterative execution feedback | | [RepoOMP](../papers/repoomp-2026.md) | arXiv 2026 | Multi-granularity dependency graph, rule-or-LLM hotspot routing, reduced transformation context, and compile/workload/speedup validation | | [RustPrint](../papers/rustprint-2026.md) | arXiv 2026 | Architecture documentation blueprint, module and data-flow planning, compile feedback, documentation mismatch repair, and source-test translation | | [SCATE](../papers/scate-2026.md) | arXiv 2026 | Contextual-bandit supervisor that selects testing actions from current coverage and testability state | +| [SEC-bench Pro](../papers/sec-bench-pro-2026.md) | arXiv 2026 | Self-evolving reconstruction pipeline, three-image execution oracle, and LLM judge for vulnerability-specific PoC attribution | | [Terminal-Bench 2.0](../papers/terminal-bench-2-2026.md) | ICLR 2026 | Evaluation-only benchmark with stronger task verification and a neutral terminal scaffold | | [BountyBench](../papers/bountybench-2025.md) | NeurIPS 2025 | Evaluation-only benchmark of offensive and defensive cybersecurity agents | | [MLZero](../papers/mlzero-2025.md) | NeurIPS 2025 | Hierarchical agents with multimodal perception, semantic and episodic memory, and a planning-coding-evaluation loop | | [TiCoder](../papers/ticoder-2022.md) | AST at ICSE 2022 | Generate tests to formalize user intent, collect lightweight feedback, and rank or prune candidate programs | -## `test-feedback` (8) +## `test-feedback` (13) | System | Venue | Method | |---|---|---| +| [Aria](../papers/aria-2026.md) | arXiv 2026 | A reusable harness that lets the code agent choose whole-proof strategy while prover kernels enforce soundness, completeness, retries, and termination | +| [Brief](../papers/brief-product-context-2026.md) | arXiv 2026 | Retrieval over recorded product decisions, personas, customer signals, and competitive context during specification and implementation | +| [EvoDev](../papers/evodev-2026.md) | ISSTA 2026 | Feature-driven multi-agent development with a dependency DAG, layered business/design/code context, iterative implementation, and build repair | +| [Execution-policy ablation](../papers/execution-ablation-2026.md) | ISSTA 2026 | Paired five-arm execution-policy ablation spanning prohibited, quota-limited, budget-guided, and unrestricted execution | | [Prefactory](../papers/prefactory-2026.md) | arXiv 2026 | LLM-synthesized lexical and structural search heuristics, deterministic candidate ranking, targeted refactoring, and differential tests | | [RustPrint](../papers/rustprint-2026.md) | arXiv 2026 | Architecture documentation blueprint, module and data-flow planning, compile feedback, documentation mismatch repair, and source-test translation | | [SCATE](../papers/scate-2026.md) | arXiv 2026 | Contextual-bandit supervisor that selects testing actions from current coverage and testability state | +| [SWE-Bench Mobile](../papers/swe-bench-mobile-2026.md) | arXiv 2026 | Multimodal mobile benchmark with private tests, cross-agent model controls, cost analysis, prompt ablations, and error taxonomy | | [WebDesignIter](../papers/webdesigniter-2026.md) | arXiv 2026 | Persistent architecture and design knowledge graph, design-informed planning, targeted patches, and sandbox repair | | [ZeroRepo](../papers/rpg-zerorepo-2026.md) | ICLR 2026 | Persistent graph over features, functions, files, interfaces, and data flow with graph-guided generation and validation | | [icat-agent](../papers/icat-agent-2026.md) | arXiv 2026 | Event-based agent communication, issue-quality routing, parallel patching and validation, and exploratory fallback | | [MLZero](../papers/mlzero-2025.md) | NeurIPS 2025 | Hierarchical agents with multimodal perception, semantic and episodic memory, and a planning-coding-evaluation loop | | [TiCoder](../papers/ticoder-2022.md) | AST at ICSE 2022 | Generate tests to formalize user intent, collect lightweight feedback, and rank or prune candidate programs | -## `context-reduction` (4) +## `context-reduction` (6) | System | Venue | Method | |---|---|---| | [AGENTS.md](../papers/agents-md-impact-2026.md) | ICSE JAWs 2026 | Persistent repository-level build, test, style, and directory guidance in AGENTS.md | | [CodeGrep](../papers/codegrep-2026.md) | arXiv 2026 | GRPO-trained 14B retrieval agent that issues multi-turn parallel grep, glob, and read operations for a frozen downstream coding agent | +| [Execution-policy ablation](../papers/execution-ablation-2026.md) | ISSTA 2026 | Paired five-arm execution-policy ablation spanning prohibited, quota-limited, budget-guided, and unrestricted execution | | [FormAct](../papers/formact-2026.md) | ICML 2026 | HTML source editor, template retrieval, rendered-page review agent, iterative refinement, and edit-triggered context compression | | [RepoOMP](../papers/repoomp-2026.md) | arXiv 2026 | Multi-granularity dependency graph, rule-or-LLM hotspot routing, reduced transformation context, and compile/workload/speedup validation | -## `multi-agent` (4) +| [Volt with LCM](../papers/lcm-2026.md) | arXiv 2026 | Lossless message storage, hierarchical summary DAGs, deterministic recursive compression, and engine-managed parallel map operators | +## `multi-agent` (6) | System | Venue | Method | |---|---|---| | [ARTEMIS](../papers/artemis-2026.md) | ICLR 2026 | Supervisor with dynamic expert agents, recursive task decomposition, parallel exploration, context management, triage, and reporting | +| [AgentRadio](../papers/agentradio-2026.md) | arXiv 2026 | Four Claude Code agents coordinated through asynchronous threads, messages, passive mention watching, and a five-phase negotiation protocol | | [Co-Coder](../papers/co-coder-2026.md) | arXiv 2026 | Static dependency graph partitioning, structural hub isolation, community detection, and dependency-aware scheduling | +| [EvoDev](../papers/evodev-2026.md) | ISSTA 2026 | Feature-driven multi-agent development with a dependency DAG, layered business/design/code context, iterative implementation, and build repair | | [icat-agent](../papers/icat-agent-2026.md) | arXiv 2026 | Event-based agent communication, issue-quality routing, parallel patching and validation, and exploratory fallback | | [MLZero](../papers/mlzero-2025.md) | NeurIPS 2025 | Hierarchical agents with multimodal perception, semantic and episodic memory, and a planning-coding-evaluation loop | -## `parallelism` (4) +## `parallelism` (6) | System | Venue | Method | |---|---|---| | [ARTEMIS](../papers/artemis-2026.md) | ICLR 2026 | Supervisor with dynamic expert agents, recursive task decomposition, parallel exploration, context management, triage, and reporting | +| [AgentRadio](../papers/agentradio-2026.md) | arXiv 2026 | Four Claude Code agents coordinated through asynchronous threads, messages, passive mention watching, and a five-phase negotiation protocol | | [Co-Coder](../papers/co-coder-2026.md) | arXiv 2026 | Static dependency graph partitioning, structural hub isolation, community detection, and dependency-aware scheduling | | [CodeGrep](../papers/codegrep-2026.md) | arXiv 2026 | GRPO-trained 14B retrieval agent that issues multi-turn parallel grep, glob, and read operations for a frozen downstream coding agent | +| [Volt with LCM](../papers/lcm-2026.md) | arXiv 2026 | Lossless message storage, hierarchical summary DAGs, deterministic recursive compression, and engine-managed parallel map operators | | [icat-agent](../papers/icat-agent-2026.md) | arXiv 2026 | Event-based agent communication, issue-quality routing, parallel patching and validation, and exploratory fallback | -## `repository-graph` (4) - -| System | Venue | Method | -|---|---|---| -| [Co-Coder](../papers/co-coder-2026.md) | arXiv 2026 | Static dependency graph partitioning, structural hub isolation, community detection, and dependency-aware scheduling | -| [RepoOMP](../papers/repoomp-2026.md) | arXiv 2026 | Multi-granularity dependency graph, rule-or-LLM hotspot routing, reduced transformation context, and compile/workload/speedup validation | -| [WebDesignIter](../papers/webdesigniter-2026.md) | arXiv 2026 | Persistent architecture and design knowledge graph, design-informed planning, targeted patches, and sandbox repair | -| [ZeroRepo](../papers/rpg-zerorepo-2026.md) | ICLR 2026 | Persistent graph over features, functions, files, interfaces, and data flow with graph-guided generation and validation | -## `retrieval` (4) +## `retrieval` (6) | System | Venue | Method | |---|---|---| +| [Brief](../papers/brief-product-context-2026.md) | arXiv 2026 | Retrieval over recorded product decisions, personas, customer signals, and competitive context during specification and implementation | | [CodeGrep](../papers/codegrep-2026.md) | arXiv 2026 | GRPO-trained 14B retrieval agent that issues multi-turn parallel grep, glob, and read operations for a frozen downstream coding agent | +| [Context-file ablation](../papers/context-files-ablation-2026.md) | arXiv 2026 | Within-task comparison of no context, always-on AGENTS.md, and selectively retrieved repository wiki context with gold tests and equivalence testing | | [FormAct](../papers/formact-2026.md) | ICML 2026 | HTML source editor, template retrieval, rendered-page review agent, iterative refinement, and edit-triggered context compression | | [QLCoder](../papers/qlcoder-2026.md) | ICLR 2026 | CVE-grounded retrieval, AST guidance, CodeQL LSP tools, and iterative execution feedback | | [WebDesignIter](../papers/webdesigniter-2026.md) | arXiv 2026 | Persistent architecture and design knowledge graph, design-informed planning, targeted patches, and sandbox repair | -## `structured-state` (4) +## `structured-state` (6) | System | Venue | Method | |---|---|---| | [AGENTS.md](../papers/agents-md-impact-2026.md) | ICSE JAWs 2026 | Persistent repository-level build, test, style, and directory guidance in AGENTS.md | +| [Brief](../papers/brief-product-context-2026.md) | arXiv 2026 | Retrieval over recorded product decisions, personas, customer signals, and competitive context during specification and implementation | +| [EvoDev](../papers/evodev-2026.md) | ISSTA 2026 | Feature-driven multi-agent development with a dependency DAG, layered business/design/code context, iterative implementation, and build repair | | [RustPrint](../papers/rustprint-2026.md) | arXiv 2026 | Architecture documentation blueprint, module and data-flow planning, compile feedback, documentation mismatch repair, and source-test translation | | [WebDesignIter](../papers/webdesigniter-2026.md) | arXiv 2026 | Persistent architecture and design knowledge graph, design-informed planning, targeted patches, and sandbox repair | | [ZeroRepo](../papers/rpg-zerorepo-2026.md) | ICLR 2026 | Persistent graph over features, functions, files, interfaces, and data flow with graph-guided generation and validation | -## `dependency-aware-planning` (3) +## `memory` (5) + +| System | Venue | Method | +|---|---|---| +| [ARTEMIS](../papers/artemis-2026.md) | ICLR 2026 | Supervisor with dynamic expert agents, recursive task decomposition, parallel exploration, context management, triage, and reporting | +| [Agentic Harness Engineering](../papers/agentic-harness-engineering-2026.md) | arXiv 2026 | Closed-loop harness evolution using component, trajectory, and decision observability with evidence-backed edits and rollback | +| [EvoDev](../papers/evodev-2026.md) | ISSTA 2026 | Feature-driven multi-agent development with a dependency DAG, layered business/design/code context, iterative implementation, and build repair | +| [Volt with LCM](../papers/lcm-2026.md) | arXiv 2026 | Lossless message storage, hierarchical summary DAGs, deterministic recursive compression, and engine-managed parallel map operators | +| [MLZero](../papers/mlzero-2025.md) | NeurIPS 2025 | Hierarchical agents with multimodal perception, semantic and episodic memory, and a planning-coding-evaluation loop | +## `dependency-aware-planning` (4) | System | Venue | Method | |---|---|---| | [Co-Coder](../papers/co-coder-2026.md) | arXiv 2026 | Static dependency graph partitioning, structural hub isolation, community detection, and dependency-aware scheduling | +| [EvoDev](../papers/evodev-2026.md) | ISSTA 2026 | Feature-driven multi-agent development with a dependency DAG, layered business/design/code context, iterative implementation, and build repair | | [RustPrint](../papers/rustprint-2026.md) | arXiv 2026 | Architecture documentation blueprint, module and data-flow planning, compile feedback, documentation mismatch repair, and source-test translation | | [ZeroRepo](../papers/rpg-zerorepo-2026.md) | ICLR 2026 | Persistent graph over features, functions, files, interfaces, and data flow with graph-guided generation and validation | -## `memory` (3) +## `observability` (4) | System | Venue | Method | |---|---|---| -| [ARTEMIS](../papers/artemis-2026.md) | ICLR 2026 | Supervisor with dynamic expert agents, recursive task decomposition, parallel exploration, context management, triage, and reporting | | [Agentic Harness Engineering](../papers/agentic-harness-engineering-2026.md) | arXiv 2026 | Closed-loop harness evolution using component, trajectory, and decision observability with evidence-backed edits and rollback | -| [MLZero](../papers/mlzero-2025.md) | NeurIPS 2025 | Hierarchical agents with multimodal perception, semantic and episodic memory, and a planning-coding-evaluation loop | -## `deterministic-search` (2) +| [Bug taxonomy](../papers/engineering-pitfalls-2026.md) | FSE Industry Track 2026 | Evaluation-only manual taxonomy of functionality, integration, invocation, and command-execution failures | +| [SWE-bench Science](../papers/swe-bench-science-2026.md) | arXiv 2026 | Three-paradigm benchmark with private tests, a scientific failure taxonomy, and paired ablations of explicit scientific guidance | +| [ToolLeak](../papers/toolleak-2026.md) | ISSTA 2026 | Tool-call schema pressure for prompt leakage followed by a two-channel injection through tool descriptions and returns | +## `repository-graph` (4) | System | Venue | Method | |---|---|---| -| [Prefactory](../papers/prefactory-2026.md) | arXiv 2026 | LLM-synthesized lexical and structural search heuristics, deterministic candidate ranking, targeted refactoring, and differential tests | +| [Co-Coder](../papers/co-coder-2026.md) | arXiv 2026 | Static dependency graph partitioning, structural hub isolation, community detection, and dependency-aware scheduling | | [RepoOMP](../papers/repoomp-2026.md) | arXiv 2026 | Multi-granularity dependency graph, rule-or-LLM hotspot routing, reduced transformation context, and compile/workload/speedup validation | -## `dynamic-specialization` (2) +| [WebDesignIter](../papers/webdesigniter-2026.md) | arXiv 2026 | Persistent architecture and design knowledge graph, design-informed planning, targeted patches, and sandbox repair | +| [ZeroRepo](../papers/rpg-zerorepo-2026.md) | ICLR 2026 | Persistent graph over features, functions, files, interfaces, and data flow with graph-guided generation and validation | +## `benchmark-design` (3) | System | Venue | Method | |---|---|---| -| [ARTEMIS](../papers/artemis-2026.md) | ICLR 2026 | Supervisor with dynamic expert agents, recursive task decomposition, parallel exploration, context management, triage, and reporting | -| [icat-agent](../papers/icat-agent-2026.md) | arXiv 2026 | Event-based agent communication, issue-quality routing, parallel patching and validation, and exploratory fallback | -## `observability` (2) +| [SEC-bench Pro](../papers/sec-bench-pro-2026.md) | arXiv 2026 | Self-evolving reconstruction pipeline, three-image execution oracle, and LLM judge for vulnerability-specific PoC attribution | +| [SWE-Bench Mobile](../papers/swe-bench-mobile-2026.md) | arXiv 2026 | Multimodal mobile benchmark with private tests, cross-agent model controls, cost analysis, prompt ablations, and error taxonomy | +| [SWE-bench Science](../papers/swe-bench-science-2026.md) | arXiv 2026 | Three-paradigm benchmark with private tests, a scientific failure taxonomy, and paired ablations of explicit scientific guidance | +## `deterministic-search` (3) | System | Venue | Method | |---|---|---| -| [Agentic Harness Engineering](../papers/agentic-harness-engineering-2026.md) | arXiv 2026 | Closed-loop harness evolution using component, trajectory, and decision observability with evidence-backed edits and rollback | -| [Bug taxonomy](../papers/engineering-pitfalls-2026.md) | FSE Industry Track 2026 | Evaluation-only manual taxonomy of functionality, integration, invocation, and command-execution failures | -## `static-analysis` (2) +| [LLM2Ltac](../papers/llm2ltac-2026.md) | ASE 2026 | LLM-mined reusable symbolic tactics, validity and generalization checks, and integration into CoqHammer | +| [Prefactory](../papers/prefactory-2026.md) | arXiv 2026 | LLM-synthesized lexical and structural search heuristics, deterministic candidate ranking, targeted refactoring, and differential tests | +| [RepoOMP](../papers/repoomp-2026.md) | arXiv 2026 | Multi-granularity dependency graph, rule-or-LLM hotspot routing, reduced transformation context, and compile/workload/speedup validation | +## `repository-instructions` (3) | System | Venue | Method | |---|---|---| +| [AGENTS.md](../papers/agents-md-impact-2026.md) | ICSE JAWs 2026 | Persistent repository-level build, test, style, and directory guidance in AGENTS.md | +| [Context-file ablation](../papers/context-files-ablation-2026.md) | arXiv 2026 | Within-task comparison of no context, always-on AGENTS.md, and selectively retrieved repository wiki context with gold tests and equivalence testing | +| [Guardrails Beat Guidance](../papers/guardrails-guidance-2026.md) | arXiv 2026 | Paired rule-file experiments over source, count, polarity, type, and composition, backed by a corpus of 25,532 rules | +## `static-analysis` (3) + +| System | Venue | Method | +|---|---|---| +| [LLM2Ltac](../papers/llm2ltac-2026.md) | ASE 2026 | LLM-mined reusable symbolic tactics, validity and generalization checks, and integration into CoqHammer | | [Prefactory](../papers/prefactory-2026.md) | arXiv 2026 | LLM-synthesized lexical and structural search heuristics, deterministic candidate ranking, targeted refactoring, and differential tests | | [QLCoder](../papers/qlcoder-2026.md) | ICLR 2026 | CVE-grounded retrieval, AST guidance, CodeQL LSP tools, and iterative execution feedback | -## `harness-evolution` (1) +## `dynamic-specialization` (2) | System | Venue | Method | |---|---|---| -| [Agentic Harness Engineering](../papers/agentic-harness-engineering-2026.md) | arXiv 2026 | Closed-loop harness evolution using component, trajectory, and decision observability with evidence-backed edits and rollback | -## `performance-feedback` (1) +| [ARTEMIS](../papers/artemis-2026.md) | ICLR 2026 | Supervisor with dynamic expert agents, recursive task decomposition, parallel exploration, context management, triage, and reporting | +| [icat-agent](../papers/icat-agent-2026.md) | arXiv 2026 | Event-based agent communication, issue-quality routing, parallel patching and validation, and exploratory fallback | +## `visual-review` (2) | System | Venue | Method | |---|---|---| -| [RepoOMP](../papers/repoomp-2026.md) | arXiv 2026 | Multi-granularity dependency graph, rule-or-LLM hotspot routing, reduced transformation context, and compile/workload/speedup validation | -## `repository-instructions` (1) +| [FormAct](../papers/formact-2026.md) | ICML 2026 | HTML source editor, template retrieval, rendered-page review agent, iterative refinement, and edit-triggered context compression | +| [SWE-Bench Mobile](../papers/swe-bench-mobile-2026.md) | arXiv 2026 | Multimodal mobile benchmark with private tests, cross-agent model controls, cost analysis, prompt ablations, and error taxonomy | +## `adversarial-testing` (1) | System | Venue | Method | |---|---|---| -| [AGENTS.md](../papers/agents-md-impact-2026.md) | ICSE JAWs 2026 | Persistent repository-level build, test, style, and directory guidance in AGENTS.md | -## `visual-review` (1) +| [ToolLeak](../papers/toolleak-2026.md) | ISSTA 2026 | Tool-call schema pressure for prompt leakage followed by a two-channel injection through tool descriptions and returns | +## `coordination` (1) | System | Venue | Method | |---|---|---| -| [FormAct](../papers/formact-2026.md) | ICML 2026 | HTML source editor, template retrieval, rendered-page review agent, iterative refinement, and edit-triggered context compression | +| [AgentRadio](../papers/agentradio-2026.md) | arXiv 2026 | Four Claude Code agents coordinated through asynchronous threads, messages, passive mention watching, and a five-phase negotiation protocol | +## `harness-evolution` (1) + +| System | Venue | Method | +|---|---|---| +| [Agentic Harness Engineering](../papers/agentic-harness-engineering-2026.md) | arXiv 2026 | Closed-loop harness evolution using component, trajectory, and decision observability with evidence-backed edits and rollback | +## `performance-feedback` (1) + +| System | Venue | Method | +|---|---|---| +| [RepoOMP](../papers/repoomp-2026.md) | arXiv 2026 | Multi-granularity dependency graph, rule-or-LLM hotspot routing, reduced transformation context, and compile/workload/speedup validation | diff --git a/views/by-product.md b/views/by-product.md index 851a72e..98f459a 100644 --- a/views/by-product.md +++ b/views/by-product.md @@ -7,25 +7,38 @@ Product means the complete agent or harness evaluated by the paper, not merely a model from the same vendor. -## Claude Code (14) +## Claude Code (27) | System | Venue | Evidence class | Task | Strength | |---|---|---|---|---| | [AGENTS.md](../papers/agents-md-impact-2026.md) | ICSE JAWs 2026 | Direct comparison | Real pull-request tasks with and without repository instructions | high | | [ARTEMIS](../papers/artemis-2026.md) | ICLR 2026 | Direct comparison | Long-horizon penetration testing in a live enterprise network | medium | +| [AgentRadio](../papers/agentradio-2026.md) | arXiv 2026 | Direct comparison | Long-horizon repository question answering | high | +| [Aria](../papers/aria-2026.md) | arXiv 2026 | Related method | Fully automatic proof construction for verified software | high | +| [Brief](../papers/brief-product-context-2026.md) | arXiv 2026 | Direct comparison | Product-decision-compliant feature implementation | high | | [Bug taxonomy](../papers/engineering-pitfalls-2026.md) | FSE Industry Track 2026 | Evaluation only | Empirical analysis of product bug reports | high | | [Co-Coder](../papers/co-coder-2026.md) | arXiv 2026 | Direct comparison | Parallel multi-agent coding | medium | | [CodeGrep](../papers/codegrep-2026.md) | arXiv 2026 | Related method | Repository retrieval for issue resolution | contextual | +| [Context-file ablation](../papers/context-files-ablation-2026.md) | arXiv 2026 | Related method | Repository repair under different persistent-context strategies | high | +| [EvoDev](../papers/evodev-2026.md) | ISSTA 2026 | Direct comparison | Iterative end-to-end Android application development | high | +| [Execution-policy ablation](../papers/execution-ablation-2026.md) | ISSTA 2026 | Related method | Repository-level program repair with and without local code execution | high | +| [Guardrails Beat Guidance](../papers/guardrails-guidance-2026.md) | arXiv 2026 | Direct comparison | Repository repair under persistent rule-file configurations | high | +| [LLM2Ltac](../papers/llm2ltac-2026.md) | ASE 2026 | Direct comparison | Automated theorem proving in Rocq projects | high | | [QLCoder](../papers/qlcoder-2026.md) | ICLR 2026 | Direct comparison | CodeQL query synthesis from CVE metadata | high | | [RepoOMP](../papers/repoomp-2026.md) | arXiv 2026 | Direct comparison | Repository-aware OpenMP hotspot parallelization | medium | | [RustPrint](../papers/rustprint-2026.md) | arXiv 2026 | Direct comparison | Whole-codebase migration from C to Rust | medium | | [SCATE](../papers/scate-2026.md) | arXiv 2026 | Related method | Automated test generation | contextual | +| [SEC-bench Pro](../papers/sec-bench-pro-2026.md) | arXiv 2026 | Evaluation only | Long-horizon proof-of-concept synthesis for disclosed vulnerabilities | high | +| [SWE-Bench Mobile](../papers/swe-bench-mobile-2026.md) | arXiv 2026 | Evaluation only | Industry-scale iOS feature development from PRDs and Figma designs | high | +| [SWE-bench Science](../papers/swe-bench-science-2026.md) | arXiv 2026 | Evaluation only | Repository repair in scientific software | high | | [Terminal-Bench 2.0](../papers/terminal-bench-2-2026.md) | ICLR 2026 | Evaluation only | General terminal-agent evaluation | high | +| [ToolLeak](../papers/toolleak-2026.md) | ISSTA 2026 | Evaluation only | Prompt exfiltration and tool-invocation hijacking in production coding agents | high | +| [Volt with LCM](../papers/lcm-2026.md) | arXiv 2026 | Direct comparison | Long-context reasoning and aggregation | high | | [WebDesignIter](../papers/webdesigniter-2026.md) | arXiv 2026 | Direct comparison | Repository-level front-end code generation | medium | | [ZeroRepo](../papers/rpg-zerorepo-2026.md) | ICLR 2026 | Direct comparison | Repository-level codebase generation | high | | [icat-agent](../papers/icat-agent-2026.md) | arXiv 2026 | Direct comparison | Repository issue resolution | medium | | [BountyBench](../papers/bountybench-2025.md) | NeurIPS 2025 | Evaluation only | Vulnerability detection, exploitation, and patching | high | -## Codex CLI (11) +## Codex CLI (16) | System | Venue | Evidence class | Task | Strength | |---|---|---|---|---| @@ -33,8 +46,13 @@ not merely a model from the same vendor. | [ARTEMIS](../papers/artemis-2026.md) | ICLR 2026 | Direct comparison | Long-horizon penetration testing in a live enterprise network | medium | | [Agentic Harness Engineering](../papers/agentic-harness-engineering-2026.md) | arXiv 2026 | Direct comparison | Automatic evolution of coding-agent harnesses | medium | | [Bug taxonomy](../papers/engineering-pitfalls-2026.md) | FSE Industry Track 2026 | Evaluation only | Empirical analysis of product bug reports | high | +| [Context-file ablation](../papers/context-files-ablation-2026.md) | arXiv 2026 | Related method | Repository repair under different persistent-context strategies | high | +| [Execution-policy ablation](../papers/execution-ablation-2026.md) | ISSTA 2026 | Related method | Repository-level program repair with and without local code execution | high | | [FormAct](../papers/formact-2026.md) | ICML 2026 | Direct comparison | Rich-format document generation and editing | high | | [Prefactory](../papers/prefactory-2026.md) | arXiv 2026 | Direct comparison | Discovery and application of library-adoption refactorings | medium | +| [SEC-bench Pro](../papers/sec-bench-pro-2026.md) | arXiv 2026 | Evaluation only | Long-horizon proof-of-concept synthesis for disclosed vulnerabilities | high | +| [SWE-Bench Mobile](../papers/swe-bench-mobile-2026.md) | arXiv 2026 | Evaluation only | Industry-scale iOS feature development from PRDs and Figma designs | high | +| [SWE-bench Science](../papers/swe-bench-science-2026.md) | arXiv 2026 | Evaluation only | Repository repair in scientific software | high | | [Terminal-Bench 2.0](../papers/terminal-bench-2-2026.md) | ICLR 2026 | Evaluation only | General terminal-agent evaluation | high | | [WebDesignIter](../papers/webdesigniter-2026.md) | arXiv 2026 | Direct comparison | Repository-level front-end code generation | medium | | [ZeroRepo](../papers/rpg-zerorepo-2026.md) | ICLR 2026 | Direct comparison | Repository-level codebase generation | high | diff --git a/views/by-venue.md b/views/by-venue.md index c3a0e3a..7dc854f 100644 --- a/views/by-venue.md +++ b/views/by-venue.md @@ -6,17 +6,26 @@ Main-conference, workshop, benchmark-track, and preprint status remain explicit. -## arXiv (9) +## arXiv (18) | System | Year | Status | Evidence class | |---|---|---|---| +| [AgentRadio](../papers/agentradio-2026.md) | 2026 | preprint | Direct comparison | | [Agentic Harness Engineering](../papers/agentic-harness-engineering-2026.md) | 2026 | preprint | Direct comparison | +| [Aria](../papers/aria-2026.md) | 2026 | preprint | Related method | +| [Brief](../papers/brief-product-context-2026.md) | 2026 | preprint | Direct comparison | | [Co-Coder](../papers/co-coder-2026.md) | 2026 | preprint | Direct comparison | | [CodeGrep](../papers/codegrep-2026.md) | 2026 | preprint | Related method | +| [Context-file ablation](../papers/context-files-ablation-2026.md) | 2026 | preprint | Related method | +| [Guardrails Beat Guidance](../papers/guardrails-guidance-2026.md) | 2026 | preprint | Direct comparison | | [Prefactory](../papers/prefactory-2026.md) | 2026 | preprint | Direct comparison | | [RepoOMP](../papers/repoomp-2026.md) | 2026 | preprint | Direct comparison | | [RustPrint](../papers/rustprint-2026.md) | 2026 | preprint | Direct comparison | | [SCATE](../papers/scate-2026.md) | 2026 | preprint | Related method | +| [SEC-bench Pro](../papers/sec-bench-pro-2026.md) | 2026 | preprint | Evaluation only | +| [SWE-Bench Mobile](../papers/swe-bench-mobile-2026.md) | 2026 | preprint | Evaluation only | +| [SWE-bench Science](../papers/swe-bench-science-2026.md) | 2026 | preprint | Evaluation only | +| [Volt with LCM](../papers/lcm-2026.md) | 2026 | preprint | Direct comparison | | [WebDesignIter](../papers/webdesigniter-2026.md) | 2026 | preprint | Direct comparison | | [icat-agent](../papers/icat-agent-2026.md) | 2026 | preprint | Direct comparison | ## ICLR (4) @@ -27,12 +36,24 @@ Main-conference, workshop, benchmark-track, and preprint status remain explicit. | [QLCoder](../papers/qlcoder-2026.md) | 2026 | main | Direct comparison | | [Terminal-Bench 2.0](../papers/terminal-bench-2-2026.md) | 2026 | main | Evaluation only | | [ZeroRepo](../papers/rpg-zerorepo-2026.md) | 2026 | main | Direct comparison | +## ISSTA (3) + +| System | Year | Status | Evidence class | +|---|---|---|---| +| [EvoDev](../papers/evodev-2026.md) | 2026 | main | Direct comparison | +| [Execution-policy ablation](../papers/execution-ablation-2026.md) | 2026 | main | Related method | +| [ToolLeak](../papers/toolleak-2026.md) | 2026 | main | Evaluation only | ## NeurIPS (2) | System | Year | Status | Evidence class | |---|---|---|---| | [BountyBench](../papers/bountybench-2025.md) | 2025 | dataset-benchmark | Evaluation only | | [MLZero](../papers/mlzero-2025.md) | 2025 | main | Direct comparison | +## ASE (1) + +| System | Year | Status | Evidence class | +|---|---|---|---| +| [LLM2Ltac](../papers/llm2ltac-2026.md) | 2026 | main | Direct comparison | ## AST at ICSE (1) | System | Year | Status | Evidence class | diff --git a/views/fair-comparisons.md b/views/fair-comparisons.md index 6538642..43d7365 100644 --- a/views/fair-comparisons.md +++ b/views/fair-comparisons.md @@ -12,23 +12,29 @@ It does not mean the model, budget, tools, or product policy were held fixed. | System | Same model | Same budget | Strength | Caveat | |---|---|---|---|---| | [AGENTS.md](../papers/agents-md-impact-2026.md) | yes | yes | high | Workshop study and configuration intervention rather than a standalone agent. | -## Known model or budget mismatch (3) +## Known model or budget mismatch (6) | System | Same model | Same budget | Strength | Caveat | |---|---|---|---|---| | [ARTEMIS](../papers/artemis-2026.md) | unknown | no | medium | Runtime limits, model combinations, safety refusals, and product policies differ across systems. | +| [AgentRadio](../papers/agentradio-2026.md) | yes | no | high | The four-agent method costs about six times a single Opus 4.6 run, although the paper includes a near-cost resampling baseline. | +| [Brief](../papers/brief-product-context-2026.md) | no | no | high | The augmented arm adds Opus planning, a longer timeout, generated specifications, tests, and retrieval, so the 49-point gain is not attributable to product context alone. | +| [EvoDev](../papers/evodev-2026.md) | yes | no | high | APPDev contains 15 Android apps and uses manual quality scoring; EvoDev takes about twice Claude Code's wall-clock time in the Claude 4 Sonnet comparison. | | [RustPrint](../papers/rustprint-2026.md) | no | unknown | medium | The comparison mixes model and scaffold choices and is specific to C-to-Rust migration. | | [ZeroRepo](../papers/rpg-zerorepo-2026.md) | no | unknown | high | ZeroRepo uses o3-mini while the Claude Code baseline uses Claude 4 Sonnet, so model and harness effects are mixed. | -## Control parity not fully reported (9) +## Control parity not fully reported (12) | System | Same model | Same budget | Strength | Caveat | |---|---|---|---|---| | [Agentic Harness Engineering](../papers/agentic-harness-engineering-2026.md) | unknown | unknown | medium | Preprint; model, harness, and evaluation-budget effects require careful separation. | | [Co-Coder](../papers/co-coder-2026.md) | unknown | unknown | medium | Small 28-task evaluation and preprint status. | | [FormAct](../papers/formact-2026.md) | unknown | unknown | high | Codex has slightly higher content alignment, and the task is document formatting rather than source-code repair. | +| [Guardrails Beat Guidance](../papers/guardrails-guidance-2026.md) | yes | unknown | high | Results are limited to Claude Code with Opus 4.6 on a discriminative Python subset; several headline pairwise differences do not reach conventional significance. | +| [LLM2Ltac](../papers/llm2ltac-2026.md) | unknown | unknown | high | The public conference record does not report the Claude Code version, backbone model, absolute theorem counts for the product ablation, or budget parity. | | [Prefactory](../papers/prefactory-2026.md) | unknown | unknown | medium | The strongest comparison concerns candidate detection, not universal end-to-end task success. | | [QLCoder](../papers/qlcoder-2026.md) | unknown | unknown | high | Security-query synthesis is narrower than general software engineering. | | [RepoOMP](../papers/repoomp-2026.md) | unknown | unknown | medium | Domain-specific high-performance-computing task and preprint status. | +| [Volt with LCM](../papers/lcm-2026.md) | yes | unknown | high | OOLONG may be contaminated in Opus 4.6, and the paper does not provide a matched end-to-end cost comparison for LCM's auxiliary calls. | | [WebDesignIter](../papers/webdesigniter-2026.md) | unknown | unknown | medium | Front-end-specific benchmark and preprint status. | | [icat-agent](../papers/icat-agent-2026.md) | unknown | unknown | medium | Exact model parity varies by reported configuration; preprint. | | [MLZero](../papers/mlzero-2025.md) | unknown | unknown | medium | Results concern machine-learning automation and should not be generalized to all software-engineering tasks. | diff --git a/website/app/CatalogExplorer.tsx b/website/app/CatalogExplorer.tsx index 5a066bd..755f10d 100644 --- a/website/app/CatalogExplorer.tsx +++ b/website/app/CatalogExplorer.tsx @@ -4,13 +4,35 @@ type ProductId = "claude-code" | "codex-cli" | "openai-codex-model"; type Classification = "direct" | "related" | "evaluation" | "historical"; type Language = "en" | "zh"; type FairnessFilter = "all" | "controlled" | "unknown" | "mismatch"; +type ConferenceId = + | "AAAI" + | "ASE" + | "FSE" + | "ICLR" + | "ICML" + | "ICSE" + | "ISSTA" + | "NeurIPS" + | "arXiv" + | "Other"; +type DomainId = + | "software-engineering" + | "security" + | "systems-performance" + | "machine-learning" + | "scientific-computing" + | "formal-methods" + | "web-ui" + | "documents"; export type Paper = { id: string; title: string; authors: string[]; year: number; + conference: ConferenceId; venue: string; + domains: DomainId[]; publication_status: string; classification: Classification; system: string; @@ -49,9 +71,48 @@ const PRODUCT_LABELS: Record = { "openai-codex-model": "Codex model", }; +const DOMAIN_LABELS: Record> = { + en: { + "software-engineering": "Software Engineering", + security: "Security", + "systems-performance": "Systems & Performance", + "machine-learning": "Machine Learning", + "scientific-computing": "Scientific Computing", + "formal-methods": "Formal Methods", + "web-ui": "Web & UI", + documents: "Documents", + }, + zh: { + "software-engineering": "软件工程", + security: "安全", + "systems-performance": "系统与性能", + "machine-learning": "机器学习", + "scientific-computing": "科学计算", + "formal-methods": "形式化方法", + "web-ui": "Web 与 UI", + documents: "文档", + }, +}; + +const CONFERENCE_ORDER: ConferenceId[] = [ + "AAAI", + "ASE", + "FSE", + "ICLR", + "ICML", + "ICSE", + "ISSTA", + "NeurIPS", + "arXiv", + "Other", +]; + +const DOMAIN_ORDER = Object.keys(DOMAIN_LABELS.en) as DomainId[]; + const copy = { en: { papers: "Papers", + coverage: "Coverage", methods: "Patterns", fairness: "Fairness", star: "Star on GitHub ↗", @@ -65,14 +126,21 @@ const copy = { reviewedPapers: "reviewed papers", directComparisons: "direct comparisons", officialArtifacts: "official artifacts", - productionAgents: "production agents", + researchDomains: "research domains", + coverageEyebrow: "What the catalog covers", + coverageTitle: "Choose a domain or conference before opening a paper.", + coverageDeck: + "Every entry uses a standardized domain and conference label, while preserving the exact venue in its evidence record.", + domains: "Research domains", + conferences: "Conferences / sources", catalogEyebrow: "Research catalog", catalogTitle: "Start with the result. Open the controls only when needed.", search: "Search systems, tasks, methods…", product: "Product", evidenceClass: "Evidence class", comparison: "Comparison", - venue: "Venue", + conference: "Conference / source", + exactVenue: "Exact venue", method: "Method", all: "All", direct: "Direct", @@ -124,6 +192,7 @@ const copy = { }, zh: { papers: "论文", + coverage: "收录范围", methods: "方法", fairness: "公平性", star: "去 GitHub 点 Star ↗", @@ -137,14 +206,21 @@ const copy = { reviewedPapers: "篇已审论文", directComparisons: "篇直接对比", officialArtifacts: "个官方 Artifact", - productionAgents: "个工业 Agent", + researchDomains: "个研究领域", + coverageEyebrow: "目录收录什么", + coverageTitle: "先选研究领域或会议，再打开论文。", + coverageDeck: + "每篇论文都使用统一的领域与会议标签，同时在证据记录中保留准确的 venue / track。", + domains: "研究领域", + conferences: "会议 / 来源", catalogEyebrow: "论文目录", catalogTitle: "先看结果；需要时，再展开实验控制和限制。", search: "搜索系统、任务或方法…", product: "产品", evidenceClass: "证据类型", comparison: "实验控制", - venue: "会议", + conference: "会议 / 来源", + exactVenue: "准确 venue / track", method: "方法", all: "全部", direct: "直接对比", @@ -241,7 +317,8 @@ export function CatalogExplorer({ papers, reviewedAt }: Props) { "all" | Classification >("all"); const [fairness, setFairness] = useState("all"); - const [venue, setVenue] = useState("all"); + const [conference, setConference] = useState<"all" | ConferenceId>("all"); + const [domain, setDomain] = useState<"all" | DomainId>("all"); const [method, setMethod] = useState("all"); const [activePaper, setActivePaper] = useState(null); const dialogCloseButton = useRef(null); @@ -254,10 +331,29 @@ export function CatalogExplorer({ papers, reviewedAt }: Props) { (paper) => paper.artifact_status === "official", ).length; - const venues = useMemo( - () => [...new Set(papers.map((paper) => paper.venue))].sort(), - [papers], - ); + const conferenceCounts = useMemo(() => { + const counts = new Map(); + papers.forEach((paper) => { + counts.set(paper.conference, (counts.get(paper.conference) ?? 0) + 1); + }); + return CONFERENCE_ORDER.flatMap((item) => { + const count = counts.get(item); + return count ? ([[item, count]] as Array<[ConferenceId, number]>) : []; + }); + }, [papers]); + + const domainCounts = useMemo(() => { + const counts = new Map(); + papers.forEach((paper) => { + paper.domains.forEach((item) => { + counts.set(item, (counts.get(item) ?? 0) + 1); + }); + }); + return DOMAIN_ORDER.flatMap((item) => { + const count = counts.get(item); + return count ? ([[item, count]] as Array<[DomainId, number]>) : []; + }); + }, [papers]); const methodCounts = useMemo(() => { const counts = new Map(); @@ -299,7 +395,10 @@ export function CatalogExplorer({ papers, reviewedAt }: Props) { [ paper.system, paper.title, + paper.conference, paper.venue, + ...paper.domains, + ...paper.domains.map((item) => DOMAIN_LABELS[language][item]), paper.task.summary, paper.task.benchmark, paper.method.summary, @@ -317,7 +416,9 @@ export function CatalogExplorer({ papers, reviewedAt }: Props) { classification === "all" || paper.classification === classification; const matchesFairness = fairness === "all" || fairnessGroup(paper) === fairness; - const matchesVenue = venue === "all" || paper.venue === venue; + const matchesConference = + conference === "all" || paper.conference === conference; + const matchesDomain = domain === "all" || paper.domains.includes(domain); const matchesMethod = method === "all" || paper.method.tags.includes(method); return ( @@ -325,7 +426,8 @@ export function CatalogExplorer({ papers, reviewedAt }: Props) { matchesProduct && matchesClass && matchesFairness && - matchesVenue && + matchesConference && + matchesDomain && matchesMethod ); }) @@ -335,14 +437,15 @@ export function CatalogExplorer({ papers, reviewedAt }: Props) { paperB.year - paperA.year || paperA.system.localeCompare(paperB.system), ); - }, [classification, fairness, method, papers, product, query, venue]); + }, [classification, conference, domain, fairness, language, method, papers, product, query]); const filtersActive = query !== "" || product !== "all" || classification !== "all" || fairness !== "all" || - venue !== "all" || + conference !== "all" || + domain !== "all" || method !== "all"; useEffect(() => { @@ -369,10 +472,21 @@ export function CatalogExplorer({ papers, reviewedAt }: Props) { setProduct("all"); setClassification("all"); setFairness("all"); - setVenue("all"); + setConference("all"); + setDomain("all"); setMethod("all"); }; + const chooseDomain = (value: DomainId) => { + setDomain(value); + document.getElementById("catalog")?.scrollIntoView({ behavior: "smooth" }); + }; + + const chooseConference = (value: ConferenceId) => { + setConference(value); + document.getElementById("catalog")?.scrollIntoView({ behavior: "smooth" }); + }; + const chooseMethod = (tag: string) => { setMethod(tag); document.getElementById("catalog")?.scrollIntoView({ behavior: "smooth" }); @@ -395,6 +509,7 @@ export function CatalogExplorer({ papers, reviewedAt }: Props) { Agent Papers