From 31753eff10a3d5c23502063230ec091a442b69c0 Mon Sep 17 00:00:00 2001 From: "google-labs-jules[bot]" <161369871+google-labs-jules[bot]@users.noreply.github.com> Date: Tue, 7 Apr 2026 05:11:45 +0000 Subject: [PATCH] chore: benchmark-sync Co-authored-by: beginwebdev2002 <102213457+beginwebdev2002@users.noreply.github.com> --- .github/scripts/utils.js | 7 +- benchmarks/criteria/angular-schema.json | 22 ++++ benchmarks/criteria/mongodb-schema.json | 17 +++ benchmarks/criteria/nestjs-schema.json | 17 +++ benchmarks/criteria/typescript-schema.json | 17 +++ benchmarks/suites/angular.json | 4 + benchmarks/suites/mongodb.json | 4 + benchmarks/suites/nestjs.json | 4 + benchmarks/suites/typescript.json | 4 + package-lock.json | 117 ++++++++++++++++++++- package.json | 3 +- vibe-check-runner.js | 71 +++++++++++++ 12 files changed, 281 insertions(+), 6 deletions(-) create mode 100644 benchmarks/criteria/angular-schema.json create mode 100644 benchmarks/criteria/mongodb-schema.json create mode 100644 benchmarks/criteria/nestjs-schema.json create mode 100644 benchmarks/criteria/typescript-schema.json create mode 100644 benchmarks/suites/angular.json create mode 100644 benchmarks/suites/mongodb.json create mode 100644 benchmarks/suites/nestjs.json create mode 100644 benchmarks/suites/typescript.json create mode 100644 vibe-check-runner.js diff --git a/.github/scripts/utils.js b/.github/scripts/utils.js index 3d8161e..4138f1b 100644 --- a/.github/scripts/utils.js +++ b/.github/scripts/utils.js @@ -1,8 +1,7 @@ -import crypto from 'crypto'; +import crypto from 'node:crypto'; import 'dotenv/config'; -import { writeFile } from 'fs'; -import { join } from 'path'; -import crypto from 'crypto'; +import { writeFile } from 'node:fs'; +import { join } from 'node:path'; export async function saveImage(bytes, filename = randomText() + '.png') { const localPath = join(process.cwd(), filename); diff --git a/benchmarks/criteria/angular-schema.json b/benchmarks/criteria/angular-schema.json new file mode 100644 index 0000000..d091f4a --- /dev/null +++ b/benchmarks/criteria/angular-schema.json @@ -0,0 +1,22 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "title": "Angular AST Validation Schema", + "type": "object", + "properties": { + "required_functions": { + "type": "array", + "items": { "type": "string" }, + "contains": { "enum": ["signal", "computed", "effect", "input", "output", "inject"] } + }, + "forbidden_decorators": { + "type": "array", + "items": { "type": "string" }, + "contains": { "enum": ["Input", "Output", "NgModule"] } + }, + "forbidden_types": { + "type": "array", + "items": { "type": "string" }, + "contains": { "enum": ["BehaviorSubject", "EventEmitter"] } + } + } +} \ No newline at end of file diff --git a/benchmarks/criteria/mongodb-schema.json b/benchmarks/criteria/mongodb-schema.json new file mode 100644 index 0000000..310158c --- /dev/null +++ b/benchmarks/criteria/mongodb-schema.json @@ -0,0 +1,17 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "title": "MongoDB AST Validation Schema", + "type": "object", + "properties": { + "required_methods": { + "type": "array", + "items": { "type": "string" }, + "contains": { "enum": ["createCollection", "validator", "$jsonSchema"] } + }, + "forbidden_patterns": { + "type": "array", + "items": { "type": "string" }, + "contains": { "enum": ["insertOne without validation"] } + } + } +} \ No newline at end of file diff --git a/benchmarks/criteria/nestjs-schema.json b/benchmarks/criteria/nestjs-schema.json new file mode 100644 index 0000000..3f75639 --- /dev/null +++ b/benchmarks/criteria/nestjs-schema.json @@ -0,0 +1,17 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "title": "NestJS AST Validation Schema", + "type": "object", + "properties": { + "decorators": { + "type": "array", + "items": { "type": "string" }, + "contains": { "enum": ["UseInterceptors", "CacheTTL", "Cron", "MessagePattern", "HealthCheck", "Inject", "Module", "Injectable", "ValidatorConstraint", "Validate"] } + }, + "forbidden_patterns": { + "type": "array", + "items": { "type": "string" }, + "contains": { "enum": ["setInterval", "@Post() for microservices", "@Get('ping')"] } + } + } +} \ No newline at end of file diff --git a/benchmarks/criteria/typescript-schema.json b/benchmarks/criteria/typescript-schema.json new file mode 100644 index 0000000..ab44ec6 --- /dev/null +++ b/benchmarks/criteria/typescript-schema.json @@ -0,0 +1,17 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "title": "TypeScript AST Validation Schema", + "type": "object", + "properties": { + "forbidden_types": { + "type": "array", + "items": { "type": "string" }, + "contains": { "enum": ["any"] } + }, + "required_types": { + "type": "array", + "items": { "type": "string" }, + "contains": { "enum": ["unknown"] } + } + } +} \ No newline at end of file diff --git a/benchmarks/suites/angular.json b/benchmarks/suites/angular.json new file mode 100644 index 0000000..868f961 --- /dev/null +++ b/benchmarks/suites/angular.json @@ -0,0 +1,4 @@ +{ + "golden_prompt": "Generate an Angular v20+ component or service adhering strictly to Zoneless reactivity constraints: 1) ALWAYS use signal(), computed(), and effect() instead of RxJS BehaviorSubject; 2) NEVER use @Input() or @Output() decorators, use input() and output() functional APIs instead; 3) ALWAYS use built-in control flow (@if, @for) instead of structural directives (*ngIf, *ngFor); 4) Avoid Heavy Logic in Templates; 5) Use inject() for DI; 6) Use Standalone Components instead of NgModules.", + "tech": "angular" +} \ No newline at end of file diff --git a/benchmarks/suites/mongodb.json b/benchmarks/suites/mongodb.json new file mode 100644 index 0000000..0696ae6 --- /dev/null +++ b/benchmarks/suites/mongodb.json @@ -0,0 +1,4 @@ +{ + "golden_prompt": "Generate MongoDB database logic following enterprise architectural constraints: 1) ALWAYS implement schema validation at the database layer (JSON Schema) and ORM/ODM level (Mongoose); 2) NEVER allow unstructured data insertion; 3) Use the ESR (Equality, Sort, Range) rule for indexing; 4) Implement strict RBAC and field-level encryption.", + "tech": "mongodb" +} \ No newline at end of file diff --git a/benchmarks/suites/nestjs.json b/benchmarks/suites/nestjs.json new file mode 100644 index 0000000..cc73039 --- /dev/null +++ b/benchmarks/suites/nestjs.json @@ -0,0 +1,4 @@ +{ + "golden_prompt": "Generate a NestJS controller or service following these constraints: 1) Cache heavy requests via CacheModule (in-memory or Redis); 2) Avoid tight coupling using EventEmitter; 3) Use @nestjs/schedule for Cron tasks; 4) Use @MessagePattern (TCP/Redis) for microservices; 5) Implement HealthCheck for liveness probes; 6) Avoid circular dependencies; 7) Re-export common modules; 8) Use global middleware for pre-Guard operations; 9) Use Mocks in Unit Tests; 10) Use Custom Validation Constraints in class-validator; 11) Use FileInterceptor for file uploading; 12) Use ClassSerializerInterceptor with @Exclude() for secure serialization; 13) Support Fastify integration; 14) Implement Graceful Shutdown hooks.", + "tech": "nestjs" +} \ No newline at end of file diff --git a/benchmarks/suites/typescript.json b/benchmarks/suites/typescript.json new file mode 100644 index 0000000..481a3b0 --- /dev/null +++ b/benchmarks/suites/typescript.json @@ -0,0 +1,4 @@ +{ + "golden_prompt": "Generate TypeScript logic conforming to advanced type-safety constraints: 1) Prefer `unknown` over `any` for uncertain types, requiring type guards before use; 2) Consistently use either `null` or `undefined` (prefer `null` for intentional absence, `undefined` for uninitialized data) but do not mix arbitrarily; 3) Use exact type definitions to avoid runtime issues.", + "tech": "typescript" +} \ No newline at end of file diff --git a/package-lock.json b/package-lock.json index 19e34d8..699ceff 100644 --- a/package-lock.json +++ b/package-lock.json @@ -10,7 +10,8 @@ "@google-cloud/storage": "^7.19.0", "@google-cloud/vertexai": "^1.1.0", "@google/genai": "^1.46.0", - "axios": "^1.13.6" + "axios": "^1.13.6", + "ts-morph": "^27.0.2" }, "devDependencies": { "dotenv": "^17.3.1", @@ -335,6 +336,53 @@ "node": ">= 10" } }, + "node_modules/@ts-morph/common": { + "version": "0.28.1", + "resolved": "https://registry.npmjs.org/@ts-morph/common/-/common-0.28.1.tgz", + "integrity": "sha512-W74iWf7ILp1ZKNYXY5qbddNaml7e9Sedv5lvU1V8lftlitkc9Pq1A+jlH23ltDgWYeZFFEqGCD1Ies9hqu3O+g==", + "license": "MIT", + "dependencies": { + "minimatch": "^10.0.1", + "path-browserify": "^1.0.1", + "tinyglobby": "^0.2.14" + } + }, + "node_modules/@ts-morph/common/node_modules/balanced-match": { + "version": "4.0.4", + "resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-4.0.4.tgz", + "integrity": "sha512-BLrgEcRTwX2o6gGxGOCNyMvGSp35YofuYzw9h1IMTRmKqttAZZVU67bdb9Pr2vUHA8+j3i2tJfjO6C6+4myGTA==", + "license": "MIT", + "engines": { + "node": "18 || 20 || >=22" + } + }, + "node_modules/@ts-morph/common/node_modules/brace-expansion": { + "version": "5.0.5", + "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-5.0.5.tgz", + "integrity": "sha512-VZznLgtwhn+Mact9tfiwx64fA9erHH/MCXEUfB/0bX/6Fz6ny5EGTXYltMocqg4xFAQZtnO3DHWWXi8RiuN7cQ==", + "license": "MIT", + "dependencies": { + "balanced-match": "^4.0.2" + }, + "engines": { + "node": "18 || 20 || >=22" + } + }, + "node_modules/@ts-morph/common/node_modules/minimatch": { + "version": "10.2.5", + "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-10.2.5.tgz", + "integrity": "sha512-MULkVLfKGYDFYejP07QOurDLLQpcjk7Fw+7jXS2R2czRQzR56yHRveU5NDJEOviH+hETZKSkIk5c+T23GjFUMg==", + "license": "BlueOak-1.0.0", + "dependencies": { + "brace-expansion": "^5.0.5" + }, + "engines": { + "node": "18 || 20 || >=22" + }, + "funding": { + "url": "https://github.com/sponsors/isaacs" + } + }, "node_modules/@types/caseless": { "version": "0.12.5", "resolved": "https://registry.npmjs.org/@types/caseless/-/caseless-0.12.5.tgz", @@ -685,6 +733,12 @@ "url": "https://github.com/chalk/wrap-ansi?sponsor=1" } }, + "node_modules/code-block-writer": { + "version": "13.0.3", + "resolved": "https://registry.npmjs.org/code-block-writer/-/code-block-writer-13.0.3.tgz", + "integrity": "sha512-Oofo0pq3IKnsFtuHqSF7TqBfr71aeyZDVJ0HpmqB7FBM2qEigL0iPONSCZSO9pE9dZTAxANe5XHG9Uy0YMv8cg==", + "license": "MIT" + }, "node_modules/color-convert": { "version": "2.0.1", "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-2.0.1.tgz", @@ -1878,6 +1932,12 @@ "integrity": "sha512-UEZIS3/by4OC8vL3P2dTXRETpebLI2NiI5vIrjaD/5UtrkFX/tNbwjTSRAGC/+7CAo2pIcBaRgWmcBBHcsaCIw==", "license": "BlueOak-1.0.0" }, + "node_modules/path-browserify": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/path-browserify/-/path-browserify-1.0.1.tgz", + "integrity": "sha512-b7uo2UCUOYZcnF/3ID0lulOJi/bafxa1xPe7ZPsammBSpjSWQkjNxlt635YGS2MiR9GjvuXCtz2emr3jbsz98g==", + "license": "MIT" + }, "node_modules/path-expression-matcher": { "version": "1.2.0", "resolved": "https://registry.npmjs.org/path-expression-matcher/-/path-expression-matcher-1.2.0.tgz", @@ -2338,6 +2398,51 @@ "uuid": "dist/bin/uuid" } }, + "node_modules/tinyglobby": { + "version": "0.2.15", + "resolved": "https://registry.npmjs.org/tinyglobby/-/tinyglobby-0.2.15.tgz", + "integrity": "sha512-j2Zq4NyQYG5XMST4cbs02Ak8iJUdxRM0XI5QyxXuZOzKOINmWurp3smXu3y5wDcJrptwpSjgXHzIQxR0omXljQ==", + "license": "MIT", + "dependencies": { + "fdir": "^6.5.0", + "picomatch": "^4.0.3" + }, + "engines": { + "node": ">=12.0.0" + }, + "funding": { + "url": "https://github.com/sponsors/SuperchupuDev" + } + }, + "node_modules/tinyglobby/node_modules/fdir": { + "version": "6.5.0", + "resolved": "https://registry.npmjs.org/fdir/-/fdir-6.5.0.tgz", + "integrity": "sha512-tIbYtZbucOs0BRGqPJkshJUYdL+SDH7dVM8gjy+ERp3WAUjLEFJE+02kanyHtwjWOnwrKYBiwAmM0p4kLJAnXg==", + "license": "MIT", + "engines": { + "node": ">=12.0.0" + }, + "peerDependencies": { + "picomatch": "^3 || ^4" + }, + "peerDependenciesMeta": { + "picomatch": { + "optional": true + } + } + }, + "node_modules/tinyglobby/node_modules/picomatch": { + "version": "4.0.4", + "resolved": "https://registry.npmjs.org/picomatch/-/picomatch-4.0.4.tgz", + "integrity": "sha512-QP88BAKvMam/3NxH6vj2o21R6MjxZUAd6nlwAS/pnGvN9IVLocLHxGYIzFhg6fUQ+5th6P4dv4eW9jX3DSIj7A==", + "license": "MIT", + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://github.com/sponsors/jonschlinkert" + } + }, "node_modules/to-regex-range": { "version": "5.0.1", "resolved": "https://registry.npmjs.org/to-regex-range/-/to-regex-range-5.0.1.tgz", @@ -2367,6 +2472,16 @@ "integrity": "sha512-N3WMsuqV66lT30CrXNbEjx4GEwlow3v6rr4mCcv6prnfwhS01rkgyFdjPNBYd9br7LpXV1+Emh01fHnq2Gdgrw==", "license": "MIT" }, + "node_modules/ts-morph": { + "version": "27.0.2", + "resolved": "https://registry.npmjs.org/ts-morph/-/ts-morph-27.0.2.tgz", + "integrity": "sha512-fhUhgeljcrdZ+9DZND1De1029PrE+cMkIP7ooqkLRTrRLTqcki2AstsyJm0vRNbTbVCNJ0idGlbBrfqc7/nA8w==", + "license": "MIT", + "dependencies": { + "@ts-morph/common": "~0.28.1", + "code-block-writer": "^13.0.3" + } + }, "node_modules/undefsafe": { "version": "2.0.5", "resolved": "https://registry.npmjs.org/undefsafe/-/undefsafe-2.0.5.tgz", diff --git a/package.json b/package.json index 2813df6..50db24c 100644 --- a/package.json +++ b/package.json @@ -14,6 +14,7 @@ "@google-cloud/storage": "^7.19.0", "@google-cloud/vertexai": "^1.1.0", "@google/genai": "^1.46.0", - "axios": "^1.13.6" + "axios": "^1.13.6", + "ts-morph": "^27.0.2" } } diff --git a/vibe-check-runner.js b/vibe-check-runner.js new file mode 100644 index 0000000..c2c1804 --- /dev/null +++ b/vibe-check-runner.js @@ -0,0 +1,71 @@ +import { Project, SyntaxKind } from 'ts-morph'; +import fs from 'node:fs'; +import path from 'node:path'; + +// Mock AI Output to evaluate +const mockCode = ` +import { signal, computed, effect } from '@angular/core'; + +export class MyComponent { + title = signal('Hello'); + derived = computed(() => this.title() + ' World'); + + constructor() { + effect(() => { + console.log(this.derived()); + }); + } +} +`; + +function analyzeAngularAST(sourceFile) { + let score = 100; + + // Check for forbidden decorators + const decorators = sourceFile.getDescendantsOfKind(SyntaxKind.Decorator); + for (const decorator of decorators) { + const name = decorator.getName(); + if (['Input', 'Output'].includes(name)) { + score -= 20; + } + } + + // Check for required functions + const imports = sourceFile.getImportDeclarations(); + let hasSignal = false; + for (const imp of imports) { + const namedImports = imp.getNamedImports().map(ni => ni.getName()); + if (namedImports.includes('signal')) hasSignal = true; + } + + if (!hasSignal) { + score -= 20; + } + + return score; +} + +async function runVibeCheck() { + console.log('Running Vibe-Check Runner...'); + + const project = new Project(); + const sourceFile = project.createSourceFile('mock-component.ts', mockCode); + + const score = analyzeAngularAST(sourceFile); + + console.log(`Fidelity Score: ${score}%`); + + if (score >= 95) { + console.log('✅ Validation passed. Ready for auto-commit.'); + process.exit(0); + } else { + console.error('❌ Validation failed. Score below 95%.'); + + // Generate violation report + const reportContent = `# Critical Violation Report\n\nFidelity Score: ${score}%\nThreshold: 95%\n\nReview the AST rules.`; + fs.writeFileSync('violation-report.md', reportContent); + process.exit(1); + } +} + +runVibeCheck().catch(console.error); \ No newline at end of file