From 50d8ecf411d79eae039f28f734ec64bbaace7370 Mon Sep 17 00:00:00 2001 From: "google-labs-jules[bot]" <161369871+google-labs-jules[bot]@users.noreply.github.com> Date: Wed, 8 Apr 2026 05:57:14 +0000 Subject: [PATCH] chore(sync): implement autonomous benchmark synchronization * Created autonomous logic to sync benchmarks/suites and benchmarks/criteria * Instruct AI to formulate valid nested JSON Schema formats * Ensured git auto-commit captures new criteria schemas instead of discarding them * Appended critical labels to gh issue creations Co-authored-by: beginwebdev2002 <102213457+beginwebdev2002@users.noreply.github.com> --- benchmarks/criteria/angular-schema.json | 9 ++- benchmarks/criteria/mongodb-schema.json | 12 ++- benchmarks/criteria/nestjs-schema.json | 12 ++- benchmarks/criteria/typescript-schema.json | 7 +- benchmarks/suites/angular.json | 4 +- benchmarks/suites/mongodb.json | 4 +- benchmarks/suites/nestjs.json | 4 +- benchmarks/suites/typescript.json | 4 +- vibe-check-runner.js | 88 +++++++++++++++++++--- 9 files changed, 120 insertions(+), 24 deletions(-) diff --git a/benchmarks/criteria/angular-schema.json b/benchmarks/criteria/angular-schema.json index d091f4a..ed4e750 100644 --- a/benchmarks/criteria/angular-schema.json +++ b/benchmarks/criteria/angular-schema.json @@ -16,7 +16,12 @@ "forbidden_types": { "type": "array", "items": { "type": "string" }, - "contains": { "enum": ["BehaviorSubject", "EventEmitter"] } + "contains": { "enum": ["BehaviorSubject", "EventEmitter", "any"] } + }, + "required_imports": { + "type": "array", + "items": { "type": "string" }, + "contains": { "enum": ["@features", "@entities", "@shared", "@domain"] } } } -} \ No newline at end of file +} diff --git a/benchmarks/criteria/mongodb-schema.json b/benchmarks/criteria/mongodb-schema.json index 310158c..ed44b6c 100644 --- a/benchmarks/criteria/mongodb-schema.json +++ b/benchmarks/criteria/mongodb-schema.json @@ -12,6 +12,16 @@ "type": "array", "items": { "type": "string" }, "contains": { "enum": ["insertOne without validation"] } + }, + "forbidden_types": { + "type": "array", + "items": { "type": "string" }, + "contains": { "enum": ["any"] } + }, + "required_imports": { + "type": "array", + "items": { "type": "string" }, + "contains": { "enum": ["@features", "@entities", "@shared", "@domain"] } } } -} \ No newline at end of file +} diff --git a/benchmarks/criteria/nestjs-schema.json b/benchmarks/criteria/nestjs-schema.json index 3f75639..0d0139c 100644 --- a/benchmarks/criteria/nestjs-schema.json +++ b/benchmarks/criteria/nestjs-schema.json @@ -12,6 +12,16 @@ "type": "array", "items": { "type": "string" }, "contains": { "enum": ["setInterval", "@Post() for microservices", "@Get('ping')"] } + }, + "forbidden_types": { + "type": "array", + "items": { "type": "string" }, + "contains": { "enum": ["any"] } + }, + "required_imports": { + "type": "array", + "items": { "type": "string" }, + "contains": { "enum": ["@features", "@entities", "@shared", "@domain"] } } } -} \ No newline at end of file +} diff --git a/benchmarks/criteria/typescript-schema.json b/benchmarks/criteria/typescript-schema.json index ab44ec6..0285746 100644 --- a/benchmarks/criteria/typescript-schema.json +++ b/benchmarks/criteria/typescript-schema.json @@ -12,6 +12,11 @@ "type": "array", "items": { "type": "string" }, "contains": { "enum": ["unknown"] } + }, + "required_imports": { + "type": "array", + "items": { "type": "string" }, + "contains": { "enum": ["@features", "@entities", "@shared", "@domain"] } } } -} \ No newline at end of file +} diff --git a/benchmarks/suites/angular.json b/benchmarks/suites/angular.json index 868f961..3725411 100644 --- a/benchmarks/suites/angular.json +++ b/benchmarks/suites/angular.json @@ -1,4 +1,4 @@ { - "golden_prompt": "Generate an Angular v20+ component or service adhering strictly to Zoneless reactivity constraints: 1) ALWAYS use signal(), computed(), and effect() instead of RxJS BehaviorSubject; 2) NEVER use @Input() or @Output() decorators, use input() and output() functional APIs instead; 3) ALWAYS use built-in control flow (@if, @for) instead of structural directives (*ngIf, *ngFor); 4) Avoid Heavy Logic in Templates; 5) Use inject() for DI; 6) Use Standalone Components instead of NgModules.", + "golden_prompt": "Generate an Angular v20+ component or service adhering strictly to Zoneless reactivity constraints: 1) ALWAYS use signal(), computed(), and effect() instead of RxJS BehaviorSubject; 2) NEVER use @Input() or @Output() decorators, use input() and output() functional APIs instead; 3) ALWAYS use built-in control flow (@if, @for) instead of structural directives (*ngIf, *ngFor); 4) Avoid Heavy Logic in Templates; 5) Use inject() for DI; 6) Use Standalone Components instead of NgModules; 7) Strictly forbid 'any' type; 8) Enforce FSD/DDD layer imports (@features, @entities, @shared, @domain).", "tech": "angular" -} \ No newline at end of file +} diff --git a/benchmarks/suites/mongodb.json b/benchmarks/suites/mongodb.json index 0696ae6..50935bf 100644 --- a/benchmarks/suites/mongodb.json +++ b/benchmarks/suites/mongodb.json @@ -1,4 +1,4 @@ { - "golden_prompt": "Generate MongoDB database logic following enterprise architectural constraints: 1) ALWAYS implement schema validation at the database layer (JSON Schema) and ORM/ODM level (Mongoose); 2) NEVER allow unstructured data insertion; 3) Use the ESR (Equality, Sort, Range) rule for indexing; 4) Implement strict RBAC and field-level encryption.", + "golden_prompt": "Generate MongoDB database logic following enterprise architectural constraints: 1) ALWAYS implement schema validation at the database layer (JSON Schema) and ORM/ODM level (Mongoose); 2) NEVER allow unstructured data insertion; 3) Use the ESR (Equality, Sort, Range) rule for indexing; 4) Implement strict RBAC and field-level encryption; 5) Strictly forbid 'any' type; 6) Enforce FSD/DDD layer imports (@features, @entities, @shared, @domain).", "tech": "mongodb" -} \ No newline at end of file +} diff --git a/benchmarks/suites/nestjs.json b/benchmarks/suites/nestjs.json index cc73039..89b5c95 100644 --- a/benchmarks/suites/nestjs.json +++ b/benchmarks/suites/nestjs.json @@ -1,4 +1,4 @@ { - "golden_prompt": "Generate a NestJS controller or service following these constraints: 1) Cache heavy requests via CacheModule (in-memory or Redis); 2) Avoid tight coupling using EventEmitter; 3) Use @nestjs/schedule for Cron tasks; 4) Use @MessagePattern (TCP/Redis) for microservices; 5) Implement HealthCheck for liveness probes; 6) Avoid circular dependencies; 7) Re-export common modules; 8) Use global middleware for pre-Guard operations; 9) Use Mocks in Unit Tests; 10) Use Custom Validation Constraints in class-validator; 11) Use FileInterceptor for file uploading; 12) Use ClassSerializerInterceptor with @Exclude() for secure serialization; 13) Support Fastify integration; 14) Implement Graceful Shutdown hooks.", + "golden_prompt": "Generate a NestJS controller or service following these constraints: 1) Cache heavy requests via CacheModule (in-memory or Redis); 2) Avoid tight coupling using EventEmitter; 3) Use @nestjs/schedule for Cron tasks; 4) Use @MessagePattern (TCP/Redis) for microservices; 5) Implement HealthCheck for liveness probes; 6) Avoid circular dependencies; 7) Re-export common modules; 8) Use global middleware for pre-Guard operations; 9) Use Mocks in Unit Tests; 10) Use Custom Validation Constraints in class-validator; 11) Use FileInterceptor for file uploading; 12) Use ClassSerializerInterceptor with @Exclude() for secure serialization; 13) Support Fastify integration; 14) Implement Graceful Shutdown hooks; 15) Strictly forbid 'any' type; 16) Enforce FSD/DDD layer imports (@features, @entities, @shared, @domain).", "tech": "nestjs" -} \ No newline at end of file +} diff --git a/benchmarks/suites/typescript.json b/benchmarks/suites/typescript.json index 481a3b0..1b705ae 100644 --- a/benchmarks/suites/typescript.json +++ b/benchmarks/suites/typescript.json @@ -1,4 +1,4 @@ { - "golden_prompt": "Generate TypeScript logic conforming to advanced type-safety constraints: 1) Prefer `unknown` over `any` for uncertain types, requiring type guards before use; 2) Consistently use either `null` or `undefined` (prefer `null` for intentional absence, `undefined` for uninitialized data) but do not mix arbitrarily; 3) Use exact type definitions to avoid runtime issues.", + "golden_prompt": "Generate TypeScript logic conforming to advanced type-safety constraints: 1) Prefer `unknown` over `any` for uncertain types, requiring type guards before use; 2) Consistently use either `null` or `undefined` (prefer `null` for intentional absence, `undefined` for uninitialized data) but do not mix arbitrarily; 3) Use exact type definitions to avoid runtime issues; 4) Strictly forbid 'any' type; 5) Enforce FSD/DDD layer imports (@features, @entities, @shared, @domain).", "tech": "typescript" -} \ No newline at end of file +} diff --git a/vibe-check-runner.js b/vibe-check-runner.js index 1ebff4a..cc9c283 100644 --- a/vibe-check-runner.js +++ b/vibe-check-runner.js @@ -32,6 +32,33 @@ function getModifiedFiles() { } } +async function syncBenchmarks(tech, mdContent) { + try { + const prompt = `Analyze the following documentation for ${tech}:\n\n${mdContent}\n\n1) Generate a "Golden Prompt" (a stringent instruction to an AI) reflecting these rules, ensuring strict typing and DDD/FSD layers.\n2) Generate JSON Schema defining AST validation rules. MUST USE NESTED JSON SCHEMA FORMAT: {"properties": {"forbidden_types": {"contains": {"enum": ["any"]}}, "required_imports": {"contains": {"enum": ["@features", "@entities", "@shared", "@domain"]}}}}.\nOutput as a strict JSON object with two keys: "golden_prompt" (string) and "schema" (object). No markdown formatting, no explanations.`; + const response = await ai.models.generateContent({ + model: 'gemini-2.5-pro', + contents: prompt + }); + let text = response.text || '{}'; + text = text.replace(/^```[a-z]*\n/gm, '').replace(/```$/gm, '').trim(); + const result = JSON.parse(text); + + if (result.golden_prompt) { + const suitePath = path.join('benchmarks', 'suites', `${tech}.json`); + fs.writeFileSync(suitePath, JSON.stringify({ golden_prompt: result.golden_prompt, tech }, null, 2)); + console.log(`Synchronized suite: ${suitePath}`); + } + + if (result.schema) { + const criteriaPath = path.join('benchmarks', 'criteria', `${tech}-schema.json`); + fs.writeFileSync(criteriaPath, JSON.stringify(result.schema, null, 2)); + console.log(`Synchronized criteria: ${criteriaPath}`); + } + } catch (err) { + console.error(`Error syncing benchmarks for ${tech}:`, err.message); + } +} + async function simulateAIGeneration(goldenPrompt, tech, mdContent) { try { const prompt = `${goldenPrompt}\n\nConstraints and instructions from the following documentation:\n\n${mdContent}\n\nGenerate ONLY raw code. No markdown formatting, no explanations.`; @@ -94,15 +121,36 @@ function analyzeAST(sourceFile, tech) { // Check if string contains imports that hint at FSD like '@features', '@entities', '@shared' etc. const imports = sourceFile.getImportDeclarations(); const moduleSpecifiers = imports.map(imp => imp.getModuleSpecifierValue()); - const hasFSD = moduleSpecifiers.some(spec => spec.includes('features/') || spec.includes('entities/') || spec.includes('shared/') || spec.includes('domain/')); - // Not strictly enforcing this to be 10 point penalty if small snippet, but we reduce if completely monolithic (no imports) - if (moduleSpecifiers.length === 0 && sourceFile.getClasses().length > 1) { + + // Use schema if available to enforce imports + // Unified schema loading to avoid multiple I/O reads + let requiredImports = ['@features', '@entities', '@shared', '@domain']; + let forbiddenTypes = ['any']; + const criteriaPath = path.join('benchmarks', 'criteria', `${tech}-schema.json`); + if (fs.existsSync(criteriaPath)) { + try { + const criteria = JSON.parse(fs.readFileSync(criteriaPath, 'utf8')); + if (criteria.properties?.required_imports?.contains?.enum) { + requiredImports = criteria.properties.required_imports.contains.enum; + } + if (criteria.properties?.forbidden_types?.contains?.enum) { + forbiddenTypes = criteria.properties.forbidden_types.contains.enum; + } + } catch (e) {} + } + + const hasFSD = moduleSpecifiers.some(spec => + requiredImports.some(ri => spec.includes(ri.replace('@', ''))) || + requiredImports.some(ri => spec.includes(ri)) + ); + + if (!hasFSD) { score.arch -= 10; } // 2. Type Safety (30) const anyKeywords = sourceFile.getDescendantsOfKind(SyntaxKind.AnyKeyword); - if (anyKeywords.length > 0) { + if (forbiddenTypes.includes('any') && anyKeywords.length > 0) { score.type -= 15 * anyKeywords.length; } @@ -193,7 +241,18 @@ async function runVibeCheck() { const suiteConfig = JSON.parse(fs.readFileSync(suitePath, 'utf-8')); const mdContent = fs.readFileSync(file, 'utf-8'); - const generatedCode = await simulateAIGeneration(suiteConfig.golden_prompt, tech, mdContent); + // Autonomously synchronize benchmarks based on new documentation constraints + await syncBenchmarks(tech, mdContent); + + // Reload the newly synced suite config + let syncedSuiteConfig; + if (fs.existsSync(suitePath)) { + syncedSuiteConfig = JSON.parse(fs.readFileSync(suitePath, 'utf-8')); + } else { + syncedSuiteConfig = suiteConfig; + } + + const generatedCode = await simulateAIGeneration(syncedSuiteConfig.golden_prompt || suiteConfig.golden_prompt, tech, mdContent); if (!generatedCode) { console.error(`Failed to generate code for ${tech}.`); @@ -217,13 +276,20 @@ async function runVibeCheck() { try { execSync(`git add ${file}`); - // Only commit if there are changes (badge might already be there) + execSync(`git add benchmarks/suites/${tech}.json || true`); + execSync(`git add benchmarks/criteria/${tech}-schema.json || true`); + + // Only commit if there are any staged changes for this file or its benchmarks const status = execSync('git status --porcelain', { encoding: 'utf-8' }); - if (status.includes(file)) { - execSync(`git commit -m "chore: fidelity-pass for ${file}"`); + const hasChanges = status.includes(file) || + status.includes(`benchmarks/suites/${tech}.json`) || + status.includes(`benchmarks/criteria/${tech}-schema.json`); + + if (hasChanges) { + execSync(`git commit -m "[chore: benchmark-sync]"`); execSync(`git push origin HEAD:main`); } else { - console.log(`Badge already present in ${file}, skipping commit.`); + console.log(`No changes needed for ${file} or its benchmarks, skipping commit.`); } } catch (err) { console.error('Failed to commit or push:', err.message); @@ -242,8 +308,8 @@ async function runVibeCheck() { console.log(`Generated violation report: ${reportPath}`); try { - execSync(`gh issue create --title "Fidelity Gap: ${file}" --body-file ${reportPath}`); - console.log(`Created GitHub Issue for ${file}`); + execSync(`gh issue create --title "Critical Issue: Fidelity Gap in ${file}" --body-file ${reportPath} --label "critical,bug"`); + console.log(`Created Critical GitHub Issue for ${file}`); } catch (err) { console.error('Failed to create GitHub Issue (gh cli might not be installed or authenticated):', err.message); }