chore: benchmark-sync

.github/scripts/utils.js

@@ -1,4 +1,4 @@
-import crypto from 'crypto';
+import crypto from 'node:crypto';
 import 'dotenv/config';
 import { writeFile } from 'fs';
 import { join } from 'path';

benchmarks/criteria/angular-schema.json

@@ -0,0 +1,22 @@
+{
+  "$schema": "http://json-schema.org/draft-07/schema#",
+  "title": "Angular AST Validation Schema",
+  "type": "object",
+  "properties": {
+    "required_functions": {
+      "type": "array",
+      "items": { "type": "string" },
+      "contains": { "enum": ["signal", "computed", "effect", "input", "output", "inject"] }
+    },
+    "forbidden_decorators": {
+      "type": "array",
+      "items": { "type": "string" },
+      "contains": { "enum": ["Input", "Output", "NgModule"] }
+    },
+    "forbidden_types": {
+      "type": "array",
+      "items": { "type": "string" },
+      "contains": { "enum": ["BehaviorSubject", "EventEmitter"] }
+    }
+  }
+}

benchmarks/criteria/mongodb-schema.json

@@ -0,0 +1,17 @@
+{
+  "$schema": "http://json-schema.org/draft-07/schema#",
+  "title": "MongoDB AST Validation Schema",
+  "type": "object",
+  "properties": {
+    "required_methods": {
+      "type": "array",
+      "items": { "type": "string" },
+      "contains": { "enum": ["createCollection", "validator", "$jsonSchema"] }
+    },
+    "forbidden_patterns": {
+      "type": "array",
+      "items": { "type": "string" },
+      "contains": { "enum": ["insertOne without validation"] }
+    }
+  }
+}

benchmarks/criteria/nestjs-schema.json

@@ -0,0 +1,17 @@
+{
+  "$schema": "http://json-schema.org/draft-07/schema#",
+  "title": "NestJS AST Validation Schema",
+  "type": "object",
+  "properties": {
+    "decorators": {
+      "type": "array",
+      "items": { "type": "string" },
+      "contains": { "enum": ["UseInterceptors", "CacheTTL", "Cron", "MessagePattern", "HealthCheck", "Inject", "Module", "Injectable", "ValidatorConstraint", "Validate"] }
+    },
+    "forbidden_patterns": {
+      "type": "array",
+      "items": { "type": "string" },
+      "contains": { "enum": ["setInterval", "@Post() for microservices", "@Get('ping')"] }
+    }
+  }
+}

benchmarks/criteria/typescript-schema.json

@@ -0,0 +1,17 @@
+{
+  "$schema": "http://json-schema.org/draft-07/schema#",
+  "title": "TypeScript AST Validation Schema",
+  "type": "object",
+  "properties": {
+    "forbidden_types": {
+      "type": "array",
+      "items": { "type": "string" },
+      "contains": { "enum": ["any"] }
+    },
+    "required_types": {
+      "type": "array",
+      "items": { "type": "string" },
+      "contains": { "enum": ["unknown"] }
+    }
+  }
+}

benchmarks/suites/angular.json

@@ -0,0 +1,4 @@
+{
+  "golden_prompt": "Generate an Angular v20+ component or service adhering strictly to Zoneless reactivity constraints: 1) ALWAYS use signal(), computed(), and effect() instead of RxJS BehaviorSubject; 2) NEVER use @Input() or @Output() decorators, use input() and output() functional APIs instead; 3) ALWAYS use built-in control flow (@if, @for) instead of structural directives (*ngIf, *ngFor); 4) Avoid Heavy Logic in Templates; 5) Use inject() for DI; 6) Use Standalone Components instead of NgModules.",
+  "tech": "angular"
+}

benchmarks/suites/mongodb.json

@@ -0,0 +1,4 @@
+{
+  "golden_prompt": "Generate MongoDB database logic following enterprise architectural constraints: 1) ALWAYS implement schema validation at the database layer (JSON Schema) and ORM/ODM level (Mongoose); 2) NEVER allow unstructured data insertion; 3) Use the ESR (Equality, Sort, Range) rule for indexing; 4) Implement strict RBAC and field-level encryption.",
+  "tech": "mongodb"
+}

benchmarks/suites/nestjs.json

@@ -0,0 +1,4 @@
+{
+  "golden_prompt": "Generate a NestJS controller or service following these constraints: 1) Cache heavy requests via CacheModule (in-memory or Redis); 2) Avoid tight coupling using EventEmitter; 3) Use @nestjs/schedule for Cron tasks; 4) Use @MessagePattern (TCP/Redis) for microservices; 5) Implement HealthCheck for liveness probes; 6) Avoid circular dependencies; 7) Re-export common modules; 8) Use global middleware for pre-Guard operations; 9) Use Mocks in Unit Tests; 10) Use Custom Validation Constraints in class-validator; 11) Use FileInterceptor for file uploading; 12) Use ClassSerializerInterceptor with @Exclude() for secure serialization; 13) Support Fastify integration; 14) Implement Graceful Shutdown hooks.",
+  "tech": "nestjs"
+}

benchmarks/suites/typescript.json

@@ -0,0 +1,4 @@
+{
+  "golden_prompt": "Generate TypeScript logic conforming to advanced type-safety constraints: 1) Prefer `unknown` over `any` for uncertain types, requiring type guards before use; 2) Consistently use either `null` or `undefined` (prefer `null` for intentional absence, `undefined` for uninitialized data) but do not mix arbitrarily; 3) Use exact type definitions to avoid runtime issues.",
+  "tech": "typescript"
+}

package.json

@@ -14,6 +14,7 @@
     "@google-cloud/storage": "^7.19.0",
     "@google-cloud/vertexai": "^1.1.0",
     "@google/genai": "^1.46.0",
-    "axios": "^1.13.6"
+    "axios": "^1.13.6",
+    "ts-morph": "^27.0.2"
   }
 }

vibe-check-runner.js

@@ -0,0 +1,71 @@
+import { Project, SyntaxKind } from 'ts-morph';
+import fs from 'node:fs';
+import path from 'node:path';
+
+// Mock AI Output to evaluate
+const mockCode = `
+import { signal, computed, effect } from '@angular/core';
+
+export class MyComponent {
+  title = signal('Hello');
+  derived = computed(() => this.title() + ' World');
+
+  constructor() {
+    effect(() => {
+      console.log(this.derived());
+    });
+  }
+}
+`;
+
+function analyzeAngularAST(sourceFile) {
+  let score = 100;
+
+  // Check for forbidden decorators
+  const decorators = sourceFile.getDescendantsOfKind(SyntaxKind.Decorator);
+  for (const decorator of decorators) {
+    const name = decorator.getName();
+    if (['Input', 'Output'].includes(name)) {
+      score -= 20;
+    }
+  }
+
+  // Check for required functions
+  const imports = sourceFile.getImportDeclarations();
+  let hasSignal = false;
+  for (const imp of imports) {
+    const namedImports = imp.getNamedImports().map(ni => ni.getName());
+    if (namedImports.includes('signal')) hasSignal = true;
+  }
+
+  if (!hasSignal) {
+    score -= 20;
+  }
+
+  return score;
+}
+
+async function runVibeCheck() {
+  console.log('Running Vibe-Check Runner...');
+
+  const project = new Project();
+  const sourceFile = project.createSourceFile('mock-component.ts', mockCode);
+
+  const score = analyzeAngularAST(sourceFile);
+
+  console.log(`Fidelity Score: ${score}%`);
+
+  if (score >= 95) {
+    console.log('✅ Validation passed. Ready for auto-commit.');
+    process.exit(0);
+  } else {
+    console.error('❌ Validation failed. Score below 95%.');
+
+    // Generate violation report
+    const reportContent = `# Critical Violation Report\n\nFidelity Score: ${score}%\nThreshold: 95%\n\nReview the AST rules.`;
+    fs.writeFileSync('violation-report.md', reportContent);
+    process.exit(1);
+  }
+}
+
+runVibeCheck().catch(console.error);