diff --git a/.github/workflows/vibe-check.yml b/.github/workflows/vibe-check.yml new file mode 100644 index 0000000..0e1e306 --- /dev/null +++ b/.github/workflows/vibe-check.yml @@ -0,0 +1,43 @@ +name: Vibe-Check Runner + +on: + schedule: + - cron: '0 2 * * *' + workflow_dispatch: + +permissions: + contents: write + issues: write + +jobs: + fidelity_verification: + name: Automated Fidelity Verification + runs-on: ubuntu-latest + + steps: + - name: Checkout Repository + uses: actions/checkout@v4 + with: + fetch-depth: 0 + + - name: Setup Node.js + uses: actions/setup-node@v4 + with: + node-version: 22 + + - name: Install Dependencies + run: npm install + + - name: Run Vibe-Check + env: + GOOGLE_AI_API_KEY: ${{ secrets.GOOGLE_AI_API_KEY }} + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: node vibe-check-runner.js + + - name: Upload Violation Reports + if: failure() + uses: actions/upload-artifact@v4 + with: + name: fidelity-reports + path: benchmarks/logs/ + if-no-files-found: ignore diff --git a/vibe-check-runner.js b/vibe-check-runner.js index c2c1804..1ebff4a 100644 --- a/vibe-check-runner.js +++ b/vibe-check-runner.js @@ -1,70 +1,255 @@ import { Project, SyntaxKind } from 'ts-morph'; import fs from 'node:fs'; import path from 'node:path'; +import { execSync } from 'node:child_process'; +import { GoogleGenAI } from '@google/genai'; -// Mock AI Output to evaluate -const mockCode = ` -import { signal, computed, effect } from '@angular/core'; +const ai = new GoogleGenAI({ apiKey: process.env.GOOGLE_AI_API_KEY }); -export class MyComponent { - title = signal('Hello'); - derived = computed(() => this.title() + ' World'); +// Constants for scoring +const SCORES = { + ARCH: 40, + TYPE: 30, + SECURITY: 20, + EFFICIENCY: 10, +}; - constructor() { - effect(() => { - console.log(this.derived()); +function getModifiedFiles() { + try { + // In CI (daily run), check files modified in the last 24 hours. + // We filter for non-empty lines that end in .md and are in frontend/ or backend/ + const output = execSync('git log --since="24 hours ago" --name-only --pretty=format: | sort | uniq', { encoding: 'utf-8' }); + const allFiles = output.split('\n') + .map(f => f.trim()) + .filter(f => f.length > 0) + .filter(f => f.endsWith('.md')) + .filter(f => f.startsWith('frontend/') || f.startsWith('backend/')); + + return [...new Set(allFiles)]; + } catch (err) { + console.error('Error finding modified files:', err); + return []; + } +} + +async function simulateAIGeneration(goldenPrompt, tech, mdContent) { + try { + const prompt = `${goldenPrompt}\n\nConstraints and instructions from the following documentation:\n\n${mdContent}\n\nGenerate ONLY raw code. No markdown formatting, no explanations.`; + const response = await ai.models.generateContent({ + model: 'gemini-2.5-pro', + contents: prompt }); + let text = response.text || ''; + // Strip markdown code block wrappers if any + text = text.replace(/^```[a-z]*\n/gm, '').replace(/```$/gm, '').trim(); + return text; + } catch (err) { + console.error('Error generating AI code:', err); + return ''; } } -`; -function analyzeAngularAST(sourceFile) { - let score = 100; +function analyzeAST(sourceFile, tech) { + let score = { + arch: SCORES.ARCH, + type: SCORES.TYPE, + security: SCORES.SECURITY, + efficiency: SCORES.EFFICIENCY, + }; - // Check for forbidden decorators + // 1. Arch Integrity (40) const decorators = sourceFile.getDescendantsOfKind(SyntaxKind.Decorator); - for (const decorator of decorators) { - const name = decorator.getName(); - if (['Input', 'Output'].includes(name)) { - score -= 20; - } + const decoratorNames = decorators.map(d => d.getName()); + + if (tech === 'nestjs') { + if (!decoratorNames.includes('Injectable') && !decoratorNames.includes('Controller')) { + score.arch -= 10; + } + + // DTO Validation check + const classDeclarations = sourceFile.getDescendantsOfKind(SyntaxKind.ClassDeclaration); + let hasValidation = false; + for (const classDecl of classDeclarations) { + const classDecorators = classDecl.getProperties().flatMap(p => p.getDecorators().map(d => d.getName())); + if (classDecorators.some(name => name.startsWith('Is'))) { + hasValidation = true; + break; + } + } + if (!hasValidation && decoratorNames.length > 0) { + // Note: naive check, only apply if we generated classes + score.arch -= 10; + } + + } else if (tech === 'angular') { + if (!decoratorNames.includes('Component') && !decoratorNames.includes('Injectable')) { + score.arch -= 10; + } + if (decoratorNames.includes('Input') || decoratorNames.includes('Output')) { + score.arch -= 10; + } } - // Check for required functions + // FSD/DDD check (Naive representation checking for related imports or folder structure hints in string) + // Check if string contains imports that hint at FSD like '@features', '@entities', '@shared' etc. const imports = sourceFile.getImportDeclarations(); - let hasSignal = false; - for (const imp of imports) { - const namedImports = imp.getNamedImports().map(ni => ni.getName()); - if (namedImports.includes('signal')) hasSignal = true; + const moduleSpecifiers = imports.map(imp => imp.getModuleSpecifierValue()); + const hasFSD = moduleSpecifiers.some(spec => spec.includes('features/') || spec.includes('entities/') || spec.includes('shared/') || spec.includes('domain/')); + // Not strictly enforcing this to be 10 point penalty if small snippet, but we reduce if completely monolithic (no imports) + if (moduleSpecifiers.length === 0 && sourceFile.getClasses().length > 1) { + score.arch -= 10; + } + + // 2. Type Safety (30) + const anyKeywords = sourceFile.getDescendantsOfKind(SyntaxKind.AnyKeyword); + if (anyKeywords.length > 0) { + score.type -= 15 * anyKeywords.length; } - if (!hasSignal) { - score -= 20; + // Error handling pattern check + const tryStatements = sourceFile.getDescendantsOfKind(SyntaxKind.TryStatement); + const catchClauses = sourceFile.getDescendantsOfKind(SyntaxKind.CatchClause); + // If we have functions that do awaiting, they probably should have try/catch + const awaitExpressions = sourceFile.getDescendantsOfKind(SyntaxKind.AwaitExpression); + if (awaitExpressions.length > 0 && tryStatements.length === 0) { + score.type -= 10; // Penalize lack of error handling } - return score; + // 3. Security (20) + const stringLiterals = sourceFile.getDescendantsOfKind(SyntaxKind.StringLiteral); + for (const literal of stringLiterals) { + const text = literal.getText(); + if (text.includes('password') || text.includes('secret') || text.includes('token')) { + score.security -= 20; + } + } + + // 4. Efficiency (10) + const callExpressions = sourceFile.getDescendantsOfKind(SyntaxKind.CallExpression); + for(const call of callExpressions) { + if(call.getText().includes('readFileSync')) { + score.efficiency -= 10; + } + } + + score.arch = Math.max(0, score.arch); + score.type = Math.max(0, score.type); + score.security = Math.max(0, score.security); + score.efficiency = Math.max(0, score.efficiency); + + const total = score.arch + score.type + score.security + score.efficiency; + return { total, breakdown: score }; } async function runVibeCheck() { console.log('Running Vibe-Check Runner...'); + const modifiedFiles = getModifiedFiles(); + if (modifiedFiles.length === 0) { + console.log('No modified .md files found in frontend/ or backend/ within the last 24 hours.'); + return; + } + const project = new Project(); - const sourceFile = project.createSourceFile('mock-component.ts', mockCode); - const score = analyzeAngularAST(sourceFile); + // Configure git user for commits + try { + execSync('git config --global user.name "github-actions[bot]"'); + execSync('git config --global user.email "github-actions[bot]@users.noreply.github.com"'); + } catch (e) { + console.warn('Failed to configure git user. If running locally, this is expected.'); + } - console.log(`Fidelity Score: ${score}%`); + for (const file of modifiedFiles) { + console.log(`Processing ${file}...`); - if (score >= 95) { - console.log('✅ Validation passed. Ready for auto-commit.'); - process.exit(0); - } else { - console.error('❌ Validation failed. Score below 95%.'); + if (!fs.existsSync(file)) { + console.log(`File ${file} does not exist. Skipping.`); + continue; + } - // Generate violation report - const reportContent = `# Critical Violation Report\n\nFidelity Score: ${score}%\nThreshold: 95%\n\nReview the AST rules.`; - fs.writeFileSync('violation-report.md', reportContent); - process.exit(1); + let tech = ''; + if (file.includes('/angular/')) tech = 'angular'; + else if (file.includes('/nestjs/')) tech = 'nestjs'; + else if (file.includes('/typescript/')) tech = 'typescript'; + else if (file.includes('/express/')) tech = 'express'; + else if (file.includes('/nodejs/')) tech = 'nodejs'; + else { + // Fallback + const parts = file.split('/'); + if (parts.length > 1) { + tech = parts[1]; + } else { + continue; + } + } + + const suitePath = path.join('benchmarks', 'suites', `${tech}.json`); + if (!fs.existsSync(suitePath)) { + console.log(`No benchmark suite found for ${tech}. Skipping.`); + continue; + } + + const suiteConfig = JSON.parse(fs.readFileSync(suitePath, 'utf-8')); + const mdContent = fs.readFileSync(file, 'utf-8'); + + const generatedCode = await simulateAIGeneration(suiteConfig.golden_prompt, tech, mdContent); + + if (!generatedCode) { + console.error(`Failed to generate code for ${tech}.`); + continue; + } + + const sourceFile = project.createSourceFile(`temp_${tech}.ts`, generatedCode, { overwrite: true }); + const { total: score, breakdown } = analyzeAST(sourceFile, tech); + + console.log(`Fidelity Score for ${file}: ${score}%`); + console.log(`Breakdown:`, breakdown); + + if (score >= 95) { + console.log(`✅ Validation passed for ${file}. Updating badge and auto-committing.`); + + let content = fs.readFileSync(file, 'utf-8'); + if (!content.includes('[![Vibe-Coding Verified]')) { + content = content.replace(/^# /, '[![Vibe-Coding Verified](https://img.shields.io/badge/Vibe--Coding-Verified-brightgreen?style=for-the-badge)](#)\n\n# '); + fs.writeFileSync(file, content); + } + + try { + execSync(`git add ${file}`); + // Only commit if there are changes (badge might already be there) + const status = execSync('git status --porcelain', { encoding: 'utf-8' }); + if (status.includes(file)) { + execSync(`git commit -m "chore: fidelity-pass for ${file}"`); + execSync(`git push origin HEAD:main`); + } else { + console.log(`Badge already present in ${file}, skipping commit.`); + } + } catch (err) { + console.error('Failed to commit or push:', err.message); + } + + } else { + console.error(`❌ Validation failed for ${file}. Score below 95%.`); + + const reportDir = path.join('benchmarks', 'logs'); + if (!fs.existsSync(reportDir)) fs.mkdirSync(reportDir, { recursive: true }); + + const reportPath = path.join(reportDir, `violation-${tech}-${Date.now()}.md`); + const reportContent = `# Critical Violation Report\n\nFile: ${file}\nFidelity Score: ${score}%\nThreshold: 95%\nBreakdown: ${JSON.stringify(breakdown)}\n\nGenerated Code:\n\`\`\`typescript\n${generatedCode}\n\`\`\`\n\nReview the AST rules.`; + + fs.writeFileSync(reportPath, reportContent); + console.log(`Generated violation report: ${reportPath}`); + + try { + execSync(`gh issue create --title "Fidelity Gap: ${file}" --body-file ${reportPath}`); + console.log(`Created GitHub Issue for ${file}`); + } catch (err) { + console.error('Failed to create GitHub Issue (gh cli might not be installed or authenticated):', err.message); + } + + process.exitCode = 1; + } } }