diff --git a/.github/scripts/utils.js b/.github/scripts/utils.js index 3d8161e..e63d58a 100644 --- a/.github/scripts/utils.js +++ b/.github/scripts/utils.js @@ -2,7 +2,6 @@ import crypto from 'crypto'; import 'dotenv/config'; import { writeFile } from 'fs'; import { join } from 'path'; -import crypto from 'crypto'; export async function saveImage(bytes, filename = randomText() + '.png') { const localPath = join(process.cwd(), filename); diff --git a/README.md b/README.md index b2a5f6c..cdeb1ee 100644 --- a/README.md +++ b/README.md @@ -193,6 +193,7 @@ graph TD * 🤖 [vibe-coding-autonomous-testing-patterns.md](./docs/vibe-coding-autonomous-testing-patterns.md) * 🤖 [vibe-coding-deterministic-patterns.md](./docs/vibe-coding-deterministic-patterns.md) * 🤖 [vibe-coding-multi-agent-state-sync.md](./docs/vibe-coding-multi-agent-state-sync.md) + * 🤖 [vibe-coding-telemetry-patterns.md](./docs/vibe-coding-telemetry-patterns.md) * ⚡ [vibe-coding-zero-approval-workflows.md](./docs/vibe-coding-zero-approval-workflows.md) * 🖥️ **[frontend/](./frontend/)** * 📄 [readme.md](./frontend/readme.md) @@ -407,6 +408,7 @@ graph TD * 🤖 [vibe-coding-autonomous-testing-patterns.md](./docs/vibe-coding-autonomous-testing-patterns.md) * 🤖 [vibe-coding-deterministic-patterns.md](./docs/vibe-coding-deterministic-patterns.md) * 🤖 [vibe-coding-multi-agent-state-sync.md](./docs/vibe-coding-multi-agent-state-sync.md) + * 🤖 [vibe-coding-telemetry-patterns.md](./docs/vibe-coding-telemetry-patterns.md) * ⚡ [vibe-coding-zero-approval-workflows.md](./docs/vibe-coding-zero-approval-workflows.md) * 🖥️ **[frontend/](./frontend/)** * 📄 [readme.md](./frontend/readme.md) diff --git a/docs/vibe-coding-telemetry-patterns.md b/docs/vibe-coding-telemetry-patterns.md new file mode 100644 index 0000000..6ac5d08 --- /dev/null +++ b/docs/vibe-coding-telemetry-patterns.md @@ -0,0 +1,150 @@ +--- +technology: TypeScript +domain: Documentation +level: Senior/Architect +version: Latest +tags: [vibe coding, ai agents, telemetry, 2026 trends, deterministic patterns, monitoring, observability] +ai_role: Senior TypeScript Architect +last_updated: 2026-04-07 +--- + +> 📦 [best-practise](../README.md) / 📄 [docs](./) + +# 📡 Vibe Coding Telemetry Patterns: AI Agent Observability + +In the paradigm of **Vibe Coding** and AI Agent Orchestration, traditional logging is insufficient. Autonomous agents operating at scale require **Deterministic Telemetry pipelines** to track intent, state transformations, and hallucination bounds in real-time. This document outlines the 2026 architectural standards for Agent Observability. + +## 🌟 The Need for Deterministic Telemetry + +When multi-agent systems interact asynchronously, standard debugging becomes obsolete. We must shift from "What went wrong?" to "Why did the agent decide to execute this state mutation?". Telemetry for Vibe Coding necessitates structured, machine-parseable data streams. + +### Core Paradigms for 2026 + +1. **Intent-Based Logging:** Logs must capture the LLM's *reasoning* alongside the action. +2. **Context Snapshots:** Capture immutable snapshots of the agent's context window at the time of execution. +3. **Hallucination Detection:** Implement automated bounds-checking and record semantic deviations. + +> [!IMPORTANT] +> To maintain deterministic observability, telemetry streams must strictly serialize LLM context windows to prevent parsing exceptions downstream. + +--- + +## 🏗️ Visual Architecture: Telemetry Data Flow + +```mermaid +graph TD + A([🤖 AI Agent]) -->|Executes Action| B{Telemetry Middleware} + B -->|Logs Intent & Context| C[(Distributed Tracing Store)] + B -->|Detects Hallucination| D[Alerting Thresholds] + D -->|Fails Bounds Check| E[Self-Healing Revert] + C -->|Aggregated Metrics| F([📊 Dashboard / Architect]) +``` + +--- + +## 🔄 The Pattern Lifecycle: Agent Telemetry + +### ❌ Bad Practice + +```typescript +// Relying on unstructured, human-centric logging for AI Agents +async function generateComponent(prompt: string) { + console.log(`Starting to generate component with prompt: ${prompt}`); + try { + const response = await ai.generate(prompt); + console.log("Component generated successfully!"); + return response; + } catch (error: any) { + console.error("Agent failed to generate component:", error.message); + throw error; + } +} +``` + +### ⚠️ Problem + +1. **Unstructured Data:** `console.log` produces string-based logs that are impossible for other AI agents to parse for self-healing or performance tuning. +2. **Missing Context:** The log lacks critical data such as model version, temperature, token usage, and system prompt constraints. +3. **Type Safety Risk:** Using `error: any` bypasses TypeScript's safety mechanisms, leading to unpredictable runtime crashes if the `error` object lacks a `message` property. + +### ✅ Best Practice + +```typescript +// Deterministic, structured telemetry with strict type safety +import { trace } from '@opentelemetry/api'; + +interface AgentTelemetryPayload { + agentId: string; + model: string; + intent: string; + tokenUsage: number; + success: boolean; + contextSnapshotId: string; +} + +async function generateComponent(prompt: string, contextId: string): Promise { + const tracer = trace.getTracer('vibe-coding-agent'); + return tracer.startActiveSpan('generateComponent', async (span) => { + try { + span.setAttribute('agent.intent', 'ui_generation'); + span.setAttribute('agent.context_id', contextId); + + const response = await ai.generate(prompt); + + const payload: AgentTelemetryPayload = { + agentId: 'ui-architect-01', + model: 'gpt-4.5-turbo', + intent: 'generate_react_component', + tokenUsage: response.usage.totalTokens, + success: true, + contextSnapshotId: contextId + }; + + span.addEvent('agent_execution_complete', payload as unknown as Record); + span.end(); + return response.text; + } catch (error: unknown) { + span.setAttribute('error', true); + if (error instanceof Error) { + span.recordException(error); + span.setAttribute('error.message', error.message); + } else { + span.setAttribute('error.message', 'Unknown agent hallucination or failure'); + } + span.end(); + throw error; + } + }); +} +``` + +### 🚀 Solution + +1. **Structured Tracing:** Utilizing OpenTelemetry allows AI Agents and orchestration platforms to systematically query the execution graph and understand exactly where a logic branch failed. +2. **Type-Safe Error Handling:** By utilizing `unknown` and an explicit `instanceof Error` check, we guarantee runtime safety and prevent secondary crashes during error handling. +3. **Contextual Immutability:** Recording the `contextSnapshotId` ensures we can perfectly recreate the agent's exact state at the time of execution for deterministic debugging. + +> [!NOTE] +> Ensure that your OpenTelemetry collector is configured to accept custom metrics related to token consumption and hallucination scores. + +--- + +## 📊 Telemetry Component Responsibility + +| Component | Responsibility | Action on Failure | +| :--- | :--- | :--- | +| **Span Attribute** | Defines the exact operational bounds and intent. | Agent context is marked as 'unbounded'. | +| **Event Payload** | Captures structured data regarding token usage and model state. | Metrics ingestion drops the malformed packet. | +| **Exception Record** | Safely captures stack traces and LLM semantic failures. | Triggers the agent self-healing rollback. | + +--- + +## ✅ Actionable Checklist for Telemetry Integration + +- [ ] Replace all `console.log` statements within Agent logic with structured OpenTelemetry spans. +- [ ] Ensure every LLM interaction logs token usage and model metadata. +- [ ] Implement strict type guards (`error: unknown`) for all external API calls. +- [ ] Record a `contextSnapshotId` for every multi-turn agent interaction. +- [ ] Set up automated alerting for threshold breaches (e.g., token limits exceeded, repeated hallucinations). + +[Back to Top](#)