From d08cb78b5a93fd5215b052874895846f4f7ec301 Mon Sep 17 00:00:00 2001 From: Daniel Paulus Date: Mon, 6 Jul 2026 09:20:20 +0200 Subject: [PATCH] fix(reporters): don't render green assertions on a request error; add gRPC/SSL detail [SIM-286] When a check result carries a requestError, no assertion was actually evaluated, yet the test reporter still printed every assertion as a green check for SSL/gRPC/traceroute (unlike API/DNS/TCP/ICMP). Suppress the Assertions block when requestError is set. Also: humanize gRPC assertion source labels, render gRPC response time, and add an explicit SSL response-time reason line plus a security-baseline verdict line. Co-Authored-By: Claude Opus 4.8 Claude-Session: https://claude.ai/code/session_01NC9pESYE5wGqy5TKRzufyc --- .../__tests__/__snapshots__/util.spec.ts.snap | 5 +- .../fixtures/uptime-check-results.ts | 4 ++ .../cli/src/reporters/__tests__/util.spec.ts | 66 +++++++++++++++++++ packages/cli/src/reporters/util.ts | 56 +++++++++++++--- 4 files changed, 120 insertions(+), 11 deletions(-) diff --git a/packages/cli/src/reporters/__tests__/__snapshots__/util.spec.ts.snap b/packages/cli/src/reporters/__tests__/__snapshots__/util.spec.ts.snap index 9a6ff4f78..286a4b9b9 100644 --- a/packages/cli/src/reporters/__tests__/__snapshots__/util.spec.ts.snap +++ b/packages/cli/src/reporters/__tests__/__snapshots__/util.spec.ts.snap @@ -253,8 +253,10 @@ Resolved IP: 203.0.113.30 TLS: TLS 1.3 / TLS_AES_256_GCM_SHA384 Expires in: expired 5 day(s) ago Handshake: 48.200ms +Response time 48.200ms exceeded max 40.000ms Chain Trusted: no -Hostname Verified: no" +Hostname Verified: no +Baseline: fail (grade F)" `; exports[`formatCheckResult() > Traceroute Check result > formats a failing Traceroute Check result > traceroute-check-result-format 1`] = ` @@ -275,6 +277,7 @@ exports[`formatCheckResult() > gRPC Check result > formats a failing gRPC Check Target: grpc.example.com:443 Method: grpc.health.v1.Health/Check Mode: HEALTH +Response Time: 90ms Status: 14 connection refused Health: NOT_SERVING Discovered Methods: grpc.health.v1.Health/Check, grpc.health.v1.Health/Watch diff --git a/packages/cli/src/reporters/__tests__/fixtures/uptime-check-results.ts b/packages/cli/src/reporters/__tests__/fixtures/uptime-check-results.ts index 4ad4c6742..b447640ca 100644 --- a/packages/cli/src/reporters/__tests__/fixtures/uptime-check-results.ts +++ b/packages/cli/src/reporters/__tests__/fixtures/uptime-check-results.ts @@ -77,6 +77,7 @@ export const grpcCheckResult = { healthStatusLabel: 'NOT_SERVING', metadata: [{ key: 'content-type', value: 'application/grpc' }], discoveredMethods: ['grpc.health.v1.Health/Check', 'grpc.health.v1.Health/Watch'], + timingPhases: { dns: 1, connect: 2, total: 90 }, }, }, logs: [], @@ -104,6 +105,8 @@ export const sslCheckResult = { responseTime: 48, checkRunData: { requestError: null, + degradedResponseTime: 20, + maxResponseTime: 40, assertions: [], response: { resolvedIp: '203.0.113.30', @@ -114,6 +117,7 @@ export const sslCheckResult = { chainTrusted: false, daysUntilExpiry: -5, ocspStapled: false, + securityBaseline: { verdict: 'fail', grade: 'F' }, }, }, logs: [], diff --git a/packages/cli/src/reporters/__tests__/util.spec.ts b/packages/cli/src/reporters/__tests__/util.spec.ts index 004ea5a47..2fa4d4596 100644 --- a/packages/cli/src/reporters/__tests__/util.spec.ts +++ b/packages/cli/src/reporters/__tests__/util.spec.ts @@ -163,6 +163,42 @@ describe('formatCheckResult()', () => { expect(output).toContain('Health: NOT_SERVING') expect(output).toContain('grpc.health.v1.Health/Watch') }) + it('renders the response time from timingPhases.total', () => { + const output = stripAnsi(formatCheckResult(grpcCheckResult)) + expect(output).toContain('Response Time: 90ms') + }) + it('humanizes gRPC assertion sources', () => { + const withAssertions = { + ...grpcCheckResult, + checkRunData: { + ...grpcCheckResult.checkRunData, + assertions: [ + { source: 'GRPC_STATUS_CODE', comparison: 'EQUALS', property: '', regex: null, target: '0', error: 'boom', actual: 14 }, + { source: 'GRPC_HEALTHCHECK_STATUS', comparison: 'EQUALS', property: '', regex: null, target: 'SERVING', error: 'boom', actual: 'NOT_SERVING' }, + ], + }, + } + const output = stripAnsi(formatCheckResult(withAssertions)) + expect(output).toContain('status code') + expect(output).toContain('health check status') + expect(output).not.toContain('GRPC_STATUS_CODE') + }) + it('suppresses the Assertions block when a requestError is set', () => { + const withError = { + ...grpcCheckResult, + checkRunData: { + ...grpcCheckResult.checkRunData, + requestError: 'connection refused', + assertions: [ + { source: 'GRPC_STATUS_CODE', comparison: 'EQUALS', property: '', regex: null, target: '0', error: '', actual: 14 }, + ], + }, + } + const output = stripAnsi(formatCheckResult(withError)) + expect(output).toContain('Request Error') + expect(output).not.toContain('Assertions') + expect(output).not.toContain('status code') + }) }) describe('SSL Check result', () => { it('formats a failing SSL Check result', () => { @@ -177,6 +213,36 @@ describe('formatCheckResult()', () => { expect(output).toContain('Chain Trusted: no') expect(output).toContain('Hostname Verified: no') }) + it('explains a response-time failure and renders the baseline verdict', () => { + const output = stripAnsi(formatCheckResult(sslCheckResult)) + // handshakeTimeMs 48.2 > maxResponseTime 40 + expect(output).toContain('Response time 48.200ms exceeded max 40.000ms') + expect(output).toContain('Baseline: fail (grade F)') + }) + it('explains a degraded response-time verdict when only degraded is exceeded', () => { + const degraded = { + ...sslCheckResult, + checkRunData: { + ...sslCheckResult.checkRunData, + degradedResponseTime: 40, + maxResponseTime: 60, + }, + } + const output = stripAnsi(formatCheckResult(degraded)) + expect(output).toContain('Response time 48.200ms exceeded degraded 40.000ms') + }) + it('omits the response-time reason when thresholds are not exceeded', () => { + const ok = { + ...sslCheckResult, + checkRunData: { + ...sslCheckResult.checkRunData, + degradedResponseTime: 100, + maxResponseTime: 200, + }, + } + const output = stripAnsi(formatCheckResult(ok)) + expect(output).not.toContain('exceeded') + }) }) }) diff --git a/packages/cli/src/reporters/util.ts b/packages/cli/src/reporters/util.ts index feeb852a1..48aa8bbe6 100644 --- a/packages/cli/src/reporters/util.ts +++ b/packages/cli/src/reporters/util.ts @@ -95,7 +95,7 @@ export function formatCheckResult (checkResult: any) { formatHttpResponse(checkResult.checkRunData.response), ]) } - if (checkResult.checkRunData?.assertions?.length) { + if (!checkResult.checkRunData?.requestError && checkResult.checkRunData?.assertions?.length) { result.push([ formatSectionTitle('Assertions'), formatAssertions(checkResult.checkRunData.assertions), @@ -146,7 +146,7 @@ export function formatCheckResult (checkResult: any) { formatDnsResponse(checkResult.checkRunData.response), ]) } - if (checkResult.checkRunData?.assertions?.length) { + if (!checkResult.checkRunData?.requestError && checkResult.checkRunData?.assertions?.length) { result.push([ formatSectionTitle('Assertions'), formatAssertions(checkResult.checkRunData.assertions, { @@ -172,7 +172,7 @@ export function formatCheckResult (checkResult: any) { formatConnectionError(checkResult.checkRunData?.response?.error), ]) } - if (checkResult.checkRunData?.assertions?.length) { + if (!checkResult.checkRunData?.requestError && checkResult.checkRunData?.assertions?.length) { result.push([ formatSectionTitle('Assertions'), formatAssertions(checkResult.checkRunData.assertions), @@ -253,7 +253,7 @@ export function formatCheckResult (checkResult: any) { ]) } } - if (checkResult.checkRunData?.assertions?.length) { + if (!checkResult.checkRunData?.requestError && checkResult.checkRunData?.assertions?.length) { result.push([ formatSectionTitle('Assertions'), formatAssertions(checkResult.checkRunData.assertions), @@ -278,7 +278,7 @@ export function formatCheckResult (checkResult: any) { ]) } } - if (checkResult.checkRunData?.assertions?.length) { + if (!checkResult.checkRunData?.requestError && checkResult.checkRunData?.assertions?.length) { result.push([ formatSectionTitle('Assertions'), formatAssertions(checkResult.checkRunData.assertions), @@ -294,7 +294,10 @@ export function formatCheckResult (checkResult: any) { } else if (checkResult.checkRunData?.response) { result.push([ formatSectionTitle('SSL Response'), - formatSslResponse(checkResult.checkRunData.response), + formatSslResponse(checkResult.checkRunData.response, { + degradedResponseTime: checkResult.checkRunData.degradedResponseTime, + maxResponseTime: checkResult.checkRunData.maxResponseTime, + }), ]) if (checkResult.checkRunData?.response?.error) { result.push([ @@ -303,7 +306,7 @@ export function formatCheckResult (checkResult: any) { ]) } } - if (checkResult.checkRunData?.assertions?.length) { + if (!checkResult.checkRunData?.requestError && checkResult.checkRunData?.assertions?.length) { result.push([ formatSectionTitle('Assertions'), formatAssertions(checkResult.checkRunData.assertions), @@ -335,7 +338,7 @@ export function formatCheckResult (checkResult: any) { formatConnectionError(checkResult.checkRunData?.response?.error), ]) } - if (checkResult.checkRunData?.assertions?.length) { + if (!checkResult.checkRunData?.requestError && checkResult.checkRunData?.assertions?.length) { result.push([ formatSectionTitle('Assertions'), formatAssertions(checkResult.checkRunData.assertions), @@ -376,6 +379,10 @@ const assertionSources: any = { RESPONSE_CODE: 'response code', LATENCY: 'latency', JSON_RESPONSE: 'response data (JSON)', + GRPC_STATUS_CODE: 'status code', + GRPC_HEALTHCHECK_STATUS: 'health check status', + GRPC_RESPONSE: 'response message', + GRPC_METADATA: 'metadata', } const assertionComparisons: any = { @@ -717,10 +724,14 @@ function formatGrpcResponse (response: any) { const methods = Array.isArray(response.discoveredMethods) ? response.discoveredMethods : [] const metadata = Array.isArray(response.metadata) ? response.metadata : [] const metadataLines = metadata.slice(0, 20).map((m: any) => ` ${m.key ?? m.name ?? '(key)'}: ${m.value ?? ''}`) + // The go-runner reports gRPC response time as `timingPhases.total` (ms); fall + // back to a flat `responseTime` if a future artifact exposes one. + const responseTime = response.timingPhases?.total ?? response.responseTime return [ target ? `Target: ${target}` : undefined, response.grpcMethod ? `Method: ${response.grpcMethod}` : undefined, response.grpcMode ? `Mode: ${response.grpcMode}` : undefined, + typeof responseTime === 'number' ? `Response Time: ${formatDuration(responseTime)}` : undefined, response.grpcStatusCode != null ? `Status: ${response.grpcStatusCode}${response.grpcStatusMessage ? ` ${response.grpcStatusMessage}` : ''}` : undefined, @@ -732,8 +743,31 @@ function formatGrpcResponse (response: any) { ].filter(Boolean).join('\n') } -function formatSslResponse (response: any) { +type SslResponseTimeLimits = { + degradedResponseTime?: number + maxResponseTime?: number +} + +function formatSslResponse (response: any, limits?: SslResponseTimeLimits) { const days = response.daysUntilExpiry + const handshake = response.handshakeTimeMs + // Explain a response-time fail/degraded verdict. The thresholds live on the + // enclosing checkRunData (not the response artifact), so they are passed in; + // when they are absent we simply skip the reason line. + let responseTimeReason + if (typeof handshake === 'number') { + if (limits?.maxResponseTime != null && handshake > limits.maxResponseTime) { + responseTimeReason = `Response time ${formatLatency(handshake)} exceeded max ${formatLatency(limits.maxResponseTime)}` + } else if (limits?.degradedResponseTime != null && handshake > limits.degradedResponseTime) { + responseTimeReason = `Response time ${formatLatency(handshake)} exceeded degraded ${formatLatency(limits.degradedResponseTime)}` + } + } + // Render the security-baseline verdict/grade when the runner provides it. + const baseline = response.securityBaseline + let baselineLine + if (baseline && (baseline.verdict || baseline.grade)) { + baselineLine = `Baseline: ${[baseline.verdict, baseline.grade ? `(grade ${baseline.grade})` : ''].filter(Boolean).join(' ')}` + } return [ response.resolvedIp ? `Resolved IP: ${response.resolvedIp}` : undefined, response.protocol || response.cipherSuite @@ -742,9 +776,11 @@ function formatSslResponse (response: any) { typeof days === 'number' ? `Expires in: ${days < 0 ? `expired ${-days} day(s) ago` : `${days} day(s)`}` : undefined, - typeof response.handshakeTimeMs === 'number' ? `Handshake: ${formatLatency(response.handshakeTimeMs)}` : undefined, + typeof handshake === 'number' ? `Handshake: ${formatLatency(handshake)}` : undefined, + responseTimeReason, response.chainTrusted != null ? `Chain Trusted: ${response.chainTrusted ? 'yes' : 'no'}` : undefined, response.hostnameVerified != null ? `Hostname Verified: ${response.hostnameVerified ? 'yes' : 'no'}` : undefined, + baselineLine, ].filter(Boolean).join('\n') }