From f77b2c798469bd9807de0c2ee01dfa3a64de542d Mon Sep 17 00:00:00 2001 From: Kyle Hounslow Date: Mon, 27 Jul 2026 11:31:48 -0700 Subject: [PATCH] feat(cli): warn on VPC security-group rules that break the data path, add e2e harness Users reported a silent failure on the VPC install: a security group with no egress rule left the demo EC2 instance unable to send anything out, so no data reached OpenSearch. The stack built clean and stayed empty, no error. Add checkSecurityGroupRules (in aws.mjs, called from executePipeline after VPC topology validation): describe the stack's security groups and warn about rules that break the data path, before the ~30-minute build. It validates existing bring-your-own groups rather than creating or modifying one, and warns without blocking (traffic may route via other groups, a NAT, or NACLs the CLI can't see). Checks intra-VPC egress 443, internet egress 443 to 0.0.0.0/0 when a demo launches, and intra-VPC ingress 443. Degrades to a single manual-verify warning when the role lacks ec2:DescribeSecurityGroups. Add a local AWS e2e harness under test/e2e/ that creates a full stack against a real account, drives telemetry through it, verifies documents land via the OpenSearch Application endpoint, and tears it down. Covers managed/serverless, public/VPC, and demo/no-demo. Not run in CI (no AWS credentials); live scenarios are opt-in and VPC scenarios skip when their env vars are unset. 41 harness unit tests run with the src suite (134 total). Signed-off-by: Kyle Hounslow --- aws/cli-installer/package.json | 5 +- aws/cli-installer/src/aws.mjs | 156 ++++++++++ aws/cli-installer/src/main.mjs | 9 + aws/cli-installer/test/e2e/README.md | 150 +++++++++ aws/cli-installer/test/e2e/e2e-unit.test.mjs | 302 +++++++++++++++++++ aws/cli-installer/test/e2e/run.mjs | 227 ++++++++++++++ aws/cli-installer/test/e2e/scenarios.mjs | 192 ++++++++++++ aws/cli-installer/test/e2e/verify.mjs | 238 +++++++++++++++ aws/cli-installer/test/unit.test.mjs | 161 ++++++++++ 9 files changed, 1439 insertions(+), 1 deletion(-) create mode 100644 aws/cli-installer/test/e2e/README.md create mode 100644 aws/cli-installer/test/e2e/e2e-unit.test.mjs create mode 100644 aws/cli-installer/test/e2e/run.mjs create mode 100644 aws/cli-installer/test/e2e/scenarios.mjs create mode 100644 aws/cli-installer/test/e2e/verify.mjs diff --git a/aws/cli-installer/package.json b/aws/cli-installer/package.json index 64d0952a..0964ee0e 100644 --- a/aws/cli-installer/package.json +++ b/aws/cli-installer/package.json @@ -6,7 +6,10 @@ "bin": "./bin/cli-installer.mjs", "scripts": { "start": "node bin/open-stack.mjs", - "test": "node --test test/unit.test.mjs" + "test": "node --test test/unit.test.mjs test/e2e/e2e-unit.test.mjs", + "test:unit": "node --test test/unit.test.mjs test/e2e/e2e-unit.test.mjs", + "e2e": "node test/e2e/run.mjs", + "e2e:list": "node test/e2e/run.mjs --list" }, "repository": { "type": "git", diff --git a/aws/cli-installer/src/aws.mjs b/aws/cli-installer/src/aws.mjs index d78f2a31..13d15367 100644 --- a/aws/cli-installer/src/aws.mjs +++ b/aws/cli-installer/src/aws.mjs @@ -1718,6 +1718,162 @@ export async function validateVpcTopology(cfg, deps = {}) { return errors; } +// ── Security-group rule analysis (best-effort data-path check) ─────────────── + +/** + * Does a permission entry cover TCP `port`? IpProtocol '-1' means all protocols + * and all ports. + */ +function permCoversPort(p, port) { + const proto = p.IpProtocol; + return proto === '-1' || (proto === 'tcp' && (p.FromPort ?? 0) <= port && (p.ToPort ?? 65535) >= port); +} + +/** + * Does a set of IpPermissions entries allow TCP `port` from/to an intra-stack + * source? A source counts as intra-stack when the rule references one of the + * stack's own security groups (`selfIds`) or a CIDR not narrower than the VPC + * (0.0.0.0/0, or a VPC-CIDR entry). + */ +export function permissionsAllowPort(permissions, port, { selfIds = [], vpcCidrs = [] } = {}) { + const selfSet = new Set(selfIds); + const cidrOk = (cidr) => cidr === '0.0.0.0/0' || vpcCidrs.includes(cidr); + for (const p of permissions || []) { + if (!permCoversPort(p, port)) continue; + if ((p.UserIdGroupPairs || []).some((g) => selfSet.has(g.GroupId))) return true; + if ((p.IpRanges || []).some((r) => cidrOk(r.CidrIp))) return true; + } + return false; +} + +/** + * Does a set of IpPermissions entries allow TCP `port` to the public internet + * (a 0.0.0.0/0 IpRanges entry)? A self-referencing SG rule or a VPC-scoped CIDR + * does NOT count; those keep traffic inside the VPC. The demo instance needs + * this for bootstrap (dnf, GitHub, container registries), which does not route + * through the VPC-private OSIS endpoint. + */ +export function permissionsAllowInternet(permissions, port) { + for (const p of permissions || []) { + if (!permCoversPort(p, port)) continue; + if ((p.IpRanges || []).some((r) => r.CidrIp === '0.0.0.0/0')) return true; + } + return false; +} + +/** + * Analyze the security groups the stack attaches to the EC2 demo, OSIS pipeline, + * and OpenSearch domain, and return WARNING strings for rules that would silently + * break the data path. Pure: takes the DescribeSecurityGroups result so it is + * unit-testable. Warnings only, never hard-fails: a user's network may route the + * same traffic via other groups, a NAT, or NACLs we can't see. + * + * All three components share `groupIds` (see ec2-demo.mjs / createOsiPipeline / + * createOpenSearch). There are two distinct egress needs, only one of which is + * intra-VPC: + * - Intra-VPC egress 443 (always): OSIS reaches the domain; with a demo, the + * demo's collector reaches the VPC-private OSIS ingest endpoint. Satisfied by + * a self-reference, a VPC-CIDR rule, or 0.0.0.0/0. + * - Internet egress 443 (only with a demo): the instance bootstraps over the + * public internet (dnf, GitHub clone, container image pulls). This traffic + * does NOT flow through the VPC-private endpoint, so only a 0.0.0.0/0 egress + * rule (to a NAT/IGW) satisfies it; a VPC-CIDR-scoped rule does not. + * Plus intra-VPC ingress 443 (always): OSIS accepts OTLP from the demo and the + * domain accepts requests from OSIS. + * + * @param {object[]} groups DescribeSecurityGroups result + * @param {object} opts + * @param {string[]} opts.groupIds the stack's SG ids (self-reference set) + * @param {string[]} [opts.vpcCidrs] the VPC's CIDR block(s) + * @param {boolean} [opts.demo=true] whether an EC2 demo will launch (adds the + * internet-egress requirement); pass false for --skip-demo + */ +export function analyzeSecurityGroupRules(groups, { groupIds = [], vpcCidrs = [], demo = true } = {}) { + const warnings = []; + const ids = groupIds.length ? groupIds : (groups || []).map((g) => g.GroupId); + const opts = { selfIds: ids, vpcCidrs }; + const list = groups || []; + + const intraVpcEgress443 = list.some((g) => permissionsAllowPort(g.IpPermissionsEgress, 443, opts)); + const internetEgress443 = list.some((g) => permissionsAllowInternet(g.IpPermissionsEgress, 443)); + const intraVpcIngress443 = list.some((g) => permissionsAllowPort(g.IpPermissions, 443, opts)); + + if (!intraVpcEgress443) { + warnings.push( + `No outbound (egress) rule allowing TCP 443 within the VPC was found on ${ids.join(', ')}. ` + + `OSIS cannot reach the OpenSearch domain${demo ? ' and the demo collector cannot reach the OSIS ingest endpoint' : ''}, so no telemetry will appear. ` + + `Add an egress rule allowing TCP 443 to these security groups themselves (self-reference), the VPC CIDR, or 0.0.0.0/0.` + ); + } + if (demo && !internetEgress443) { + warnings.push( + `No outbound (egress) rule allowing TCP 443 to the internet (0.0.0.0/0) was found on ${ids.join(', ')}. ` + + `The EC2 demo bootstraps over the public internet (package install, GitHub clone, container image pulls); that traffic does not use the VPC-private endpoint, so a VPC-scoped egress rule is not enough. ` + + `Add an egress rule allowing TCP 443 to 0.0.0.0/0 (reachable via a NAT gateway for private subnets, or an internet gateway for public ones).` + ); + } + if (!intraVpcIngress443) { + warnings.push( + `No inbound (ingress) rule allowing TCP 443 from within the VPC was found on ${ids.join(', ')}. ` + + `OSIS will reject OTLP${demo ? ' from the demo' : ''} and the domain will reject requests from OSIS. ` + + `Add an ingress rule allowing TCP 443 from these security groups themselves (self-reference) or from the VPC CIDR.` + ); + } + return warnings; +} + +/** + * Best-effort check of the stack's security-group rules against the data path. + * Returns WARNING strings (never throws, never blocks the run). When the caller + * lacks ec2:DescribeSecurityGroups, returns a single warning telling the user to + * verify the rules manually, since we can't inspect them. + * + * @param {object} cfg resolved config (needs region, vpcId, securityGroupIds) + * @param {object} [deps] optional injected accessors for testing + * @returns {Promise} + */ +export async function checkSecurityGroupRules(cfg, deps = {}) { + if (!cfg.vpcId || !(cfg.securityGroupIds || []).length) return []; + + const demo = !cfg.skipDemo; + + const describeSecurityGroups = deps.describeSecurityGroups || (async (region, ids) => { + const { EC2Client, DescribeSecurityGroupsCommand } = await import('@aws-sdk/client-ec2'); + const client = new EC2Client({ region }); + return (await client.send(new DescribeSecurityGroupsCommand({ GroupIds: ids }))).SecurityGroups || []; + }); + const describeVpcs = deps.describeVpcs || (async (region, ids) => { + const { EC2Client, DescribeVpcsCommand } = await import('@aws-sdk/client-ec2'); + const client = new EC2Client({ region }); + return (await client.send(new DescribeVpcsCommand({ VpcIds: ids }))).Vpcs || []; + }); + + const ids = cfg.securityGroupIds; + let groups; + try { + groups = await describeSecurityGroups(cfg.region, ids); + } catch (err) { + if (/UnauthorizedOperation|AccessDenied|not authorized/i.test(err.message || err.name || '')) { + return [ + `Could not verify security-group rules (the current role lacks ec2:DescribeSecurityGroups). ` + + `Ensure ${ids.join(', ')} allow outbound TCP 443 within the VPC (OSIS to domain${demo ? ', demo collector to OSIS ingest' : ''}), ` + + `inbound TCP 443 from within the VPC (OSIS from ${demo ? 'demo, ' : ''}domain from OSIS)` + + `${demo ? ', and outbound TCP 443 to 0.0.0.0/0 for the demo to bootstrap over the internet' : ''}, or telemetry will not appear.`, + ]; + } + return [`Could not verify security-group rules: ${err.message}`]; + } + + let vpcCidrs = []; + try { + const vpcs = await describeVpcs(cfg.region, [cfg.vpcId]); + vpcCidrs = (vpcs[0]?.CidrBlockAssociationSet || []).map((a) => a.CidrBlock).filter(Boolean); + if (!vpcCidrs.length && vpcs[0]?.CidrBlock) vpcCidrs = [vpcs[0].CidrBlock]; + } catch { /* VPC CIDR is a refinement; self-reference checks still work without it */ } + + return analyzeSecurityGroupRules(groups, { groupIds: ids, vpcCidrs, demo }); +} + // ── Pipeline listing / describe / update ───────────────────────────────────── /** diff --git a/aws/cli-installer/src/main.mjs b/aws/cli-installer/src/main.mjs index 3a62f0a4..af291159 100644 --- a/aws/cli-installer/src/main.mjs +++ b/aws/cli-installer/src/main.mjs @@ -15,11 +15,13 @@ import { createConnectedDataSource, createOpenSearchApplication, validateVpcTopology, + checkSecurityGroupRules, } from './aws.mjs'; import { printError, printSuccess, printStep, + printWarning, printPanel, printBox, STAR, @@ -107,6 +109,13 @@ export async function executePipeline(cfg) { throw new Error('VPC configuration is invalid; no resources were created.'); } printSuccess('VPC, subnets, and security groups validated'); + + // Best-effort: the topology check above proves the SGs exist and belong to + // the VPC, but not that their rules permit the 443 data path. A stripped + // egress rule leaves the demo unable to send anything out: no error, just + // no data. Warn (never block) before the ~30-min build so it's caught early. + const sgWarnings = await checkSecurityGroupRules(cfg); + for (const w of sgWarnings) printWarning(w); console.error(); } diff --git a/aws/cli-installer/test/e2e/README.md b/aws/cli-installer/test/e2e/README.md new file mode 100644 index 00000000..aa3cba39 --- /dev/null +++ b/aws/cli-installer/test/e2e/README.md @@ -0,0 +1,150 @@ +# Local AWS end-to-end tests + +These tests exercise the CLI against **real AWS resources**: they create a full +stack (OpenSearch domain or serverless collection, OSIS pipeline, IAM roles, AMP +workspace, OpenSearch Application, and optionally an EC2 demo instance), drive +telemetry through it, confirm the data lands in OpenSearch, then tear everything +down. + +Because they need AWS credentials and cost real money/time, they are **not run in +CI**; GitHub Actions has no AWS credentials by policy. Run them locally against a +disposable sandbox account when you touch AWS-specific code (`aws.mjs`, +`ec2-demo.mjs`, `opensearch-ui-init.mjs`, `render.mjs`, VPC handling, destroy). + +> ⚠️ **Sandbox only.** The runner creates and **deletes** OpenSearch domains, +> OSIS pipelines, IAM roles, EC2 instances, and AMP workspaces. Every resource it +> touches is prefixed `e2e-`, but never point it at a production account. It +> prints the target account id up front so you can confirm before it proceeds. + +## What is covered + +The scenario matrix crosses the AWS-specific dimensions called out in the design: + +| Scenario | Backend | Network | Demo | Data-flow verified | Default | +|---|---|---|---|---|---| +| `managed-public-demo` | Managed domain | Public | EC2 OTel-demo | ✅ | ✅ | +| `managed-public-nodemo` | Managed domain | Public | Synthetic OTLP push | ✅ | ✅ | +| `serverless-public-nodemo` | Serverless (AOSS) | Public | Synthetic OTLP push | ✅ | ✅ | +| `managed-vpc-demo` | Managed domain | VPC-private | EC2 OTel-demo | ✅ | opt-in | +| `managed-vpc-nodemo` | Managed domain | VPC-private | (none) | create/destroy only | opt-in | + +- **Demo scenarios** launch the EC2 instance that runs the OTel demo + example + agents and generate real telemetry. In VPC mode the instance is placed **inside + the configured VPC**, so it can reach the VPC-private OSIS ingest endpoint. This + is what actually proves end-to-end VPC data flow. +- **No-demo (public) scenarios** pass `--skip-demo` and instead push a small + synthetic OTLP payload (logs + traces) straight at the OSIS ingest endpoint, so + they are much faster and cheaper while still proving the ingest to index path. +- **`managed-vpc-nodemo` is create/destroy only.** A VPC-attached pipeline's ingest + endpoint is VPC-private (RFC-1918), so a synthetic push from the developer's host + cannot reach it; data-flow verification is not applicable. This scenario still + exercises the full VPC create path (the CLI exits 0 only after the VPC-attached + pipeline reaches `ACTIVE`) and the teardown path. To verify actual VPC data flow, + use `managed-vpc-demo`, whose demo runs in-VPC. +- **VPC scenarios** are opt-in because they need pre-existing VPC infrastructure + (see below). Without the VPC env vars they are reported as `skipped`, not + failed, so the default matrix runs anywhere. + +When data flow **is** verified, the runner always queries through the managed +**OpenSearch UI (Application)** endpoint, which is reachable from outside the VPC +and proxies to the domain over the AWS-internal network, the same path the +installer uses. This is what lets a VPC-private *domain* be verified from a laptop +with no bastion or VPN. (The *ingest* endpoint, unlike the domain, is not proxied, +which is why VPC ingest can only be driven from inside the VPC.) + +## Prerequisites + +- Node.js 18+ (repo uses 22). +- AWS credentials for a sandbox account (`aws sts get-caller-identity` succeeds). +- Permissions for OpenSearch, OpenSearch Serverless, OSIS, AMP, IAM, EC2, SSM, + Secrets Manager, and Resource Groups Tagging. +- For VPC scenarios: a VPC with 1-3 private subnets in distinct AZs and a security + group that allows intra-VPC traffic (a group that allows traffic from itself + works well). + +## Running + +From `aws/cli-installer/`: + +```bash +# List the scenario matrix +npm run e2e:list + +# Run the default (public) scenarios in one region +AWS_PROFILE=my-sandbox npm run e2e -- --region us-east-1 + +# Run a single scenario +AWS_PROFILE=my-sandbox node test/e2e/run.mjs --scenario managed-public-nodemo --region us-east-1 + +# Include the VPC scenarios (requires the env vars below) +AWS_PROFILE=my-sandbox \ + E2E_VPC_ID=vpc-0123456789abcdef0 \ + E2E_SUBNET_IDS=subnet-aaa,subnet-bbb \ + E2E_SECURITY_GROUP_IDS=sg-0123456789abcdef0 \ + node test/e2e/run.mjs --all --region us-east-1 + +# Leave resources up for debugging (remember to destroy them yourself) +node test/e2e/run.mjs --scenario managed-public-demo --no-teardown --region us-east-1 +``` + +### Options + +| Flag | Meaning | +|---|---| +| `--region ` | AWS region (default `$AWS_REGION` or `us-east-1`) | +| `--scenario ` | Run only this scenario (repeatable) | +| `--all` | Include opt-in (VPC) scenarios | +| `--list` | Print the matrix and exit | +| `--no-teardown` | Skip destroy after the run (debugging) | +| `--data-timeout ` | Minutes to wait for data to land (default 20) | + +### Timing + +- Managed + demo: ~15 min to create, plus up to ~15 min for the EC2 demo to + bootstrap and telemetry to appear. +- Serverless / no-demo: a few minutes to create; synthetic data lands quickly + once ingest is warm. +- A freshly created (especially VPC-attached) OSIS pipeline can take ~10-15 min + after it reports `ACTIVE` before it actually accepts ingest; the runner keeps + retrying the synthetic push during that warmup, so give it the default timeout. + +The process exits non-zero if any scenario fails or errors, so it works as a +pass/fail gate in a local script. + +## Troubleshooting + +- **OSIS `CreatePipeline` fails with "Unable to create pipeline due to an internal + exception."** This is an AWS-side error, not a CLI or harness defect. It is + intermittent (observed on a serverless create that then passed unchanged on the + next run). Re-run the scenario; if it persists across several attempts, check the + OSIS service health for the region. +- **A run left resources behind (killed mid-run, interrupted, or `--no-teardown`).** + `destroy` is idempotent: it skips resources that no longer exist and deletes what + remains. Re-run it with the same pipeline name to finish cleanup: + ```bash + node bin/cli-installer.mjs destroy --pipeline-name --region + ``` + For a VPC scenario, resolve `` from the leftover domain/pipeline (the + runner names both after the pipeline). Managed-domain deletion is asynchronous, + so the domain may still list as `Deleted: true, Processing: true` for a while + after `destroy` returns; its VPC ENIs (described `ES `) release + automatically once deletion completes. + +## Unit tests (no AWS) + +The pure logic of the harness (the scenario matrix, CLI arg construction, +document-count evaluation, and synthetic OTLP payload building) is unit-tested in +`e2e-unit.test.mjs` and runs with the normal unit suite (no credentials needed): + +```bash +npm test # runs src unit tests + e2e harness unit tests +``` + +## Files + +| File | Purpose | +|---|---| +| `scenarios.mjs` | Scenario matrix + pure CLI arg construction | +| `verify.mjs` | Synthetic OTLP push + data-flow verification via the UI endpoint | +| `run.mjs` | Live runner: create → drive telemetry → verify → destroy per scenario | +| `e2e-unit.test.mjs` | Unit tests for the pure logic above | diff --git a/aws/cli-installer/test/e2e/e2e-unit.test.mjs b/aws/cli-installer/test/e2e/e2e-unit.test.mjs new file mode 100644 index 00000000..58d1f542 --- /dev/null +++ b/aws/cli-installer/test/e2e/e2e-unit.test.mjs @@ -0,0 +1,302 @@ +/** + * Unit tests for the PURE parts of the e2e harness — arg construction, the + * scenario matrix, count evaluation, and synthetic OTLP payloads. These run + * anywhere (no AWS, no credentials) and guard the harness logic itself. + * + * Run: node --test test/e2e/e2e-unit.test.mjs + * (Also picked up by `node --test` over the whole test/ tree.) + */ +import { describe, it } from 'node:test'; +import assert from 'node:assert/strict'; + +import { + SCENARIOS, getScenario, selectScenarios, envToVpc, + buildPipelineName, buildCreateArgs, buildDestroyArgs, +} from '../e2e/scenarios.mjs'; +import { + EXPECTED_INDICES, evaluateCounts, parseCount, buildOtlpPayload, otlpUrl, +} from '../e2e/verify.mjs'; + +// ── scenario matrix ─────────────────────────────────────────────────────────── + +describe('scenario matrix', () => { + it('covers both network topologies and both demo modes', () => { + assert.ok(SCENARIOS.some((s) => s.vpc), 'expected at least one VPC scenario'); + assert.ok(SCENARIOS.some((s) => !s.vpc), 'expected at least one public scenario'); + assert.ok(SCENARIOS.some((s) => s.demo), 'expected at least one demo scenario'); + assert.ok(SCENARIOS.some((s) => !s.demo), 'expected at least one no-demo scenario'); + }); + + it('covers both backends', () => { + assert.ok(SCENARIOS.some((s) => s.backend === 'managed')); + assert.ok(SCENARIOS.some((s) => s.backend === 'serverless')); + }); + + it('all scenario names are unique', () => { + const names = SCENARIOS.map((s) => s.name); + assert.equal(new Set(names).size, names.length); + }); + + it('VPC scenarios are opt-in (not default) so the base matrix runs anywhere', () => { + for (const s of SCENARIOS) { + if (s.vpc) assert.equal(s.enabledByDefault, false, `${s.name} should be opt-in`); + } + }); + + it('every scenario declares whether data flow is verified', () => { + for (const s of SCENARIOS) { + assert.equal(typeof s.verifyDataFlow, 'boolean', `${s.name} must set verifyDataFlow`); + } + }); + + it('the VPC no-demo scenario does not verify data flow (private ingest unreachable from outside)', () => { + // A VPC-attached OSIS ingest endpoint is VPC-private, so a synthetic push from + // the runner host can never reach it — verifying data flow there would be a + // guaranteed false failure. VPC data flow is proven by the in-VPC demo instead. + assert.equal(getScenario('managed-vpc-nodemo').verifyDataFlow, false); + assert.equal(getScenario('managed-vpc-demo').verifyDataFlow, true); + }); + + it('every public scenario verifies data flow', () => { + for (const s of SCENARIOS) { + if (!s.vpc) assert.equal(s.verifyDataFlow, true, `${s.name} (public) should verify data flow`); + } + }); +}); + +describe('getScenario / selectScenarios', () => { + it('getScenario returns the named scenario', () => { + assert.equal(getScenario('managed-public-demo').name, 'managed-public-demo'); + }); + it('getScenario throws on an unknown name', () => { + assert.throws(() => getScenario('nope'), /Unknown scenario/); + }); + it('selectScenarios defaults to enabled-by-default only', () => { + const sel = selectScenarios(); + assert.ok(sel.length > 0); + assert.ok(sel.every((s) => s.enabledByDefault)); + }); + it('selectScenarios --all includes opt-in scenarios', () => { + assert.equal(selectScenarios({ all: true }).length, SCENARIOS.length); + }); + it('selectScenarios by name overrides defaults', () => { + const sel = selectScenarios({ names: ['managed-vpc-demo'] }); + assert.deepEqual(sel.map((s) => s.name), ['managed-vpc-demo']); + }); +}); + +// ── env → VPC parsing ─────────────────────────────────────────────────────── + +describe('envToVpc', () => { + it('returns null when nothing is set', () => { + assert.equal(envToVpc({}), null); + }); + it('returns null when only partially set', () => { + assert.equal(envToVpc({ E2E_VPC_ID: 'vpc-1' }), null); + }); + it('parses a complete VPC config with comma-separated lists', () => { + const v = envToVpc({ + E2E_VPC_ID: 'vpc-03e1', + E2E_SUBNET_IDS: 'subnet-a, subnet-b', + E2E_SECURITY_GROUP_IDS: 'sg-1', + }); + assert.deepEqual(v, { vpcId: 'vpc-03e1', subnetIds: ['subnet-a', 'subnet-b'], securityGroupIds: ['sg-1'] }); + }); +}); + +// ── pipeline naming ─────────────────────────────────────────────────────────── + +describe('buildPipelineName', () => { + it('stays within the 28-char OSIS limit', () => { + for (const s of SCENARIOS) { + const name = buildPipelineName(s, 'abcde'); + assert.ok(name.length <= 28, `${s.name} -> ${name} (${name.length})`); + } + }); + it('starts with a lowercase letter and is alnum/hyphen only', () => { + const name = buildPipelineName(getScenario('managed-public-demo'), 'zzzzz'); + assert.match(name, /^[a-z][a-z0-9-]*$/); + }); + it('is distinct per scenario for the same suffix', () => { + const suffix = 'q1w2e'; + const names = SCENARIOS.map((s) => buildPipelineName(s, suffix)); + assert.equal(new Set(names).size, names.length); + }); +}); + +// ── create/destroy arg construction ───────────────────────────────────────── + +describe('buildCreateArgs', () => { + const base = { pipelineName: 'e2e-mpd-abcde', region: 'us-east-1' }; + + it('managed public demo: managed domain, no VPC, demo not skipped', () => { + const args = buildCreateArgs(getScenario('managed-public-demo'), base); + assert.ok(args.includes('--managed')); + assert.ok(args.includes('--os-domain-name')); + assert.ok(!args.includes('--vpc-id')); + assert.ok(!args.includes('--skip-demo')); + // Must NOT use --advanced: advanced mode only creates explicitly-named + // resources, so it would skip the OSI role and hand OSIS an empty + // PipelineRoleArn. Quick mode (the default) auto-creates the full stack. + assert.ok(!args.includes('--advanced')); + }); + + it('no-demo scenario adds --skip-demo', () => { + const args = buildCreateArgs(getScenario('managed-public-nodemo'), base); + assert.ok(args.includes('--skip-demo')); + }); + + it('serverless scenario uses --serverless and --aoss-collection-name', () => { + const args = buildCreateArgs(getScenario('serverless-public-nodemo'), base); + assert.ok(args.includes('--serverless')); + assert.ok(args.includes('--aoss-collection-name')); + assert.ok(!args.includes('--managed')); + }); + + it('VPC scenario threads vpc/subnet/sg flags', () => { + const args = buildCreateArgs(getScenario('managed-vpc-demo'), { + ...base, vpc: { vpcId: 'vpc-1', subnetIds: ['subnet-a', 'subnet-b'], securityGroupIds: ['sg-1'] }, + }); + const i = args.indexOf('--vpc-id'); + assert.equal(args[i + 1], 'vpc-1'); + assert.equal(args[args.indexOf('--subnet-ids') + 1], 'subnet-a,subnet-b'); + assert.equal(args[args.indexOf('--security-group-ids') + 1], 'sg-1'); + }); + + it('VPC scenario without vpc params throws', () => { + assert.throws(() => buildCreateArgs(getScenario('managed-vpc-demo'), base), /requires vpc/); + }); + + it('requires pipelineName and region', () => { + assert.throws(() => buildCreateArgs(getScenario('managed-public-demo'), { region: 'us-east-1' }), /pipelineName/); + assert.throws(() => buildCreateArgs(getScenario('managed-public-demo'), { pipelineName: 'x' }), /region/); + }); + + it('produces args that validateConfig accepts and resolve a full stack (integration with cli.mjs)', async () => { + // Cross-check: the args we build should parse and validate cleanly, AND once + // defaults are applied they must resolve into an actionable full stack — an + // OSI role, an APS workspace, and an app. This guards against a regression + // where the harness selected --advanced and left iamAction empty, which made + // the CLI create only the domain and then fail deep in OSIS pipeline creation + // with an empty PipelineRoleArn ("Cross-account pass role is not allowed"). + const { parseCli, applyQuickDefaults, validateConfig } = await import('../../src/cli.mjs'); + for (const name of ['managed-public-nodemo', 'serverless-public-nodemo', 'managed-public-demo']) { + const args = buildCreateArgs(getScenario(name), base); + const cfg = parseCli(['node', 'cli', ...args]); + if (cfg.mode === 'quick') applyQuickDefaults(cfg); + assert.deepEqual(validateConfig(cfg), [], `${name} should validate`); + // The OSI pipeline role must be resolved to "create" with a name, or the + // pipeline gets an empty role ARN. + assert.equal(cfg.iamAction, 'create', `${name} must create an OSI role`); + assert.ok(cfg.iamRoleName, `${name} must have an OSI role name`); + assert.equal(cfg.apsAction, 'create', `${name} must create an APS workspace`); + assert.ok(cfg.appName, `${name} must have an OpenSearch Application name`); + } + }); + + it('VPC scenario args also resolve a full stack (quick mode + VPC flags)', async () => { + const { parseCli, applyQuickDefaults, validateConfig } = await import('../../src/cli.mjs'); + const args = buildCreateArgs(getScenario('managed-vpc-nodemo'), { + ...base, vpc: { vpcId: 'vpc-1', subnetIds: ['subnet-a', 'subnet-b'], securityGroupIds: ['sg-1'] }, + }); + const cfg = parseCli(['node', 'cli', ...args]); + if (cfg.mode === 'quick') applyQuickDefaults(cfg); + assert.deepEqual(validateConfig(cfg), []); + assert.equal(cfg.iamAction, 'create'); + assert.equal(cfg.vpcId, 'vpc-1'); + assert.equal(cfg.osAction, 'create'); + }); +}); + +describe('buildDestroyArgs', () => { + it('builds a destroy invocation with pipeline + region', () => { + const args = buildDestroyArgs(getScenario('managed-public-demo'), { pipelineName: 'p', region: 'us-east-1' }); + assert.equal(args[0], 'destroy'); + assert.ok(args.includes('--pipeline-name')); + assert.ok(args.includes('--region')); + }); + it('serverless destroy passes the collection name', () => { + const args = buildDestroyArgs(getScenario('serverless-public-nodemo'), { pipelineName: 'p', region: 'us-east-1' }); + assert.ok(args.includes('--aoss-collection-name')); + }); +}); + +// ── count evaluation ────────────────────────────────────────────────────────── + +describe('evaluateCounts', () => { + const full = { 'logs-otel-v1': 5, 'otel-v1-apm-span': 12, 'otel-v2-apm-service-map': 3 }; + + it('passes when every expected index has data', () => { + const r = evaluateCounts(full); + assert.equal(r.ok, true); + assert.equal(r.results.length, EXPECTED_INDICES.length); + }); + + it('fails when a required index is empty', () => { + const r = evaluateCounts({ ...full, 'logs-otel-v1': 0 }); + assert.equal(r.ok, false); + assert.equal(r.results.find((x) => x.pattern === 'logs-otel-v1').ok, false); + }); + + it('treats non-required signals as optional (service-map for synthetic push)', () => { + const r = evaluateCounts( + { 'logs-otel-v1': 2, 'otel-v1-apm-span': 2, 'otel-v2-apm-service-map': 0 }, + { requireSignals: ['logs', 'traces'] }, + ); + assert.equal(r.ok, true); + assert.equal(r.results.find((x) => x.pattern === 'otel-v2-apm-service-map').ok, true); + }); + + it('missing keys count as zero', () => { + const r = evaluateCounts({}); + assert.equal(r.ok, false); + }); +}); + +describe('parseCount', () => { + it('reads count from an object', () => assert.equal(parseCount({ count: 42 }), 42)); + it('reads count from a JSON string', () => assert.equal(parseCount('{"count":7}'), 7)); + it('returns 0 on garbage', () => assert.equal(parseCount('not json'), 0)); + it('returns 0 when count is missing', () => assert.equal(parseCount({ hits: 1 }), 0)); +}); + +// ── synthetic OTLP ──────────────────────────────────────────────────────────── + +describe('buildOtlpPayload', () => { + it('builds a logs payload with service.name and a timestamp', () => { + const p = buildOtlpPayload('logs', { nowNanos: '1700000000000000000', serviceName: 'svc' }); + const rl = p.resourceLogs[0]; + assert.equal(rl.resource.attributes[0].value.stringValue, 'svc'); + assert.equal(rl.scopeLogs[0].logRecords[0].timeUnixNano, '1700000000000000000'); + }); + + it('builds a traces payload with a server span and derived endTime', () => { + const p = buildOtlpPayload('traces', { nowNanos: '1000', serviceName: 'svc' }); + const span = p.resourceSpans[0].scopeSpans[0].spans[0]; + assert.equal(span.startTimeUnixNano, '1000'); + assert.equal(span.endTimeUnixNano, '1001000'); // start + 1_000_000 + assert.equal(span.kind, 2); + }); + + it('carries service.name on traces so service-map edges can be derived', () => { + const p = buildOtlpPayload('traces', { serviceName: 'checkout' }); + assert.equal(p.resourceSpans[0].resource.attributes[0].value.stringValue, 'checkout'); + }); + + it('throws on unsupported signals', () => { + assert.throws(() => buildOtlpPayload('metrics'), /unsupported signal/); + }); +}); + +describe('otlpUrl', () => { + it('builds the per-pipeline OTLP path and strips any scheme on the endpoint', () => { + assert.equal( + otlpUrl('https://abc.us-east-1.osis.amazonaws.com', 'my-pipe', 'logs'), + 'https://abc.us-east-1.osis.amazonaws.com/my-pipe/v1/logs', + ); + assert.equal( + otlpUrl('abc.us-east-1.osis.amazonaws.com', 'my-pipe', 'traces'), + 'https://abc.us-east-1.osis.amazonaws.com/my-pipe/v1/traces', + ); + }); +}); diff --git a/aws/cli-installer/test/e2e/run.mjs b/aws/cli-installer/test/e2e/run.mjs new file mode 100644 index 00000000..35fac81e --- /dev/null +++ b/aws/cli-installer/test/e2e/run.mjs @@ -0,0 +1,227 @@ +#!/usr/bin/env node +/** + * Local AWS end-to-end test runner for the observability-stack CLI. + * + * These tests hit real AWS resources, so they are NOT run in CI (GitHub has no + * AWS credentials by policy). Developers run them locally against a sandbox + * account to gain confidence that AWS-specific changes still create a working + * stack end to end. + * + * For each selected scenario it: + * 1. creates the stack via the real CLI (`bin/cli-installer.mjs`), + * 2. drives telemetry in — the EC2 OTel-demo (demo scenarios) or a synthetic + * OTLP push (no-demo scenarios), + * 3. verifies documents land in the expected OpenSearch indices by querying + * through the managed OpenSearch UI endpoint (works for VPC-private domains), + * 4. tears the stack down — always, even on failure (unless --no-teardown). + * + * Usage: + * AWS_PROFILE= node test/e2e/run.mjs [options] + * + * Options: + * --region AWS region (default: $AWS_REGION or us-east-1) + * --scenario run only this scenario (repeatable) + * --all include opt-in scenarios (VPC) too + * --list print the scenario matrix and exit + * --no-teardown leave resources up after the run (for debugging) + * --data-timeout minutes to wait for data to land (default 20) + * + * VPC scenarios additionally need: + * E2E_VPC_ID, E2E_SUBNET_IDS (comma-sep), E2E_SECURITY_GROUP_IDS (comma-sep) + * + * SAFETY: intended for a disposable sandbox account. It creates and DELETES + * OpenSearch domains, OSIS pipelines, IAM roles, EC2 instances, and AMP + * workspaces prefixed `e2e-`. Never point it at a production account. + */ +import { spawn } from 'node:child_process'; +import { fileURLToPath } from 'node:url'; +import { dirname, join } from 'node:path'; +import { STSClient, GetCallerIdentityCommand } from '@aws-sdk/client-sts'; +import { OSISClient, GetPipelineCommand } from '@aws-sdk/client-osis'; +import { OpenSearchClient, ListApplicationsCommand, GetApplicationCommand } from '@aws-sdk/client-opensearch'; +import { selectScenarios, envToVpc, buildPipelineName, buildCreateArgs, buildDestroyArgs, SCENARIOS } from './scenarios.mjs'; +import { findDataSourceId, waitForData, pushOtlp, buildOtlpPayload } from './verify.mjs'; + +const __dirname = dirname(fileURLToPath(import.meta.url)); +const CLI = join(__dirname, '..', '..', 'bin', 'cli-installer.mjs'); + +// ── arg parsing ───────────────────────────────────────────────────────────── +function parseArgs(argv) { + const out = { scenarios: [], all: false, list: false, teardown: true, region: process.env.AWS_REGION || process.env.AWS_DEFAULT_REGION || 'us-east-1', dataTimeoutMin: 20 }; + for (let i = 2; i < argv.length; i++) { + const a = argv[i]; + if (a === '--all') out.all = true; + else if (a === '--list') out.list = true; + else if (a === '--no-teardown') out.teardown = false; + else if (a === '--scenario') out.scenarios.push(argv[++i]); + else if (a === '--region') out.region = argv[++i]; + else if (a === '--data-timeout') out.dataTimeoutMin = Number(argv[++i]); + else { console.error(`Unknown option: ${a}`); process.exit(2); } + } + return out; +} + +function log(scenario, msg) { + const ts = new Date().toISOString().slice(11, 19); + console.log(`[${ts}] [${scenario}] ${msg}`); +} + +/** Run the CLI as a subprocess, streaming output. Resolves with exit code. */ +function runCli(args) { + return new Promise((resolve) => { + const child = spawn('node', [CLI, ...args], { stdio: 'inherit', env: process.env }); + child.on('close', (code) => resolve(code ?? 1)); + }); +} + +/** Short run id from the current time — collision-avoidance suffix for names. */ +function runSuffix() { + return Math.floor(Date.now() / 1000).toString(36).slice(-5); +} + +async function verifyScenario(scenario, { pipelineName, region, dataTimeoutMs }) { + // Resolve the Application endpoint + ingest endpoint from live resources. + const os = new OpenSearchClient({ region }); + const { ApplicationSummaries } = await os.send(new ListApplicationsCommand({})); + const app = (ApplicationSummaries || []).find((a) => a.name === pipelineName); + if (!app) throw new Error('OpenSearch Application not found — cannot verify data flow'); + const { endpoint: appEndpoint } = await os.send(new GetApplicationCommand({ id: app.id })); + if (!appEndpoint) throw new Error('OpenSearch Application endpoint not populated'); + log(scenario.name, `app endpoint: ${appEndpoint}`); + + const dataSourceId = await findDataSourceId({ appEndpoint, region }); + log(scenario.name, `data source id: ${dataSourceId || '(none)'}`); + + // No-demo scenarios: push synthetic telemetry so there is something to verify. + let requireSignals; + if (!scenario.demo) { + const osis = new OSISClient({ region }); + const { Pipeline } = await osis.send(new GetPipelineCommand({ PipelineName: pipelineName })); + const ingest = Pipeline?.IngestEndpointUrls?.[0]; + if (!ingest) throw new Error('OSIS ingest endpoint not available for synthetic push'); + requireSignals = ['logs', 'traces']; // service-map is derived async; treat as optional + await pushSynthetic({ scenario, ingestEndpoint: ingest, pipelineName, region }); + } + + const result = await waitForData({ + appEndpoint, region, dataSourceId, requireSignals, + timeoutMs: dataTimeoutMs, log: (m) => log(scenario.name, m), + }); + return result; +} + +/** Push a few synthetic OTLP records repeatedly so counts clear zero. */ +async function pushSynthetic({ scenario, ingestEndpoint, pipelineName, region }) { + log(scenario.name, `pushing synthetic OTLP to ${ingestEndpoint}`); + // OSIS ingest for a fresh (esp. VPC) pipeline can warm up for ~10-15 min; + // retry pushes until at least one is accepted. + for (let attempt = 0; attempt < 30; attempt++) { + const nowNanos = String(BigInt(Date.now()) * 1_000_000n); + let accepted = 0; + for (const signal of ['logs', 'traces']) { + const payload = buildOtlpPayload(signal, { nowNanos, serviceName: `e2e-${scenario.name}` }); + try { + const { status } = await pushOtlp({ ingestEndpoint, pipelineName, region, signal, payload }); + if (status >= 200 && status < 300) accepted++; + else log(scenario.name, ` ${signal} push -> HTTP ${status}`); + } catch (e) { + log(scenario.name, ` ${signal} push error: ${e.message}`); + } + } + if (accepted === 2) { log(scenario.name, 'synthetic push accepted'); } + await new Promise((r) => setTimeout(r, 30_000)); + } +} + +async function main() { + const opts = parseArgs(process.argv); + + if (opts.list) { + console.log('Available scenarios:\n'); + for (const s of SCENARIOS) { + console.log(` ${s.name.padEnd(26)} ${s.enabledByDefault ? '(default)' : '(opt-in) '} ${s.description}`); + } + console.log('\nVPC scenarios need E2E_VPC_ID / E2E_SUBNET_IDS / E2E_SECURITY_GROUP_IDS.'); + return; + } + + // Confirm credentials up front and print the target account (safety). + const sts = new STSClient({ region: opts.region }); + const id = await sts.send(new GetCallerIdentityCommand({})); + console.log(`Target AWS account: ${id.Account} (${id.Arn})`); + console.log(`Region: ${opts.region}\n`); + + const selected = selectScenarios({ names: opts.scenarios, all: opts.all }); + const vpc = envToVpc(); + const suffix = runSuffix(); + const dataTimeoutMs = opts.dataTimeoutMin * 60_000; + const summary = []; + + for (const scenario of selected) { + if (scenario.vpc && !vpc) { + log(scenario.name, 'SKIP — VPC env vars not set (E2E_VPC_ID / E2E_SUBNET_IDS / E2E_SECURITY_GROUP_IDS)'); + summary.push({ scenario: scenario.name, status: 'skipped', reason: 'no VPC env' }); + continue; + } + + const pipelineName = buildPipelineName(scenario, suffix); + const base = { pipelineName, region: opts.region, vpc }; + let status = 'unknown'; + let detail = ''; + + log(scenario.name, `=== START (pipeline: ${pipelineName}) ===`); + try { + const createArgs = buildCreateArgs(scenario, base); + log(scenario.name, `create: node cli-installer.mjs ${createArgs.join(' ')}`); + const code = await runCli(createArgs); + if (code !== 0) throw new Error(`CLI create exited ${code}`); + + if (scenario.verifyDataFlow === false) { + // Data-flow verification isn't applicable (e.g. a VPC-private ingest + // endpoint the runner host can't reach). A clean create — the CLI exits 0 + // only after every resource, including the pipeline reaching ACTIVE — plus + // the teardown below is what this scenario proves. + status = 'passed'; + detail = 'created (data-flow verification not applicable; teardown exercised)'; + log(scenario.name, 'skipping data-flow verification for this scenario (not applicable)'); + } else { + const result = await verifyScenario(scenario, { pipelineName, region: opts.region, dataTimeoutMs }); + if (result.ok) { + status = 'passed'; + detail = result.results.map((r) => `${r.pattern}=${r.count}`).join(' '); + } else { + status = 'failed'; + detail = `data not confirmed${result.timedOut ? ' (timeout)' : ''}: ` + + result.results.map((r) => `${r.pattern}=${r.count}${r.ok ? '' : ' ✗'}`).join(' '); + } + } + } catch (e) { + status = 'error'; + detail = e.message; + log(scenario.name, `ERROR: ${e.message}`); + } finally { + if (opts.teardown) { + log(scenario.name, 'tearing down...'); + const code = await runCli(buildDestroyArgs(scenario, base)); + if (code !== 0) log(scenario.name, `WARNING: destroy exited ${code} — check for leftover resources`); + } else { + log(scenario.name, 'teardown skipped (--no-teardown) — remember to destroy manually'); + } + } + + log(scenario.name, `=== ${status.toUpperCase()} — ${detail} ===\n`); + summary.push({ scenario: scenario.name, status, detail }); + } + + // ── Summary ── + console.log('\n================ E2E SUMMARY ================'); + for (const s of summary) { + console.log(` ${s.status.toUpperCase().padEnd(8)} ${s.scenario.padEnd(26)} ${s.detail || s.reason || ''}`); + } + const failed = summary.filter((s) => s.status === 'failed' || s.status === 'error'); + console.log(`\n${summary.length} scenario(s): ${summary.filter(s => s.status === 'passed').length} passed, ` + + `${failed.length} failed/errored, ${summary.filter(s => s.status === 'skipped').length} skipped`); + process.exit(failed.length ? 1 : 0); +} + +main().catch((e) => { console.error(e); process.exit(1); }); diff --git a/aws/cli-installer/test/e2e/scenarios.mjs b/aws/cli-installer/test/e2e/scenarios.mjs new file mode 100644 index 00000000..78cebe29 --- /dev/null +++ b/aws/cli-installer/test/e2e/scenarios.mjs @@ -0,0 +1,192 @@ +/** + * E2E scenario matrix + CLI argument construction. + * + * These functions are PURE (no AWS, no I/O) so they can be unit-tested without + * credentials. The live runner (run.mjs) turns each scenario into a real CLI + * invocation, verifies data flow, then tears the stack down. + * + * The matrix covers the two dimensions called out for AWS-specific coverage: + * - network topology: public endpoints vs. VPC-private endpoints + * - demo workload: launch the EC2 OTel-demo instance vs. skip it + * plus the managed-domain vs. serverless backend split. + */ + +/** + * The default scenario matrix. `enabledByDefault` scenarios run in a plain + * `run.mjs` invocation; the rest are opt-in (they cost more time/money or need + * extra inputs such as a VPC). A developer can also select scenarios by name. + * + * VPC scenarios require --vpc-id/--subnet-ids/--security-group-ids, supplied via + * env (see envToVpc). They are opt-in because they need pre-existing VPC infra. + */ +export const SCENARIOS = [ + { + name: 'managed-public-demo', + description: 'Managed domain, public endpoints, EC2 OTel-demo launched', + backend: 'managed', + vpc: false, + demo: true, + enabledByDefault: true, + verifyDataFlow: true, + }, + { + name: 'managed-public-nodemo', + description: 'Managed domain, public endpoints, no demo (synthetic OTLP push)', + backend: 'managed', + vpc: false, + demo: false, + enabledByDefault: true, + verifyDataFlow: true, + }, + { + name: 'serverless-public-nodemo', + description: 'Serverless (AOSS) collection, public endpoints, synthetic OTLP push', + backend: 'serverless', + vpc: false, + demo: false, + enabledByDefault: true, + verifyDataFlow: true, + }, + { + name: 'managed-vpc-demo', + description: 'Managed domain in a VPC (private endpoints), EC2 OTel-demo launched', + backend: 'managed', + vpc: true, + demo: true, + enabledByDefault: false, + // The EC2 demo launches INSIDE the configured VPC (see ec2-demo.mjs), so it + // can reach the VPC-private OSIS ingest endpoint. This is the scenario that + // actually proves end-to-end VPC data flow; verification queries the domain + // through the reachable-from-outside Application (OpenSearch UI) endpoint. + verifyDataFlow: true, + }, + { + name: 'managed-vpc-nodemo', + description: 'Managed domain in a VPC (private endpoints), no demo — create/destroy only', + backend: 'managed', + vpc: true, + demo: false, + enabledByDefault: false, + // A VPC-attached OSIS pipeline exposes a VPC-PRIVATE ingest endpoint (resolves + // to RFC-1918 addresses reachable only inside the VPC). The synthetic OTLP push + // originates from the developer's host, which has no route into the VPC, so it + // can never reach that endpoint — data-flow verification is not applicable here. + // This scenario therefore exercises create + the full teardown path only (the + // CLI still proves the VPC-attached pipeline reaches ACTIVE before exiting 0). + // Use managed-vpc-demo to verify actual VPC data flow (demo runs in-VPC). + verifyDataFlow: false, + }, +]; + +/** + * Look up a scenario by name. Throws with the list of valid names if unknown. + */ +export function getScenario(name) { + const s = SCENARIOS.find((x) => x.name === name); + if (!s) { + throw new Error(`Unknown scenario '${name}'. Valid: ${SCENARIOS.map((x) => x.name).join(', ')}`); + } + return s; +} + +/** + * Select the scenarios to run. + * - names: explicit list (highest precedence) + * - all: include opt-in scenarios too + * - otherwise: the default-enabled scenarios + */ +export function selectScenarios({ names, all } = {}) { + if (names?.length) return names.map(getScenario); + if (all) return [...SCENARIOS]; + return SCENARIOS.filter((s) => s.enabledByDefault); +} + +/** + * Derive VPC parameters from the environment. VPC scenarios are skipped (not + * failed) when these are absent, so the default matrix stays runnable anywhere. + * Returns { vpcId, subnetIds, securityGroupIds } or null when unset. + */ +export function envToVpc(env = process.env) { + const vpcId = env.E2E_VPC_ID || ''; + const subnetIds = (env.E2E_SUBNET_IDS || '').split(',').map((s) => s.trim()).filter(Boolean); + const securityGroupIds = (env.E2E_SECURITY_GROUP_IDS || '').split(',').map((s) => s.trim()).filter(Boolean); + if (!vpcId || !subnetIds.length || !securityGroupIds.length) return null; + return { vpcId, subnetIds, securityGroupIds }; +} + +/** + * Build a unique, DNS/OSIS-safe pipeline name for a scenario run. + * OSIS caps pipeline names at 28 chars, lowercase-alnum-hyphen, letter-first. + * `suffix` is typically a short timestamp/run id to avoid collisions. + */ +export function buildPipelineName(scenario, suffix) { + // Short, stable tag per scenario so the name stays within 28 chars. + const tagByName = { + 'managed-public-demo': 'mpd', + 'managed-public-nodemo': 'mpn', + 'serverless-public-nodemo': 'spn', + 'managed-vpc-demo': 'mvd', + 'managed-vpc-nodemo': 'mvn', + }; + const tag = tagByName[scenario.name] || scenario.name.replace(/[^a-z0-9]/g, '').slice(0, 6); + const raw = `e2e-${tag}-${suffix}`.toLowerCase(); + const cleaned = raw.replace(/[^a-z0-9-]/g, '').replace(/^-+/, '').slice(0, 28); + // Guarantee it starts with a letter (OSIS requirement). + return /^[a-z]/.test(cleaned) ? cleaned : `e${cleaned}`.slice(0, 28); +} + +/** + * Build the argv (after `node cli-installer.mjs`) for creating a scenario's stack. + * + * @param {object} scenario one of SCENARIOS + * @param {object} opts + * @param {string} opts.pipelineName + * @param {string} opts.region + * @param {object|null} [opts.vpc] { vpcId, subnetIds[], securityGroupIds[] } — required if scenario.vpc + * @returns {string[]} argv + */ +export function buildCreateArgs(scenario, opts) { + const { pipelineName, region, vpc } = opts; + if (!pipelineName) throw new Error('buildCreateArgs: pipelineName is required'); + if (!region) throw new Error('buildCreateArgs: region is required'); + + // Quick mode (the default — no --advanced) is what we want here: it auto-creates + // the whole stack (IAM OSI role, APS workspace, Application) named after the + // pipeline, while still honoring every explicit flag we pass below. Advanced mode + // only creates the resources you name explicitly, so passing --advanced with just + // --os-domain-name would create the domain but skip the OSI role, leaving OSIS to + // reject an empty PipelineRoleArn ("Cross-account pass role is not allowed"). The + // e2e always creates fresh resources, so quick mode is the correct, realistic path. + const args = ['--pipeline-name', pipelineName, '--region', region]; + + if (scenario.backend === 'serverless') { + args.push('--serverless', '--aoss-collection-name', pipelineName); + } else { + args.push('--managed', '--os-domain-name', pipelineName); + } + + if (scenario.vpc) { + if (!vpc?.vpcId || !vpc.subnetIds?.length || !vpc.securityGroupIds?.length) { + throw new Error(`buildCreateArgs: scenario '${scenario.name}' requires vpc {vpcId, subnetIds, securityGroupIds}`); + } + args.push( + '--vpc-id', vpc.vpcId, + '--subnet-ids', vpc.subnetIds.join(','), + '--security-group-ids', vpc.securityGroupIds.join(','), + ); + } + + if (!scenario.demo) args.push('--skip-demo'); + + return args; +} + +/** + * Build the argv for tearing a scenario's stack down. + */ +export function buildDestroyArgs(scenario, opts) { + const { pipelineName, region } = opts; + const args = ['destroy', '--pipeline-name', pipelineName, '--region', region]; + if (scenario.backend === 'serverless') args.push('--aoss-collection-name', pipelineName); + return args; +} diff --git a/aws/cli-installer/test/e2e/verify.mjs b/aws/cli-installer/test/e2e/verify.mjs new file mode 100644 index 00000000..e83f13e9 --- /dev/null +++ b/aws/cli-installer/test/e2e/verify.mjs @@ -0,0 +1,238 @@ +/** + * E2E data-flow verification. + * + * After a stack is created and telemetry is pushed (either by the EC2 OTel-demo + * or a synthetic OTLP push), these helpers confirm that documents actually + * landed in the expected OpenSearch indices by querying through the managed + * OpenSearch UI (Application) endpoint — the same SigV4 proxy path the installer + * uses, which works even for VPC-private domains from outside the VPC. + * + * Pure helpers (index expectations, count evaluation, OTLP payload building) are + * separated from the network calls so they can be unit-tested without AWS. + */ +import { createHash } from 'node:crypto'; +import { SignatureV4 } from '@aws-sdk/signature-v4'; +import { Sha256 } from '@aws-crypto/sha256-js'; +import { HttpRequest } from '@smithy/protocol-http'; +import { defaultProvider } from '@aws-sdk/credential-provider-node'; + +// ── Pure helpers ────────────────────────────────────────────────────────────── + +/** + * The index patterns the stack ingests into, matching render.mjs sink config and + * the index patterns created in opensearch-ui-init.mjs. Each is checked for a + * positive document count during verification. + * + * `signal` labels how the index is populated: 'logs' and 'traces' map directly + * to the OTLP signal that fills them; 'service-map' is derived asynchronously by + * the pipeline from spans, so it gets its own key and can be required separately. + */ +export const EXPECTED_INDICES = [ + { pattern: 'logs-otel-v1', signal: 'logs' }, + { pattern: 'otel-v1-apm-span', signal: 'traces' }, + { pattern: 'otel-v2-apm-service-map', signal: 'service-map' }, +]; + +/** + * Given a map of index-pattern -> observed count, decide pass/fail per index and + * overall. `require` narrows which signals must have data (e.g. a synthetic push + * that only sends logs+traces should not require the service-map index, which is + * derived asynchronously by the pipeline from spans). + * + * @returns {{ ok: boolean, results: Array<{pattern, signal, count, ok}> }} + */ +export function evaluateCounts(counts, { requireSignals } = {}) { + const results = EXPECTED_INDICES.map(({ pattern, signal }) => { + const count = Number(counts[pattern] ?? 0); + const required = requireSignals ? requireSignals.includes(signal) : true; + return { pattern, signal, count, required, ok: required ? count > 0 : true }; + }); + return { ok: results.every((r) => r.ok), results }; +} + +/** + * Parse a document count out of an OpenSearch `_count` response body, which may + * arrive as a parsed object or a raw JSON string (the console proxy returns text). + * Returns 0 when the shape is unexpected rather than throwing. + */ +export function parseCount(body) { + let obj = body; + if (typeof body === 'string') { + try { obj = JSON.parse(body); } catch { return 0; } + } + const n = obj?.count; + return Number.isFinite(n) ? n : 0; +} + +/** + * Build a minimal OTLP/HTTP JSON payload for a single signal, used by the + * no-demo scenarios to push synthetic telemetry directly at the OSIS ingest + * endpoint. `nowNanos` is injected (not read from the clock) so this stays pure + * and unit-testable. + * + * Supports 'logs' and 'traces'. The trace payload carries service.name so the + * pipeline can derive service-map edges. + */ +export function buildOtlpPayload(signal, { serviceName = 'e2e-synthetic', nowNanos, traceId, spanId } = {}) { + const ts = String(nowNanos ?? '0'); + const resource = { + attributes: [{ key: 'service.name', value: { stringValue: serviceName } }], + }; + + if (signal === 'logs') { + return { + resourceLogs: [{ + resource, + scopeLogs: [{ + scope: { name: 'e2e-verifier' }, + logRecords: [{ + timeUnixNano: ts, + observedTimeUnixNano: ts, + severityNumber: 9, + severityText: 'INFO', + body: { stringValue: 'e2e synthetic log' }, + traceId: traceId || '', + spanId: spanId || '', + }], + }], + }], + }; + } + + if (signal === 'traces') { + return { + resourceSpans: [{ + resource, + scopeSpans: [{ + scope: { name: 'e2e-verifier' }, + spans: [{ + traceId: traceId || '00000000000000000000000000000001', + spanId: spanId || '0000000000000001', + name: 'e2e-synthetic-span', + kind: 2, // SERVER + startTimeUnixNano: ts, + endTimeUnixNano: String(BigInt(ts || '0') + 1_000_000n), + attributes: [{ key: 'e2e', value: { boolValue: true } }], + status: { code: 1 }, + }], + }], + }], + }; + } + + throw new Error(`buildOtlpPayload: unsupported signal '${signal}'`); +} + +/** + * Build the OSIS OTLP ingest URL for a signal. + * OSIS exposes per-pipeline paths: https:////v1/{logs,traces,metrics} + */ +export function otlpUrl(ingestEndpoint, pipelineName, signal) { + const host = ingestEndpoint.replace(/^https?:\/\//, ''); + return `https://${host}/${pipelineName}/v1/${signal}`; +} + +// ── SigV4 request helper (shared by ingest + query) ───────────────────────────── + +async function sigv4Fetch({ method, url, body, service, region }) { + const isBodyless = method === 'GET' || method === 'DELETE'; + const bodyBytes = !isBodyless && body != null + ? (typeof body === 'string' ? body : JSON.stringify(body)) + : ''; + const parsed = new URL(url); + const query = {}; + parsed.searchParams.forEach((v, k) => { query[k] = v; }); + + const request = new HttpRequest({ + method, + protocol: parsed.protocol, + hostname: parsed.hostname, + port: parsed.port ? Number(parsed.port) : undefined, + path: parsed.pathname, + query, + headers: { + host: parsed.hostname, + 'Content-Type': 'application/json', + 'osd-xsrf': 'osd-fetch', + 'x-amz-content-sha256': createHash('sha256').update(bodyBytes).digest('hex'), + }, + body: bodyBytes || undefined, + }); + + const signer = new SignatureV4({ credentials: defaultProvider(), region, service, sha256: Sha256 }); + const signed = await signer.sign(request); + const resp = await fetch(url, { method, headers: signed.headers, body: isBodyless ? undefined : bodyBytes }); + const text = await resp.text(); + let data; try { data = JSON.parse(text); } catch { data = text; } + return { status: resp.status, data }; +} + +/** + * Push a synthetic OTLP payload to the OSIS ingest endpoint (service `osis`). + * Used by no-demo scenarios. Returns { status }. + */ +export async function pushOtlp({ ingestEndpoint, pipelineName, region, signal, payload }) { + const url = otlpUrl(ingestEndpoint, pipelineName, signal); + const { status } = await sigv4Fetch({ + method: 'POST', url, body: payload, service: 'osis', region, + }); + return { status }; +} + +// ── Query the domain through the OpenSearch UI (Application) endpoint ─────────── + +/** + * Discover the auto-created data-source id behind the OpenSearch UI, needed to + * proxy queries to the underlying domain/collection. + */ +export async function findDataSourceId({ appEndpoint, region }) { + const { status, data } = await sigv4Fetch({ + method: 'GET', + url: `${appEndpoint}/api/saved_objects/_find?type=data-source&per_page=10`, + service: 'opensearch', region, + }); + if (status !== 200) return null; + return data?.saved_objects?.[0]?.id || null; +} + +/** + * Get the document count for an index pattern by proxying an OpenSearch + * `_count` request through the UI console proxy. This is the reachable path for + * VPC-private domains (the UI proxies over the AWS-internal network). + */ +export async function countDocs({ appEndpoint, region, dataSourceId, indexPattern }) { + const path = encodeURIComponent(`/${indexPattern}*/_count`); + const url = `${appEndpoint}/api/console/proxy?path=${path}&method=GET` + + (dataSourceId ? `&dataSourceId=${dataSourceId}` : ''); + const { status, data } = await sigv4Fetch({ method: 'POST', url, service: 'opensearch', region }); + if (status !== 200) return { status, count: 0 }; + return { status, count: parseCount(data) }; +} + +/** + * Poll all expected indices until every required signal has a positive count or + * the deadline passes. Returns the final evaluateCounts() result plus timing. + * + * Data can take several minutes to land (EC2 demo bootstrap, or OSIS ingest + * warmup for a freshly-created VPC pipeline), so this polls patiently. + */ +export async function waitForData({ + appEndpoint, region, dataSourceId, requireSignals, + timeoutMs = 20 * 60_000, intervalMs = 30_000, log = () => {}, +}) { + const start = Date.now(); + let last; + while (Date.now() - start < timeoutMs) { + const counts = {}; + for (const { pattern } of EXPECTED_INDICES) { + const { count } = await countDocs({ appEndpoint, region, dataSourceId, indexPattern: pattern }); + counts[pattern] = count; + } + last = evaluateCounts(counts, { requireSignals }); + const summary = last.results.map((r) => `${r.pattern}=${r.count}${r.required ? '' : '(opt)'}`).join(' '); + log(`counts: ${summary} — ${last.ok ? 'OK' : 'waiting'}`); + if (last.ok) return { ...last, elapsedMs: Date.now() - start }; + await new Promise((r) => setTimeout(r, intervalMs)); + } + return { ...(last || { ok: false, results: [] }), elapsedMs: Date.now() - start, timedOut: true }; +} diff --git a/aws/cli-installer/test/unit.test.mjs b/aws/cli-installer/test/unit.test.mjs index 6684e27b..e76df85c 100644 --- a/aws/cli-installer/test/unit.test.mjs +++ b/aws/cli-installer/test/unit.test.mjs @@ -452,6 +452,10 @@ import { fgacPrincipals, validateVpcTopology, pipelineRoleArnError, + permissionsAllowPort, + permissionsAllowInternet, + analyzeSecurityGroupRules, + checkSecurityGroupRules, _withRetry, _isRoleNotPropagatedError, _isTransientHttpError, @@ -603,6 +607,163 @@ describe('validateVpcTopology', () => { }); }); +// ── Security-group rule analysis (best-effort data-path check) ──────────────── + +describe('permissionsAllowPort', () => { + const opts = { selfIds: ['sg-1'], vpcCidrs: ['172.30.0.0/16'] }; + + it('matches a tcp/443 range from a self-referencing group', () => { + const perms = [{ IpProtocol: 'tcp', FromPort: 443, ToPort: 443, UserIdGroupPairs: [{ GroupId: 'sg-1' }], IpRanges: [] }]; + assert.equal(permissionsAllowPort(perms, 443, opts), true); + }); + + it('matches all-protocols (-1) regardless of port fields', () => { + const perms = [{ IpProtocol: '-1', UserIdGroupPairs: [{ GroupId: 'sg-1' }], IpRanges: [] }]; + assert.equal(permissionsAllowPort(perms, 443, opts), true); + }); + + it('matches a 0.0.0.0/0 CIDR range covering the port', () => { + const perms = [{ IpProtocol: 'tcp', FromPort: 0, ToPort: 65535, UserIdGroupPairs: [], IpRanges: [{ CidrIp: '0.0.0.0/0' }] }]; + assert.equal(permissionsAllowPort(perms, 443, opts), true); + }); + + it('matches the VPC CIDR', () => { + const perms = [{ IpProtocol: 'tcp', FromPort: 443, ToPort: 443, IpRanges: [{ CidrIp: '172.30.0.0/16' }] }]; + assert.equal(permissionsAllowPort(perms, 443, opts), true); + }); + + it('does not match a foreign group or a narrower CIDR', () => { + const perms = [{ IpProtocol: 'tcp', FromPort: 443, ToPort: 443, UserIdGroupPairs: [{ GroupId: 'sg-other' }], IpRanges: [{ CidrIp: '10.0.0.5/32' }] }]; + assert.equal(permissionsAllowPort(perms, 443, opts), false); + }); + + it('does not match a rule whose port range excludes 443', () => { + const perms = [{ IpProtocol: 'tcp', FromPort: 80, ToPort: 80, IpRanges: [{ CidrIp: '0.0.0.0/0' }] }]; + assert.equal(permissionsAllowPort(perms, 443, opts), false); + }); +}); + +describe('permissionsAllowInternet', () => { + it('matches only a 0.0.0.0/0 range, not a self-ref or VPC CIDR', () => { + assert.equal(permissionsAllowInternet([{ IpProtocol: 'tcp', FromPort: 443, ToPort: 443, IpRanges: [{ CidrIp: '0.0.0.0/0' }] }], 443), true); + assert.equal(permissionsAllowInternet([{ IpProtocol: '-1', IpRanges: [{ CidrIp: '0.0.0.0/0' }] }], 443), true); + assert.equal(permissionsAllowInternet([{ IpProtocol: 'tcp', FromPort: 443, ToPort: 443, UserIdGroupPairs: [{ GroupId: 'sg-1' }] }], 443), false); + assert.equal(permissionsAllowInternet([{ IpProtocol: 'tcp', FromPort: 443, ToPort: 443, IpRanges: [{ CidrIp: '172.30.0.0/16' }] }], 443), false); + }); +}); + +describe('analyzeSecurityGroupRules', () => { + const vpcCidrs = ['172.30.0.0/16']; + + it('no warnings for a demo when egress allows internet and 443 is open in-VPC', () => { + const groups = [{ + GroupId: 'sg-1', + IpPermissions: [{ IpProtocol: 'tcp', FromPort: 443, ToPort: 443, UserIdGroupPairs: [{ GroupId: 'sg-1' }] }], + IpPermissionsEgress: [{ IpProtocol: '-1', IpRanges: [{ CidrIp: '0.0.0.0/0' }] }], + }]; + assert.deepEqual(analyzeSecurityGroupRules(groups, { groupIds: ['sg-1'], vpcCidrs, demo: true }), []); + }); + + it('warns about both intra-VPC and internet egress when egress is stripped (demo)', () => { + // The SA's failure: default allow-all egress removed, so the demo can't reach + // OSIS in-VPC OR bootstrap over the internet. + const groups = [{ + GroupId: 'sg-1', + IpPermissions: [{ IpProtocol: 'tcp', FromPort: 443, ToPort: 443, UserIdGroupPairs: [{ GroupId: 'sg-1' }] }], + IpPermissionsEgress: [], + }]; + const w = analyzeSecurityGroupRules(groups, { groupIds: ['sg-1'], vpcCidrs, demo: true }); + assert.equal(w.length, 2); + assert.ok(w.some((x) => /within the VPC/i.test(x))); + assert.ok(w.some((x) => /0\.0\.0\.0\/0/.test(x) && /internet/i.test(x))); + }); + + it('flags missing internet egress even when in-VPC egress is present (demo)', () => { + // Egress scoped to the VPC CIDR: telemetry to OSIS works, but the demo cannot + // pull images. This is the case the earlier version wrongly passed. + const groups = [{ + GroupId: 'sg-1', + IpPermissions: [{ IpProtocol: 'tcp', FromPort: 443, ToPort: 443, UserIdGroupPairs: [{ GroupId: 'sg-1' }] }], + IpPermissionsEgress: [{ IpProtocol: 'tcp', FromPort: 443, ToPort: 443, IpRanges: [{ CidrIp: '172.30.0.0/16' }] }], + }]; + const w = analyzeSecurityGroupRules(groups, { groupIds: ['sg-1'], vpcCidrs, demo: true }); + assert.equal(w.length, 1); + assert.ok(/internet/i.test(w[0]) && /0\.0\.0\.0\/0/.test(w[0])); + }); + + it('does NOT require internet egress when no demo launches (--skip-demo)', () => { + // No demo: only OSIS→domain in-VPC egress is needed. VPC-scoped egress suffices. + const groups = [{ + GroupId: 'sg-1', + IpPermissions: [{ IpProtocol: 'tcp', FromPort: 443, ToPort: 443, UserIdGroupPairs: [{ GroupId: 'sg-1' }] }], + IpPermissionsEgress: [{ IpProtocol: 'tcp', FromPort: 443, ToPort: 443, IpRanges: [{ CidrIp: '172.30.0.0/16' }] }], + }]; + assert.deepEqual(analyzeSecurityGroupRules(groups, { groupIds: ['sg-1'], vpcCidrs, demo: false }), []); + }); + + it('warns about missing intra-VPC ingress on 443', () => { + const groups = [{ + GroupId: 'sg-1', + IpPermissions: [{ IpProtocol: 'tcp', FromPort: 22, ToPort: 22, IpRanges: [{ CidrIp: '0.0.0.0/0' }] }], + IpPermissionsEgress: [{ IpProtocol: '-1', IpRanges: [{ CidrIp: '0.0.0.0/0' }] }], + }]; + const w = analyzeSecurityGroupRules(groups, { groupIds: ['sg-1'], vpcCidrs, demo: true }); + assert.equal(w.length, 1); + assert.ok(/ingress.*443/i.test(w[0])); + }); + + it('is satisfied when the union across multiple groups covers ingress and egress', () => { + const groups = [ + { GroupId: 'sg-in', IpPermissions: [{ IpProtocol: 'tcp', FromPort: 443, ToPort: 443, UserIdGroupPairs: [{ GroupId: 'sg-in' }] }], IpPermissionsEgress: [] }, + { GroupId: 'sg-out', IpPermissions: [], IpPermissionsEgress: [{ IpProtocol: 'tcp', FromPort: 443, ToPort: 443, IpRanges: [{ CidrIp: '0.0.0.0/0' }] }] }, + ]; + assert.deepEqual(analyzeSecurityGroupRules(groups, { groupIds: ['sg-in', 'sg-out'], vpcCidrs, demo: true }), []); + }); +}); + +describe('checkSecurityGroupRules', () => { + const cfg = { region: 'us-east-1', vpcId: 'vpc-aaa', securityGroupIds: ['sg-1'] }; + + it('returns [] when no VPC is configured', async () => { + assert.deepEqual(await checkSecurityGroupRules({ region: 'us-east-1', vpcId: '', securityGroupIds: [] }), []); + }); + + it('degrades to a single manual-verify warning when describe is unauthorized', async () => { + const w = await checkSecurityGroupRules(cfg, { + describeSecurityGroups: async () => { const e = new Error('UnauthorizedOperation'); e.name = 'UnauthorizedOperation'; throw e; }, + describeVpcs: async () => [{ VpcId: 'vpc-aaa', CidrBlock: '172.30.0.0/16' }], + }); + assert.equal(w.length, 1); + assert.ok(/lacks ec2:DescribeSecurityGroups/.test(w[0])); + }); + + it('flags a stripped egress rule end-to-end (demo default)', async () => { + const w = await checkSecurityGroupRules(cfg, { + describeSecurityGroups: async () => [{ + GroupId: 'sg-1', + IpPermissions: [{ IpProtocol: 'tcp', FromPort: 443, ToPort: 443, UserIdGroupPairs: [{ GroupId: 'sg-1' }] }], + IpPermissionsEgress: [], + }], + describeVpcs: async () => [{ VpcId: 'vpc-aaa', CidrBlockAssociationSet: [{ CidrBlock: '172.30.0.0/16' }] }], + }); + // demo defaults on (skipDemo unset): both in-VPC and internet egress missing. + assert.equal(w.length, 2); + assert.ok(w.some((x) => /internet/i.test(x))); + }); + + it('does not flag internet egress when skipDemo is set', async () => { + const w = await checkSecurityGroupRules({ ...cfg, skipDemo: true }, { + describeSecurityGroups: async () => [{ + GroupId: 'sg-1', + IpPermissions: [{ IpProtocol: 'tcp', FromPort: 443, ToPort: 443, UserIdGroupPairs: [{ GroupId: 'sg-1' }] }], + IpPermissionsEgress: [{ IpProtocol: 'tcp', FromPort: 443, ToPort: 443, IpRanges: [{ CidrIp: '172.30.0.0/16' }] }], + }], + describeVpcs: async () => [{ VpcId: 'vpc-aaa', CidrBlockAssociationSet: [{ CidrBlock: '172.30.0.0/16' }] }], + }); + assert.deepEqual(w, []); + }); +}); + // ── Creation ordering / race-condition guards ──────────────────────────────── describe('withRetry', () => {