kylehounslow · kylehounslow · Jul 27, 2026
@@ -6,7 +6,10 @@
   "bin": "./bin/cli-installer.mjs",
   "scripts": {
     "start": "node bin/open-stack.mjs",
-    "test": "node --test test/unit.test.mjs"
+    "test": "node --test test/unit.test.mjs test/e2e/e2e-unit.test.mjs",
+    "test:unit": "node --test test/unit.test.mjs test/e2e/e2e-unit.test.mjs",
+    "e2e": "node test/e2e/run.mjs",
+    "e2e:list": "node test/e2e/run.mjs --list"
   },
   "repository": {
     "type": "git",

@@ -1718,6 +1718,162 @@ export async function validateVpcTopology(cfg, deps = {}) {
   return errors;
 }
 
+// ── Security-group rule analysis (best-effort data-path check) ───────────────
+
+/**
+ * Does a permission entry cover TCP `port`? IpProtocol '-1' means all protocols
+ * and all ports.
+ */
+function permCoversPort(p, port) {
+  const proto = p.IpProtocol;
+  return proto === '-1' || (proto === 'tcp' && (p.FromPort ?? 0) <= port && (p.ToPort ?? 65535) >= port);
+}
+
+/**
+ * Does a set of IpPermissions entries allow TCP `port` from/to an intra-stack
+ * source? A source counts as intra-stack when the rule references one of the
+ * stack's own security groups (`selfIds`) or a CIDR not narrower than the VPC
+ * (0.0.0.0/0, or a VPC-CIDR entry).
+ */
+export function permissionsAllowPort(permissions, port, { selfIds = [], vpcCidrs = [] } = {}) {
+  const selfSet = new Set(selfIds);
+  const cidrOk = (cidr) => cidr === '0.0.0.0/0' || vpcCidrs.includes(cidr);
+  for (const p of permissions || []) {
+    if (!permCoversPort(p, port)) continue;
+    if ((p.UserIdGroupPairs || []).some((g) => selfSet.has(g.GroupId))) return true;
+    if ((p.IpRanges || []).some((r) => cidrOk(r.CidrIp))) return true;
+  }
+  return false;
+}
+
+/**
+ * Does a set of IpPermissions entries allow TCP `port` to the public internet
+ * (a 0.0.0.0/0 IpRanges entry)? A self-referencing SG rule or a VPC-scoped CIDR
+ * does NOT count; those keep traffic inside the VPC. The demo instance needs
+ * this for bootstrap (dnf, GitHub, container registries), which does not route
+ * through the VPC-private OSIS endpoint.
+ */
+export function permissionsAllowInternet(permissions, port) {
+  for (const p of permissions || []) {
+    if (!permCoversPort(p, port)) continue;
+    if ((p.IpRanges || []).some((r) => r.CidrIp === '0.0.0.0/0')) return true;
+  }
+  return false;
+}
+
+/**
+ * Analyze the security groups the stack attaches to the EC2 demo, OSIS pipeline,
+ * and OpenSearch domain, and return WARNING strings for rules that would silently
+ * break the data path. Pure: takes the DescribeSecurityGroups result so it is
+ * unit-testable. Warnings only, never hard-fails: a user's network may route the
+ * same traffic via other groups, a NAT, or NACLs we can't see.
+ *
+ * All three components share `groupIds` (see ec2-demo.mjs / createOsiPipeline /
+ * createOpenSearch). There are two distinct egress needs, only one of which is
+ * intra-VPC:
+ *   - Intra-VPC egress 443 (always): OSIS reaches the domain; with a demo, the
+ *     demo's collector reaches the VPC-private OSIS ingest endpoint. Satisfied by
+ *     a self-reference, a VPC-CIDR rule, or 0.0.0.0/0.
+ *   - Internet egress 443 (only with a demo): the instance bootstraps over the
+ *     public internet (dnf, GitHub clone, container image pulls). This traffic
+ *     does NOT flow through the VPC-private endpoint, so only a 0.0.0.0/0 egress
+ *     rule (to a NAT/IGW) satisfies it; a VPC-CIDR-scoped rule does not.
+ * Plus intra-VPC ingress 443 (always): OSIS accepts OTLP from the demo and the
+ * domain accepts requests from OSIS.
+ *
+ * @param {object[]} groups  DescribeSecurityGroups result
+ * @param {object} opts
+ * @param {string[]} opts.groupIds  the stack's SG ids (self-reference set)
+ * @param {string[]} [opts.vpcCidrs]  the VPC's CIDR block(s)
+ * @param {boolean} [opts.demo=true]  whether an EC2 demo will launch (adds the
+ *   internet-egress requirement); pass false for --skip-demo
+ */
+export function analyzeSecurityGroupRules(groups, { groupIds = [], vpcCidrs = [], demo = true } = {}) {
+  const warnings = [];
+  const ids = groupIds.length ? groupIds : (groups || []).map((g) => g.GroupId);
+  const opts = { selfIds: ids, vpcCidrs };
+  const list = groups || [];
+
+  const intraVpcEgress443 = list.some((g) => permissionsAllowPort(g.IpPermissionsEgress, 443, opts));
+  const internetEgress443 = list.some((g) => permissionsAllowInternet(g.IpPermissionsEgress, 443));
+  const intraVpcIngress443 = list.some((g) => permissionsAllowPort(g.IpPermissions, 443, opts));
+
+  if (!intraVpcEgress443) {
+    warnings.push(
+      `No outbound (egress) rule allowing TCP 443 within the VPC was found on ${ids.join(', ')}. ` +
+      `OSIS cannot reach the OpenSearch domain${demo ? ' and the demo collector cannot reach the OSIS ingest endpoint' : ''}, so no telemetry will appear. ` +
+      `Add an egress rule allowing TCP 443 to these security groups themselves (self-reference), the VPC CIDR, or 0.0.0.0/0.`
+    );
+  }
+  if (demo && !internetEgress443) {
+    warnings.push(
+      `No outbound (egress) rule allowing TCP 443 to the internet (0.0.0.0/0) was found on ${ids.join(', ')}. ` +
+      `The EC2 demo bootstraps over the public internet (package install, GitHub clone, container image pulls); that traffic does not use the VPC-private endpoint, so a VPC-scoped egress rule is not enough. ` +
+      `Add an egress rule allowing TCP 443 to 0.0.0.0/0 (reachable via a NAT gateway for private subnets, or an internet gateway for public ones).`
+    );
+  }
+  if (!intraVpcIngress443) {
+    warnings.push(
+      `No inbound (ingress) rule allowing TCP 443 from within the VPC was found on ${ids.join(', ')}. ` +
+      `OSIS will reject OTLP${demo ? ' from the demo' : ''} and the domain will reject requests from OSIS. ` +
+      `Add an ingress rule allowing TCP 443 from these security groups themselves (self-reference) or from the VPC CIDR.`
+    );
+  }
+  return warnings;
+}
+
+/**
+ * Best-effort check of the stack's security-group rules against the data path.
+ * Returns WARNING strings (never throws, never blocks the run). When the caller
+ * lacks ec2:DescribeSecurityGroups, returns a single warning telling the user to
+ * verify the rules manually, since we can't inspect them.
+ *
+ * @param {object} cfg  resolved config (needs region, vpcId, securityGroupIds)
+ * @param {object} [deps]  optional injected accessors for testing
+ * @returns {Promise<string[]>}
+ */
+export async function checkSecurityGroupRules(cfg, deps = {}) {
+  if (!cfg.vpcId || !(cfg.securityGroupIds || []).length) return [];
+
+  const demo = !cfg.skipDemo;
+
+  const describeSecurityGroups = deps.describeSecurityGroups || (async (region, ids) => {
+    const { EC2Client, DescribeSecurityGroupsCommand } = await import('@aws-sdk/client-ec2');
+    const client = new EC2Client({ region });
+    return (await client.send(new DescribeSecurityGroupsCommand({ GroupIds: ids }))).SecurityGroups || [];
+  });
+  const describeVpcs = deps.describeVpcs || (async (region, ids) => {
+    const { EC2Client, DescribeVpcsCommand } = await import('@aws-sdk/client-ec2');
+    const client = new EC2Client({ region });
+    return (await client.send(new DescribeVpcsCommand({ VpcIds: ids }))).Vpcs || [];
+  });
+
+  const ids = cfg.securityGroupIds;
+  let groups;
+  try {
+    groups = await describeSecurityGroups(cfg.region, ids);
+  } catch (err) {
+    if (/UnauthorizedOperation|AccessDenied|not authorized/i.test(err.message || err.name || '')) {
+      return [
+        `Could not verify security-group rules (the current role lacks ec2:DescribeSecurityGroups). ` +
+        `Ensure ${ids.join(', ')} allow outbound TCP 443 within the VPC (OSIS to domain${demo ? ', demo collector to OSIS ingest' : ''}), ` +
+        `inbound TCP 443 from within the VPC (OSIS from ${demo ? 'demo, ' : ''}domain from OSIS)` +
+        `${demo ? ', and outbound TCP 443 to 0.0.0.0/0 for the demo to bootstrap over the internet' : ''}, or telemetry will not appear.`,
+      ];
+    }
+    return [`Could not verify security-group rules: ${err.message}`];
+  }
+
+  let vpcCidrs = [];
+  try {
+    const vpcs = await describeVpcs(cfg.region, [cfg.vpcId]);
+    vpcCidrs = (vpcs[0]?.CidrBlockAssociationSet || []).map((a) => a.CidrBlock).filter(Boolean);
+    if (!vpcCidrs.length && vpcs[0]?.CidrBlock) vpcCidrs = [vpcs[0].CidrBlock];
+  } catch { /* VPC CIDR is a refinement; self-reference checks still work without it */ }
+
+  return analyzeSecurityGroupRules(groups, { groupIds: ids, vpcCidrs, demo });
+}
+
 // ── Pipeline listing / describe / update ─────────────────────────────────────
 
 /**

@@ -15,11 +15,13 @@ import {
   createConnectedDataSource,
   createOpenSearchApplication,
   validateVpcTopology,
+  checkSecurityGroupRules,
 } from './aws.mjs';
 import {
   printError,
   printSuccess,
   printStep,
+  printWarning,
   printPanel,
   printBox,
   STAR,
@@ -107,6 +109,13 @@ export async function executePipeline(cfg) {
       throw new Error('VPC configuration is invalid; no resources were created.');
     }
     printSuccess('VPC, subnets, and security groups validated');
+
+    // Best-effort: the topology check above proves the SGs exist and belong to
+    // the VPC, but not that their rules permit the 443 data path. A stripped
+    // egress rule leaves the demo unable to send anything out: no error, just
+    // no data. Warn (never block) before the ~30-min build so it's caught early.
+    const sgWarnings = await checkSecurityGroupRules(cfg);
+    for (const w of sgWarnings) printWarning(w);
     console.error();
   }
 

@@ -0,0 +1,150 @@
+# Local AWS end-to-end tests
+
+These tests exercise the CLI against **real AWS resources**: they create a full
+stack (OpenSearch domain or serverless collection, OSIS pipeline, IAM roles, AMP
+workspace, OpenSearch Application, and optionally an EC2 demo instance), drive
+telemetry through it, confirm the data lands in OpenSearch, then tear everything
+down.
+
+Because they need AWS credentials and cost real money/time, they are **not run in
+CI**; GitHub Actions has no AWS credentials by policy. Run them locally against a
+disposable sandbox account when you touch AWS-specific code (`aws.mjs`,
+`ec2-demo.mjs`, `opensearch-ui-init.mjs`, `render.mjs`, VPC handling, destroy).
+
+> ⚠️ **Sandbox only.** The runner creates and **deletes** OpenSearch domains,
+> OSIS pipelines, IAM roles, EC2 instances, and AMP workspaces. Every resource it
+> touches is prefixed `e2e-`, but never point it at a production account. It
+> prints the target account id up front so you can confirm before it proceeds.
+
+## What is covered
+
+The scenario matrix crosses the AWS-specific dimensions called out in the design:
+
+| Scenario | Backend | Network | Demo | Data-flow verified | Default |
+|---|---|---|---|---|---|
+| `managed-public-demo` | Managed domain | Public | EC2 OTel-demo | ✅ | ✅ |
+| `managed-public-nodemo` | Managed domain | Public | Synthetic OTLP push | ✅ | ✅ |
+| `serverless-public-nodemo` | Serverless (AOSS) | Public | Synthetic OTLP push | ✅ | ✅ |
+| `managed-vpc-demo` | Managed domain | VPC-private | EC2 OTel-demo | ✅ | opt-in |
+| `managed-vpc-nodemo` | Managed domain | VPC-private | (none) | create/destroy only | opt-in |
+
+- **Demo scenarios** launch the EC2 instance that runs the OTel demo + example
+  agents and generate real telemetry. In VPC mode the instance is placed **inside
+  the configured VPC**, so it can reach the VPC-private OSIS ingest endpoint. This
+  is what actually proves end-to-end VPC data flow.
+- **No-demo (public) scenarios** pass `--skip-demo` and instead push a small
+  synthetic OTLP payload (logs + traces) straight at the OSIS ingest endpoint, so
+  they are much faster and cheaper while still proving the ingest to index path.
+- **`managed-vpc-nodemo` is create/destroy only.** A VPC-attached pipeline's ingest
+  endpoint is VPC-private (RFC-1918), so a synthetic push from the developer's host
+  cannot reach it; data-flow verification is not applicable. This scenario still
+  exercises the full VPC create path (the CLI exits 0 only after the VPC-attached
+  pipeline reaches `ACTIVE`) and the teardown path. To verify actual VPC data flow,
+  use `managed-vpc-demo`, whose demo runs in-VPC.
+- **VPC scenarios** are opt-in because they need pre-existing VPC infrastructure
+  (see below). Without the VPC env vars they are reported as `skipped`, not
+  failed, so the default matrix runs anywhere.
+
+When data flow **is** verified, the runner always queries through the managed
+**OpenSearch UI (Application)** endpoint, which is reachable from outside the VPC
+and proxies to the domain over the AWS-internal network, the same path the
+installer uses. This is what lets a VPC-private *domain* be verified from a laptop
+with no bastion or VPN. (The *ingest* endpoint, unlike the domain, is not proxied,
+which is why VPC ingest can only be driven from inside the VPC.)
+
+## Prerequisites
+
+- Node.js 18+ (repo uses 22).
+- AWS credentials for a sandbox account (`aws sts get-caller-identity` succeeds).
+- Permissions for OpenSearch, OpenSearch Serverless, OSIS, AMP, IAM, EC2, SSM,
+  Secrets Manager, and Resource Groups Tagging.
+- For VPC scenarios: a VPC with 1-3 private subnets in distinct AZs and a security
+  group that allows intra-VPC traffic (a group that allows traffic from itself
+  works well).
+
+## Running
+
+From `aws/cli-installer/`:
+
+```bash
+# List the scenario matrix
+npm run e2e:list
+
+# Run the default (public) scenarios in one region
+AWS_PROFILE=my-sandbox npm run e2e -- --region us-east-1
+
+# Run a single scenario
+AWS_PROFILE=my-sandbox node test/e2e/run.mjs --scenario managed-public-nodemo --region us-east-1
+
+# Include the VPC scenarios (requires the env vars below)
+AWS_PROFILE=my-sandbox \
+  E2E_VPC_ID=vpc-0123456789abcdef0 \
+  E2E_SUBNET_IDS=subnet-aaa,subnet-bbb \
+  E2E_SECURITY_GROUP_IDS=sg-0123456789abcdef0 \
+  node test/e2e/run.mjs --all --region us-east-1
+
+# Leave resources up for debugging (remember to destroy them yourself)
+node test/e2e/run.mjs --scenario managed-public-demo --no-teardown --region us-east-1
+```
+
+### Options
+
+| Flag | Meaning |
+|---|---|
+| `--region <r>` | AWS region (default `$AWS_REGION` or `us-east-1`) |
+| `--scenario <name>` | Run only this scenario (repeatable) |
+| `--all` | Include opt-in (VPC) scenarios |
+| `--list` | Print the matrix and exit |
+| `--no-teardown` | Skip destroy after the run (debugging) |
+| `--data-timeout <min>` | Minutes to wait for data to land (default 20) |
+
+### Timing
+
+- Managed + demo: ~15 min to create, plus up to ~15 min for the EC2 demo to
+  bootstrap and telemetry to appear.
+- Serverless / no-demo: a few minutes to create; synthetic data lands quickly
+  once ingest is warm.
+- A freshly created (especially VPC-attached) OSIS pipeline can take ~10-15 min
+  after it reports `ACTIVE` before it actually accepts ingest; the runner keeps
+  retrying the synthetic push during that warmup, so give it the default timeout.
+
+The process exits non-zero if any scenario fails or errors, so it works as a
+pass/fail gate in a local script.
+
+## Troubleshooting
+
+- **OSIS `CreatePipeline` fails with "Unable to create pipeline due to an internal
+  exception."** This is an AWS-side error, not a CLI or harness defect. It is
+  intermittent (observed on a serverless create that then passed unchanged on the
+  next run). Re-run the scenario; if it persists across several attempts, check the
+  OSIS service health for the region.
+- **A run left resources behind (killed mid-run, interrupted, or `--no-teardown`).**
+  `destroy` is idempotent: it skips resources that no longer exist and deletes what
+  remains. Re-run it with the same pipeline name to finish cleanup:
+  ```bash
+  node bin/cli-installer.mjs destroy --pipeline-name <name> --region <region>
+  ```
+  For a VPC scenario, resolve `<name>` from the leftover domain/pipeline (the
+  runner names both after the pipeline). Managed-domain deletion is asynchronous,
+  so the domain may still list as `Deleted: true, Processing: true` for a while
+  after `destroy` returns; its VPC ENIs (described `ES <domain>`) release
+  automatically once deletion completes.
+
+## Unit tests (no AWS)
+
+The pure logic of the harness (the scenario matrix, CLI arg construction,
+document-count evaluation, and synthetic OTLP payload building) is unit-tested in
+`e2e-unit.test.mjs` and runs with the normal unit suite (no credentials needed):
+
+```bash
+npm test          # runs src unit tests + e2e harness unit tests
+```
+
+## Files
+
+| File | Purpose |
+|---|---|
+| `scenarios.mjs` | Scenario matrix + pure CLI arg construction |
+| `verify.mjs` | Synthetic OTLP push + data-flow verification via the UI endpoint |
+| `run.mjs` | Live runner: create → drive telemetry → verify → destroy per scenario |
+| `e2e-unit.test.mjs` | Unit tests for the pure logic above |