diff --git a/aws/cli-installer/package.json b/aws/cli-installer/package.json index 64d0952a..0964ee0e 100644 --- a/aws/cli-installer/package.json +++ b/aws/cli-installer/package.json @@ -6,7 +6,10 @@ "bin": "./bin/cli-installer.mjs", "scripts": { "start": "node bin/open-stack.mjs", - "test": "node --test test/unit.test.mjs" + "test": "node --test test/unit.test.mjs test/e2e/e2e-unit.test.mjs", + "test:unit": "node --test test/unit.test.mjs test/e2e/e2e-unit.test.mjs", + "e2e": "node test/e2e/run.mjs", + "e2e:list": "node test/e2e/run.mjs --list" }, "repository": { "type": "git", diff --git a/aws/cli-installer/src/aws.mjs b/aws/cli-installer/src/aws.mjs index d78f2a31..13d15367 100644 --- a/aws/cli-installer/src/aws.mjs +++ b/aws/cli-installer/src/aws.mjs @@ -1718,6 +1718,162 @@ export async function validateVpcTopology(cfg, deps = {}) { return errors; } +// ── Security-group rule analysis (best-effort data-path check) ─────────────── + +/** + * Does a permission entry cover TCP `port`? IpProtocol '-1' means all protocols + * and all ports. + */ +function permCoversPort(p, port) { + const proto = p.IpProtocol; + return proto === '-1' || (proto === 'tcp' && (p.FromPort ?? 0) <= port && (p.ToPort ?? 65535) >= port); +} + +/** + * Does a set of IpPermissions entries allow TCP `port` from/to an intra-stack + * source? A source counts as intra-stack when the rule references one of the + * stack's own security groups (`selfIds`) or a CIDR not narrower than the VPC + * (0.0.0.0/0, or a VPC-CIDR entry). + */ +export function permissionsAllowPort(permissions, port, { selfIds = [], vpcCidrs = [] } = {}) { + const selfSet = new Set(selfIds); + const cidrOk = (cidr) => cidr === '0.0.0.0/0' || vpcCidrs.includes(cidr); + for (const p of permissions || []) { + if (!permCoversPort(p, port)) continue; + if ((p.UserIdGroupPairs || []).some((g) => selfSet.has(g.GroupId))) return true; + if ((p.IpRanges || []).some((r) => cidrOk(r.CidrIp))) return true; + } + return false; +} + +/** + * Does a set of IpPermissions entries allow TCP `port` to the public internet + * (a 0.0.0.0/0 IpRanges entry)? A self-referencing SG rule or a VPC-scoped CIDR + * does NOT count; those keep traffic inside the VPC. The demo instance needs + * this for bootstrap (dnf, GitHub, container registries), which does not route + * through the VPC-private OSIS endpoint. + */ +export function permissionsAllowInternet(permissions, port) { + for (const p of permissions || []) { + if (!permCoversPort(p, port)) continue; + if ((p.IpRanges || []).some((r) => r.CidrIp === '0.0.0.0/0')) return true; + } + return false; +} + +/** + * Analyze the security groups the stack attaches to the EC2 demo, OSIS pipeline, + * and OpenSearch domain, and return WARNING strings for rules that would silently + * break the data path. Pure: takes the DescribeSecurityGroups result so it is + * unit-testable. Warnings only, never hard-fails: a user's network may route the + * same traffic via other groups, a NAT, or NACLs we can't see. + * + * All three components share `groupIds` (see ec2-demo.mjs / createOsiPipeline / + * createOpenSearch). There are two distinct egress needs, only one of which is + * intra-VPC: + * - Intra-VPC egress 443 (always): OSIS reaches the domain; with a demo, the + * demo's collector reaches the VPC-private OSIS ingest endpoint. Satisfied by + * a self-reference, a VPC-CIDR rule, or 0.0.0.0/0. + * - Internet egress 443 (only with a demo): the instance bootstraps over the + * public internet (dnf, GitHub clone, container image pulls). This traffic + * does NOT flow through the VPC-private endpoint, so only a 0.0.0.0/0 egress + * rule (to a NAT/IGW) satisfies it; a VPC-CIDR-scoped rule does not. + * Plus intra-VPC ingress 443 (always): OSIS accepts OTLP from the demo and the + * domain accepts requests from OSIS. + * + * @param {object[]} groups DescribeSecurityGroups result + * @param {object} opts + * @param {string[]} opts.groupIds the stack's SG ids (self-reference set) + * @param {string[]} [opts.vpcCidrs] the VPC's CIDR block(s) + * @param {boolean} [opts.demo=true] whether an EC2 demo will launch (adds the + * internet-egress requirement); pass false for --skip-demo + */ +export function analyzeSecurityGroupRules(groups, { groupIds = [], vpcCidrs = [], demo = true } = {}) { + const warnings = []; + const ids = groupIds.length ? groupIds : (groups || []).map((g) => g.GroupId); + const opts = { selfIds: ids, vpcCidrs }; + const list = groups || []; + + const intraVpcEgress443 = list.some((g) => permissionsAllowPort(g.IpPermissionsEgress, 443, opts)); + const internetEgress443 = list.some((g) => permissionsAllowInternet(g.IpPermissionsEgress, 443)); + const intraVpcIngress443 = list.some((g) => permissionsAllowPort(g.IpPermissions, 443, opts)); + + if (!intraVpcEgress443) { + warnings.push( + `No outbound (egress) rule allowing TCP 443 within the VPC was found on ${ids.join(', ')}. ` + + `OSIS cannot reach the OpenSearch domain${demo ? ' and the demo collector cannot reach the OSIS ingest endpoint' : ''}, so no telemetry will appear. ` + + `Add an egress rule allowing TCP 443 to these security groups themselves (self-reference), the VPC CIDR, or 0.0.0.0/0.` + ); + } + if (demo && !internetEgress443) { + warnings.push( + `No outbound (egress) rule allowing TCP 443 to the internet (0.0.0.0/0) was found on ${ids.join(', ')}. ` + + `The EC2 demo bootstraps over the public internet (package install, GitHub clone, container image pulls); that traffic does not use the VPC-private endpoint, so a VPC-scoped egress rule is not enough. ` + + `Add an egress rule allowing TCP 443 to 0.0.0.0/0 (reachable via a NAT gateway for private subnets, or an internet gateway for public ones).` + ); + } + if (!intraVpcIngress443) { + warnings.push( + `No inbound (ingress) rule allowing TCP 443 from within the VPC was found on ${ids.join(', ')}. ` + + `OSIS will reject OTLP${demo ? ' from the demo' : ''} and the domain will reject requests from OSIS. ` + + `Add an ingress rule allowing TCP 443 from these security groups themselves (self-reference) or from the VPC CIDR.` + ); + } + return warnings; +} + +/** + * Best-effort check of the stack's security-group rules against the data path. + * Returns WARNING strings (never throws, never blocks the run). When the caller + * lacks ec2:DescribeSecurityGroups, returns a single warning telling the user to + * verify the rules manually, since we can't inspect them. + * + * @param {object} cfg resolved config (needs region, vpcId, securityGroupIds) + * @param {object} [deps] optional injected accessors for testing + * @returns {Promise} + */ +export async function checkSecurityGroupRules(cfg, deps = {}) { + if (!cfg.vpcId || !(cfg.securityGroupIds || []).length) return []; + + const demo = !cfg.skipDemo; + + const describeSecurityGroups = deps.describeSecurityGroups || (async (region, ids) => { + const { EC2Client, DescribeSecurityGroupsCommand } = await import('@aws-sdk/client-ec2'); + const client = new EC2Client({ region }); + return (await client.send(new DescribeSecurityGroupsCommand({ GroupIds: ids }))).SecurityGroups || []; + }); + const describeVpcs = deps.describeVpcs || (async (region, ids) => { + const { EC2Client, DescribeVpcsCommand } = await import('@aws-sdk/client-ec2'); + const client = new EC2Client({ region }); + return (await client.send(new DescribeVpcsCommand({ VpcIds: ids }))).Vpcs || []; + }); + + const ids = cfg.securityGroupIds; + let groups; + try { + groups = await describeSecurityGroups(cfg.region, ids); + } catch (err) { + if (/UnauthorizedOperation|AccessDenied|not authorized/i.test(err.message || err.name || '')) { + return [ + `Could not verify security-group rules (the current role lacks ec2:DescribeSecurityGroups). ` + + `Ensure ${ids.join(', ')} allow outbound TCP 443 within the VPC (OSIS to domain${demo ? ', demo collector to OSIS ingest' : ''}), ` + + `inbound TCP 443 from within the VPC (OSIS from ${demo ? 'demo, ' : ''}domain from OSIS)` + + `${demo ? ', and outbound TCP 443 to 0.0.0.0/0 for the demo to bootstrap over the internet' : ''}, or telemetry will not appear.`, + ]; + } + return [`Could not verify security-group rules: ${err.message}`]; + } + + let vpcCidrs = []; + try { + const vpcs = await describeVpcs(cfg.region, [cfg.vpcId]); + vpcCidrs = (vpcs[0]?.CidrBlockAssociationSet || []).map((a) => a.CidrBlock).filter(Boolean); + if (!vpcCidrs.length && vpcs[0]?.CidrBlock) vpcCidrs = [vpcs[0].CidrBlock]; + } catch { /* VPC CIDR is a refinement; self-reference checks still work without it */ } + + return analyzeSecurityGroupRules(groups, { groupIds: ids, vpcCidrs, demo }); +} + // ── Pipeline listing / describe / update ───────────────────────────────────── /** diff --git a/aws/cli-installer/src/main.mjs b/aws/cli-installer/src/main.mjs index 3a62f0a4..af291159 100644 --- a/aws/cli-installer/src/main.mjs +++ b/aws/cli-installer/src/main.mjs @@ -15,11 +15,13 @@ import { createConnectedDataSource, createOpenSearchApplication, validateVpcTopology, + checkSecurityGroupRules, } from './aws.mjs'; import { printError, printSuccess, printStep, + printWarning, printPanel, printBox, STAR, @@ -107,6 +109,13 @@ export async function executePipeline(cfg) { throw new Error('VPC configuration is invalid; no resources were created.'); } printSuccess('VPC, subnets, and security groups validated'); + + // Best-effort: the topology check above proves the SGs exist and belong to + // the VPC, but not that their rules permit the 443 data path. A stripped + // egress rule leaves the demo unable to send anything out: no error, just + // no data. Warn (never block) before the ~30-min build so it's caught early. + const sgWarnings = await checkSecurityGroupRules(cfg); + for (const w of sgWarnings) printWarning(w); console.error(); } diff --git a/aws/cli-installer/test/e2e/README.md b/aws/cli-installer/test/e2e/README.md new file mode 100644 index 00000000..aa3cba39 --- /dev/null +++ b/aws/cli-installer/test/e2e/README.md @@ -0,0 +1,150 @@ +# Local AWS end-to-end tests + +These tests exercise the CLI against **real AWS resources**: they create a full +stack (OpenSearch domain or serverless collection, OSIS pipeline, IAM roles, AMP +workspace, OpenSearch Application, and optionally an EC2 demo instance), drive +telemetry through it, confirm the data lands in OpenSearch, then tear everything +down. + +Because they need AWS credentials and cost real money/time, they are **not run in +CI**; GitHub Actions has no AWS credentials by policy. Run them locally against a +disposable sandbox account when you touch AWS-specific code (`aws.mjs`, +`ec2-demo.mjs`, `opensearch-ui-init.mjs`, `render.mjs`, VPC handling, destroy). + +> ⚠️ **Sandbox only.** The runner creates and **deletes** OpenSearch domains, +> OSIS pipelines, IAM roles, EC2 instances, and AMP workspaces. Every resource it +> touches is prefixed `e2e-`, but never point it at a production account. It +> prints the target account id up front so you can confirm before it proceeds. + +## What is covered + +The scenario matrix crosses the AWS-specific dimensions called out in the design: + +| Scenario | Backend | Network | Demo | Data-flow verified | Default | +|---|---|---|---|---|---| +| `managed-public-demo` | Managed domain | Public | EC2 OTel-demo | ✅ | ✅ | +| `managed-public-nodemo` | Managed domain | Public | Synthetic OTLP push | ✅ | ✅ | +| `serverless-public-nodemo` | Serverless (AOSS) | Public | Synthetic OTLP push | ✅ | ✅ | +| `managed-vpc-demo` | Managed domain | VPC-private | EC2 OTel-demo | ✅ | opt-in | +| `managed-vpc-nodemo` | Managed domain | VPC-private | (none) | create/destroy only | opt-in | + +- **Demo scenarios** launch the EC2 instance that runs the OTel demo + example + agents and generate real telemetry. In VPC mode the instance is placed **inside + the configured VPC**, so it can reach the VPC-private OSIS ingest endpoint. This + is what actually proves end-to-end VPC data flow. +- **No-demo (public) scenarios** pass `--skip-demo` and instead push a small + synthetic OTLP payload (logs + traces) straight at the OSIS ingest endpoint, so + they are much faster and cheaper while still proving the ingest to index path. +- **`managed-vpc-nodemo` is create/destroy only.** A VPC-attached pipeline's ingest + endpoint is VPC-private (RFC-1918), so a synthetic push from the developer's host + cannot reach it; data-flow verification is not applicable. This scenario still + exercises the full VPC create path (the CLI exits 0 only after the VPC-attached + pipeline reaches `ACTIVE`) and the teardown path. To verify actual VPC data flow, + use `managed-vpc-demo`, whose demo runs in-VPC. +- **VPC scenarios** are opt-in because they need pre-existing VPC infrastructure + (see below). Without the VPC env vars they are reported as `skipped`, not + failed, so the default matrix runs anywhere. + +When data flow **is** verified, the runner always queries through the managed +**OpenSearch UI (Application)** endpoint, which is reachable from outside the VPC +and proxies to the domain over the AWS-internal network, the same path the +installer uses. This is what lets a VPC-private *domain* be verified from a laptop +with no bastion or VPN. (The *ingest* endpoint, unlike the domain, is not proxied, +which is why VPC ingest can only be driven from inside the VPC.) + +## Prerequisites + +- Node.js 18+ (repo uses 22). +- AWS credentials for a sandbox account (`aws sts get-caller-identity` succeeds). +- Permissions for OpenSearch, OpenSearch Serverless, OSIS, AMP, IAM, EC2, SSM, + Secrets Manager, and Resource Groups Tagging. +- For VPC scenarios: a VPC with 1-3 private subnets in distinct AZs and a security + group that allows intra-VPC traffic (a group that allows traffic from itself + works well). + +## Running + +From `aws/cli-installer/`: + +```bash +# List the scenario matrix +npm run e2e:list + +# Run the default (public) scenarios in one region +AWS_PROFILE=my-sandbox npm run e2e -- --region us-east-1 + +# Run a single scenario +AWS_PROFILE=my-sandbox node test/e2e/run.mjs --scenario managed-public-nodemo --region us-east-1 + +# Include the VPC scenarios (requires the env vars below) +AWS_PROFILE=my-sandbox \ + E2E_VPC_ID=vpc-0123456789abcdef0 \ + E2E_SUBNET_IDS=subnet-aaa,subnet-bbb \ + E2E_SECURITY_GROUP_IDS=sg-0123456789abcdef0 \ + node test/e2e/run.mjs --all --region us-east-1 + +# Leave resources up for debugging (remember to destroy them yourself) +node test/e2e/run.mjs --scenario managed-public-demo --no-teardown --region us-east-1 +``` + +### Options + +| Flag | Meaning | +|---|---| +| `--region ` | AWS region (default `$AWS_REGION` or `us-east-1`) | +| `--scenario ` | Run only this scenario (repeatable) | +| `--all` | Include opt-in (VPC) scenarios | +| `--list` | Print the matrix and exit | +| `--no-teardown` | Skip destroy after the run (debugging) | +| `--data-timeout ` | Minutes to wait for data to land (default 20) | + +### Timing + +- Managed + demo: ~15 min to create, plus up to ~15 min for the EC2 demo to + bootstrap and telemetry to appear. +- Serverless / no-demo: a few minutes to create; synthetic data lands quickly + once ingest is warm. +- A freshly created (especially VPC-attached) OSIS pipeline can take ~10-15 min + after it reports `ACTIVE` before it actually accepts ingest; the runner keeps + retrying the synthetic push during that warmup, so give it the default timeout. + +The process exits non-zero if any scenario fails or errors, so it works as a +pass/fail gate in a local script. + +## Troubleshooting + +- **OSIS `CreatePipeline` fails with "Unable to create pipeline due to an internal + exception."** This is an AWS-side error, not a CLI or harness defect. It is + intermittent (observed on a serverless create that then passed unchanged on the + next run). Re-run the scenario; if it persists across several attempts, check the + OSIS service health for the region. +- **A run left resources behind (killed mid-run, interrupted, or `--no-teardown`).** + `destroy` is idempotent: it skips resources that no longer exist and deletes what + remains. Re-run it with the same pipeline name to finish cleanup: + ```bash + node bin/cli-installer.mjs destroy --pipeline-name --region + ``` + For a VPC scenario, resolve `` from the leftover domain/pipeline (the + runner names both after the pipeline). Managed-domain deletion is asynchronous, + so the domain may still list as `Deleted: true, Processing: true` for a while + after `destroy` returns; its VPC ENIs (described `ES `) release + automatically once deletion completes. + +## Unit tests (no AWS) + +The pure logic of the harness (the scenario matrix, CLI arg construction, +document-count evaluation, and synthetic OTLP payload building) is unit-tested in +`e2e-unit.test.mjs` and runs with the normal unit suite (no credentials needed): + +```bash +npm test # runs src unit tests + e2e harness unit tests +``` + +## Files + +| File | Purpose | +|---|---| +| `scenarios.mjs` | Scenario matrix + pure CLI arg construction | +| `verify.mjs` | Synthetic OTLP push + data-flow verification via the UI endpoint | +| `run.mjs` | Live runner: create → drive telemetry → verify → destroy per scenario | +| `e2e-unit.test.mjs` | Unit tests for the pure logic above | diff --git a/aws/cli-installer/test/e2e/e2e-unit.test.mjs b/aws/cli-installer/test/e2e/e2e-unit.test.mjs new file mode 100644 index 00000000..58d1f542 --- /dev/null +++ b/aws/cli-installer/test/e2e/e2e-unit.test.mjs @@ -0,0 +1,302 @@ +/** + * Unit tests for the PURE parts of the e2e harness — arg construction, the + * scenario matrix, count evaluation, and synthetic OTLP payloads. These run + * anywhere (no AWS, no credentials) and guard the harness logic itself. + * + * Run: node --test test/e2e/e2e-unit.test.mjs + * (Also picked up by `node --test` over the whole test/ tree.) + */ +import { describe, it } from 'node:test'; +import assert from 'node:assert/strict'; + +import { + SCENARIOS, getScenario, selectScenarios, envToVpc, + buildPipelineName, buildCreateArgs, buildDestroyArgs, +} from '../e2e/scenarios.mjs'; +import { + EXPECTED_INDICES, evaluateCounts, parseCount, buildOtlpPayload, otlpUrl, +} from '../e2e/verify.mjs'; + +// ── scenario matrix ─────────────────────────────────────────────────────────── + +describe('scenario matrix', () => { + it('covers both network topologies and both demo modes', () => { + assert.ok(SCENARIOS.some((s) => s.vpc), 'expected at least one VPC scenario'); + assert.ok(SCENARIOS.some((s) => !s.vpc), 'expected at least one public scenario'); + assert.ok(SCENARIOS.some((s) => s.demo), 'expected at least one demo scenario'); + assert.ok(SCENARIOS.some((s) => !s.demo), 'expected at least one no-demo scenario'); + }); + + it('covers both backends', () => { + assert.ok(SCENARIOS.some((s) => s.backend === 'managed')); + assert.ok(SCENARIOS.some((s) => s.backend === 'serverless')); + }); + + it('all scenario names are unique', () => { + const names = SCENARIOS.map((s) => s.name); + assert.equal(new Set(names).size, names.length); + }); + + it('VPC scenarios are opt-in (not default) so the base matrix runs anywhere', () => { + for (const s of SCENARIOS) { + if (s.vpc) assert.equal(s.enabledByDefault, false, `${s.name} should be opt-in`); + } + }); + + it('every scenario declares whether data flow is verified', () => { + for (const s of SCENARIOS) { + assert.equal(typeof s.verifyDataFlow, 'boolean', `${s.name} must set verifyDataFlow`); + } + }); + + it('the VPC no-demo scenario does not verify data flow (private ingest unreachable from outside)', () => { + // A VPC-attached OSIS ingest endpoint is VPC-private, so a synthetic push from + // the runner host can never reach it — verifying data flow there would be a + // guaranteed false failure. VPC data flow is proven by the in-VPC demo instead. + assert.equal(getScenario('managed-vpc-nodemo').verifyDataFlow, false); + assert.equal(getScenario('managed-vpc-demo').verifyDataFlow, true); + }); + + it('every public scenario verifies data flow', () => { + for (const s of SCENARIOS) { + if (!s.vpc) assert.equal(s.verifyDataFlow, true, `${s.name} (public) should verify data flow`); + } + }); +}); + +describe('getScenario / selectScenarios', () => { + it('getScenario returns the named scenario', () => { + assert.equal(getScenario('managed-public-demo').name, 'managed-public-demo'); + }); + it('getScenario throws on an unknown name', () => { + assert.throws(() => getScenario('nope'), /Unknown scenario/); + }); + it('selectScenarios defaults to enabled-by-default only', () => { + const sel = selectScenarios(); + assert.ok(sel.length > 0); + assert.ok(sel.every((s) => s.enabledByDefault)); + }); + it('selectScenarios --all includes opt-in scenarios', () => { + assert.equal(selectScenarios({ all: true }).length, SCENARIOS.length); + }); + it('selectScenarios by name overrides defaults', () => { + const sel = selectScenarios({ names: ['managed-vpc-demo'] }); + assert.deepEqual(sel.map((s) => s.name), ['managed-vpc-demo']); + }); +}); + +// ── env → VPC parsing ─────────────────────────────────────────────────────── + +describe('envToVpc', () => { + it('returns null when nothing is set', () => { + assert.equal(envToVpc({}), null); + }); + it('returns null when only partially set', () => { + assert.equal(envToVpc({ E2E_VPC_ID: 'vpc-1' }), null); + }); + it('parses a complete VPC config with comma-separated lists', () => { + const v = envToVpc({ + E2E_VPC_ID: 'vpc-03e1', + E2E_SUBNET_IDS: 'subnet-a, subnet-b', + E2E_SECURITY_GROUP_IDS: 'sg-1', + }); + assert.deepEqual(v, { vpcId: 'vpc-03e1', subnetIds: ['subnet-a', 'subnet-b'], securityGroupIds: ['sg-1'] }); + }); +}); + +// ── pipeline naming ─────────────────────────────────────────────────────────── + +describe('buildPipelineName', () => { + it('stays within the 28-char OSIS limit', () => { + for (const s of SCENARIOS) { + const name = buildPipelineName(s, 'abcde'); + assert.ok(name.length <= 28, `${s.name} -> ${name} (${name.length})`); + } + }); + it('starts with a lowercase letter and is alnum/hyphen only', () => { + const name = buildPipelineName(getScenario('managed-public-demo'), 'zzzzz'); + assert.match(name, /^[a-z][a-z0-9-]*$/); + }); + it('is distinct per scenario for the same suffix', () => { + const suffix = 'q1w2e'; + const names = SCENARIOS.map((s) => buildPipelineName(s, suffix)); + assert.equal(new Set(names).size, names.length); + }); +}); + +// ── create/destroy arg construction ───────────────────────────────────────── + +describe('buildCreateArgs', () => { + const base = { pipelineName: 'e2e-mpd-abcde', region: 'us-east-1' }; + + it('managed public demo: managed domain, no VPC, demo not skipped', () => { + const args = buildCreateArgs(getScenario('managed-public-demo'), base); + assert.ok(args.includes('--managed')); + assert.ok(args.includes('--os-domain-name')); + assert.ok(!args.includes('--vpc-id')); + assert.ok(!args.includes('--skip-demo')); + // Must NOT use --advanced: advanced mode only creates explicitly-named + // resources, so it would skip the OSI role and hand OSIS an empty + // PipelineRoleArn. Quick mode (the default) auto-creates the full stack. + assert.ok(!args.includes('--advanced')); + }); + + it('no-demo scenario adds --skip-demo', () => { + const args = buildCreateArgs(getScenario('managed-public-nodemo'), base); + assert.ok(args.includes('--skip-demo')); + }); + + it('serverless scenario uses --serverless and --aoss-collection-name', () => { + const args = buildCreateArgs(getScenario('serverless-public-nodemo'), base); + assert.ok(args.includes('--serverless')); + assert.ok(args.includes('--aoss-collection-name')); + assert.ok(!args.includes('--managed')); + }); + + it('VPC scenario threads vpc/subnet/sg flags', () => { + const args = buildCreateArgs(getScenario('managed-vpc-demo'), { + ...base, vpc: { vpcId: 'vpc-1', subnetIds: ['subnet-a', 'subnet-b'], securityGroupIds: ['sg-1'] }, + }); + const i = args.indexOf('--vpc-id'); + assert.equal(args[i + 1], 'vpc-1'); + assert.equal(args[args.indexOf('--subnet-ids') + 1], 'subnet-a,subnet-b'); + assert.equal(args[args.indexOf('--security-group-ids') + 1], 'sg-1'); + }); + + it('VPC scenario without vpc params throws', () => { + assert.throws(() => buildCreateArgs(getScenario('managed-vpc-demo'), base), /requires vpc/); + }); + + it('requires pipelineName and region', () => { + assert.throws(() => buildCreateArgs(getScenario('managed-public-demo'), { region: 'us-east-1' }), /pipelineName/); + assert.throws(() => buildCreateArgs(getScenario('managed-public-demo'), { pipelineName: 'x' }), /region/); + }); + + it('produces args that validateConfig accepts and resolve a full stack (integration with cli.mjs)', async () => { + // Cross-check: the args we build should parse and validate cleanly, AND once + // defaults are applied they must resolve into an actionable full stack — an + // OSI role, an APS workspace, and an app. This guards against a regression + // where the harness selected --advanced and left iamAction empty, which made + // the CLI create only the domain and then fail deep in OSIS pipeline creation + // with an empty PipelineRoleArn ("Cross-account pass role is not allowed"). + const { parseCli, applyQuickDefaults, validateConfig } = await import('../../src/cli.mjs'); + for (const name of ['managed-public-nodemo', 'serverless-public-nodemo', 'managed-public-demo']) { + const args = buildCreateArgs(getScenario(name), base); + const cfg = parseCli(['node', 'cli', ...args]); + if (cfg.mode === 'quick') applyQuickDefaults(cfg); + assert.deepEqual(validateConfig(cfg), [], `${name} should validate`); + // The OSI pipeline role must be resolved to "create" with a name, or the + // pipeline gets an empty role ARN. + assert.equal(cfg.iamAction, 'create', `${name} must create an OSI role`); + assert.ok(cfg.iamRoleName, `${name} must have an OSI role name`); + assert.equal(cfg.apsAction, 'create', `${name} must create an APS workspace`); + assert.ok(cfg.appName, `${name} must have an OpenSearch Application name`); + } + }); + + it('VPC scenario args also resolve a full stack (quick mode + VPC flags)', async () => { + const { parseCli, applyQuickDefaults, validateConfig } = await import('../../src/cli.mjs'); + const args = buildCreateArgs(getScenario('managed-vpc-nodemo'), { + ...base, vpc: { vpcId: 'vpc-1', subnetIds: ['subnet-a', 'subnet-b'], securityGroupIds: ['sg-1'] }, + }); + const cfg = parseCli(['node', 'cli', ...args]); + if (cfg.mode === 'quick') applyQuickDefaults(cfg); + assert.deepEqual(validateConfig(cfg), []); + assert.equal(cfg.iamAction, 'create'); + assert.equal(cfg.vpcId, 'vpc-1'); + assert.equal(cfg.osAction, 'create'); + }); +}); + +describe('buildDestroyArgs', () => { + it('builds a destroy invocation with pipeline + region', () => { + const args = buildDestroyArgs(getScenario('managed-public-demo'), { pipelineName: 'p', region: 'us-east-1' }); + assert.equal(args[0], 'destroy'); + assert.ok(args.includes('--pipeline-name')); + assert.ok(args.includes('--region')); + }); + it('serverless destroy passes the collection name', () => { + const args = buildDestroyArgs(getScenario('serverless-public-nodemo'), { pipelineName: 'p', region: 'us-east-1' }); + assert.ok(args.includes('--aoss-collection-name')); + }); +}); + +// ── count evaluation ────────────────────────────────────────────────────────── + +describe('evaluateCounts', () => { + const full = { 'logs-otel-v1': 5, 'otel-v1-apm-span': 12, 'otel-v2-apm-service-map': 3 }; + + it('passes when every expected index has data', () => { + const r = evaluateCounts(full); + assert.equal(r.ok, true); + assert.equal(r.results.length, EXPECTED_INDICES.length); + }); + + it('fails when a required index is empty', () => { + const r = evaluateCounts({ ...full, 'logs-otel-v1': 0 }); + assert.equal(r.ok, false); + assert.equal(r.results.find((x) => x.pattern === 'logs-otel-v1').ok, false); + }); + + it('treats non-required signals as optional (service-map for synthetic push)', () => { + const r = evaluateCounts( + { 'logs-otel-v1': 2, 'otel-v1-apm-span': 2, 'otel-v2-apm-service-map': 0 }, + { requireSignals: ['logs', 'traces'] }, + ); + assert.equal(r.ok, true); + assert.equal(r.results.find((x) => x.pattern === 'otel-v2-apm-service-map').ok, true); + }); + + it('missing keys count as zero', () => { + const r = evaluateCounts({}); + assert.equal(r.ok, false); + }); +}); + +describe('parseCount', () => { + it('reads count from an object', () => assert.equal(parseCount({ count: 42 }), 42)); + it('reads count from a JSON string', () => assert.equal(parseCount('{"count":7}'), 7)); + it('returns 0 on garbage', () => assert.equal(parseCount('not json'), 0)); + it('returns 0 when count is missing', () => assert.equal(parseCount({ hits: 1 }), 0)); +}); + +// ── synthetic OTLP ──────────────────────────────────────────────────────────── + +describe('buildOtlpPayload', () => { + it('builds a logs payload with service.name and a timestamp', () => { + const p = buildOtlpPayload('logs', { nowNanos: '1700000000000000000', serviceName: 'svc' }); + const rl = p.resourceLogs[0]; + assert.equal(rl.resource.attributes[0].value.stringValue, 'svc'); + assert.equal(rl.scopeLogs[0].logRecords[0].timeUnixNano, '1700000000000000000'); + }); + + it('builds a traces payload with a server span and derived endTime', () => { + const p = buildOtlpPayload('traces', { nowNanos: '1000', serviceName: 'svc' }); + const span = p.resourceSpans[0].scopeSpans[0].spans[0]; + assert.equal(span.startTimeUnixNano, '1000'); + assert.equal(span.endTimeUnixNano, '1001000'); // start + 1_000_000 + assert.equal(span.kind, 2); + }); + + it('carries service.name on traces so service-map edges can be derived', () => { + const p = buildOtlpPayload('traces', { serviceName: 'checkout' }); + assert.equal(p.resourceSpans[0].resource.attributes[0].value.stringValue, 'checkout'); + }); + + it('throws on unsupported signals', () => { + assert.throws(() => buildOtlpPayload('metrics'), /unsupported signal/); + }); +}); + +describe('otlpUrl', () => { + it('builds the per-pipeline OTLP path and strips any scheme on the endpoint', () => { + assert.equal( + otlpUrl('https://abc.us-east-1.osis.amazonaws.com', 'my-pipe', 'logs'), + 'https://abc.us-east-1.osis.amazonaws.com/my-pipe/v1/logs', + ); + assert.equal( + otlpUrl('abc.us-east-1.osis.amazonaws.com', 'my-pipe', 'traces'), + 'https://abc.us-east-1.osis.amazonaws.com/my-pipe/v1/traces', + ); + }); +}); diff --git a/aws/cli-installer/test/e2e/run.mjs b/aws/cli-installer/test/e2e/run.mjs new file mode 100644 index 00000000..35fac81e --- /dev/null +++ b/aws/cli-installer/test/e2e/run.mjs @@ -0,0 +1,227 @@ +#!/usr/bin/env node +/** + * Local AWS end-to-end test runner for the observability-stack CLI. + * + * These tests hit real AWS resources, so they are NOT run in CI (GitHub has no + * AWS credentials by policy). Developers run them locally against a sandbox + * account to gain confidence that AWS-specific changes still create a working + * stack end to end. + * + * For each selected scenario it: + * 1. creates the stack via the real CLI (`bin/cli-installer.mjs`), + * 2. drives telemetry in — the EC2 OTel-demo (demo scenarios) or a synthetic + * OTLP push (no-demo scenarios), + * 3. verifies documents land in the expected OpenSearch indices by querying + * through the managed OpenSearch UI endpoint (works for VPC-private domains), + * 4. tears the stack down — always, even on failure (unless --no-teardown). + * + * Usage: + * AWS_PROFILE= node test/e2e/run.mjs [options] + * + * Options: + * --region AWS region (default: $AWS_REGION or us-east-1) + * --scenario run only this scenario (repeatable) + * --all include opt-in scenarios (VPC) too + * --list print the scenario matrix and exit + * --no-teardown leave resources up after the run (for debugging) + * --data-timeout minutes to wait for data to land (default 20) + * + * VPC scenarios additionally need: + * E2E_VPC_ID, E2E_SUBNET_IDS (comma-sep), E2E_SECURITY_GROUP_IDS (comma-sep) + * + * SAFETY: intended for a disposable sandbox account. It creates and DELETES + * OpenSearch domains, OSIS pipelines, IAM roles, EC2 instances, and AMP + * workspaces prefixed `e2e-`. Never point it at a production account. + */ +import { spawn } from 'node:child_process'; +import { fileURLToPath } from 'node:url'; +import { dirname, join } from 'node:path'; +import { STSClient, GetCallerIdentityCommand } from '@aws-sdk/client-sts'; +import { OSISClient, GetPipelineCommand } from '@aws-sdk/client-osis'; +import { OpenSearchClient, ListApplicationsCommand, GetApplicationCommand } from '@aws-sdk/client-opensearch'; +import { selectScenarios, envToVpc, buildPipelineName, buildCreateArgs, buildDestroyArgs, SCENARIOS } from './scenarios.mjs'; +import { findDataSourceId, waitForData, pushOtlp, buildOtlpPayload } from './verify.mjs'; + +const __dirname = dirname(fileURLToPath(import.meta.url)); +const CLI = join(__dirname, '..', '..', 'bin', 'cli-installer.mjs'); + +// ── arg parsing ───────────────────────────────────────────────────────────── +function parseArgs(argv) { + const out = { scenarios: [], all: false, list: false, teardown: true, region: process.env.AWS_REGION || process.env.AWS_DEFAULT_REGION || 'us-east-1', dataTimeoutMin: 20 }; + for (let i = 2; i < argv.length; i++) { + const a = argv[i]; + if (a === '--all') out.all = true; + else if (a === '--list') out.list = true; + else if (a === '--no-teardown') out.teardown = false; + else if (a === '--scenario') out.scenarios.push(argv[++i]); + else if (a === '--region') out.region = argv[++i]; + else if (a === '--data-timeout') out.dataTimeoutMin = Number(argv[++i]); + else { console.error(`Unknown option: ${a}`); process.exit(2); } + } + return out; +} + +function log(scenario, msg) { + const ts = new Date().toISOString().slice(11, 19); + console.log(`[${ts}] [${scenario}] ${msg}`); +} + +/** Run the CLI as a subprocess, streaming output. Resolves with exit code. */ +function runCli(args) { + return new Promise((resolve) => { + const child = spawn('node', [CLI, ...args], { stdio: 'inherit', env: process.env }); + child.on('close', (code) => resolve(code ?? 1)); + }); +} + +/** Short run id from the current time — collision-avoidance suffix for names. */ +function runSuffix() { + return Math.floor(Date.now() / 1000).toString(36).slice(-5); +} + +async function verifyScenario(scenario, { pipelineName, region, dataTimeoutMs }) { + // Resolve the Application endpoint + ingest endpoint from live resources. + const os = new OpenSearchClient({ region }); + const { ApplicationSummaries } = await os.send(new ListApplicationsCommand({})); + const app = (ApplicationSummaries || []).find((a) => a.name === pipelineName); + if (!app) throw new Error('OpenSearch Application not found — cannot verify data flow'); + const { endpoint: appEndpoint } = await os.send(new GetApplicationCommand({ id: app.id })); + if (!appEndpoint) throw new Error('OpenSearch Application endpoint not populated'); + log(scenario.name, `app endpoint: ${appEndpoint}`); + + const dataSourceId = await findDataSourceId({ appEndpoint, region }); + log(scenario.name, `data source id: ${dataSourceId || '(none)'}`); + + // No-demo scenarios: push synthetic telemetry so there is something to verify. + let requireSignals; + if (!scenario.demo) { + const osis = new OSISClient({ region }); + const { Pipeline } = await osis.send(new GetPipelineCommand({ PipelineName: pipelineName })); + const ingest = Pipeline?.IngestEndpointUrls?.[0]; + if (!ingest) throw new Error('OSIS ingest endpoint not available for synthetic push'); + requireSignals = ['logs', 'traces']; // service-map is derived async; treat as optional + await pushSynthetic({ scenario, ingestEndpoint: ingest, pipelineName, region }); + } + + const result = await waitForData({ + appEndpoint, region, dataSourceId, requireSignals, + timeoutMs: dataTimeoutMs, log: (m) => log(scenario.name, m), + }); + return result; +} + +/** Push a few synthetic OTLP records repeatedly so counts clear zero. */ +async function pushSynthetic({ scenario, ingestEndpoint, pipelineName, region }) { + log(scenario.name, `pushing synthetic OTLP to ${ingestEndpoint}`); + // OSIS ingest for a fresh (esp. VPC) pipeline can warm up for ~10-15 min; + // retry pushes until at least one is accepted. + for (let attempt = 0; attempt < 30; attempt++) { + const nowNanos = String(BigInt(Date.now()) * 1_000_000n); + let accepted = 0; + for (const signal of ['logs', 'traces']) { + const payload = buildOtlpPayload(signal, { nowNanos, serviceName: `e2e-${scenario.name}` }); + try { + const { status } = await pushOtlp({ ingestEndpoint, pipelineName, region, signal, payload }); + if (status >= 200 && status < 300) accepted++; + else log(scenario.name, ` ${signal} push -> HTTP ${status}`); + } catch (e) { + log(scenario.name, ` ${signal} push error: ${e.message}`); + } + } + if (accepted === 2) { log(scenario.name, 'synthetic push accepted'); } + await new Promise((r) => setTimeout(r, 30_000)); + } +} + +async function main() { + const opts = parseArgs(process.argv); + + if (opts.list) { + console.log('Available scenarios:\n'); + for (const s of SCENARIOS) { + console.log(` ${s.name.padEnd(26)} ${s.enabledByDefault ? '(default)' : '(opt-in) '} ${s.description}`); + } + console.log('\nVPC scenarios need E2E_VPC_ID / E2E_SUBNET_IDS / E2E_SECURITY_GROUP_IDS.'); + return; + } + + // Confirm credentials up front and print the target account (safety). + const sts = new STSClient({ region: opts.region }); + const id = await sts.send(new GetCallerIdentityCommand({})); + console.log(`Target AWS account: ${id.Account} (${id.Arn})`); + console.log(`Region: ${opts.region}\n`); + + const selected = selectScenarios({ names: opts.scenarios, all: opts.all }); + const vpc = envToVpc(); + const suffix = runSuffix(); + const dataTimeoutMs = opts.dataTimeoutMin * 60_000; + const summary = []; + + for (const scenario of selected) { + if (scenario.vpc && !vpc) { + log(scenario.name, 'SKIP — VPC env vars not set (E2E_VPC_ID / E2E_SUBNET_IDS / E2E_SECURITY_GROUP_IDS)'); + summary.push({ scenario: scenario.name, status: 'skipped', reason: 'no VPC env' }); + continue; + } + + const pipelineName = buildPipelineName(scenario, suffix); + const base = { pipelineName, region: opts.region, vpc }; + let status = 'unknown'; + let detail = ''; + + log(scenario.name, `=== START (pipeline: ${pipelineName}) ===`); + try { + const createArgs = buildCreateArgs(scenario, base); + log(scenario.name, `create: node cli-installer.mjs ${createArgs.join(' ')}`); + const code = await runCli(createArgs); + if (code !== 0) throw new Error(`CLI create exited ${code}`); + + if (scenario.verifyDataFlow === false) { + // Data-flow verification isn't applicable (e.g. a VPC-private ingest + // endpoint the runner host can't reach). A clean create — the CLI exits 0 + // only after every resource, including the pipeline reaching ACTIVE — plus + // the teardown below is what this scenario proves. + status = 'passed'; + detail = 'created (data-flow verification not applicable; teardown exercised)'; + log(scenario.name, 'skipping data-flow verification for this scenario (not applicable)'); + } else { + const result = await verifyScenario(scenario, { pipelineName, region: opts.region, dataTimeoutMs }); + if (result.ok) { + status = 'passed'; + detail = result.results.map((r) => `${r.pattern}=${r.count}`).join(' '); + } else { + status = 'failed'; + detail = `data not confirmed${result.timedOut ? ' (timeout)' : ''}: ` + + result.results.map((r) => `${r.pattern}=${r.count}${r.ok ? '' : ' ✗'}`).join(' '); + } + } + } catch (e) { + status = 'error'; + detail = e.message; + log(scenario.name, `ERROR: ${e.message}`); + } finally { + if (opts.teardown) { + log(scenario.name, 'tearing down...'); + const code = await runCli(buildDestroyArgs(scenario, base)); + if (code !== 0) log(scenario.name, `WARNING: destroy exited ${code} — check for leftover resources`); + } else { + log(scenario.name, 'teardown skipped (--no-teardown) — remember to destroy manually'); + } + } + + log(scenario.name, `=== ${status.toUpperCase()} — ${detail} ===\n`); + summary.push({ scenario: scenario.name, status, detail }); + } + + // ── Summary ── + console.log('\n================ E2E SUMMARY ================'); + for (const s of summary) { + console.log(` ${s.status.toUpperCase().padEnd(8)} ${s.scenario.padEnd(26)} ${s.detail || s.reason || ''}`); + } + const failed = summary.filter((s) => s.status === 'failed' || s.status === 'error'); + console.log(`\n${summary.length} scenario(s): ${summary.filter(s => s.status === 'passed').length} passed, ` + + `${failed.length} failed/errored, ${summary.filter(s => s.status === 'skipped').length} skipped`); + process.exit(failed.length ? 1 : 0); +} + +main().catch((e) => { console.error(e); process.exit(1); }); diff --git a/aws/cli-installer/test/e2e/scenarios.mjs b/aws/cli-installer/test/e2e/scenarios.mjs new file mode 100644 index 00000000..78cebe29 --- /dev/null +++ b/aws/cli-installer/test/e2e/scenarios.mjs @@ -0,0 +1,192 @@ +/** + * E2E scenario matrix + CLI argument construction. + * + * These functions are PURE (no AWS, no I/O) so they can be unit-tested without + * credentials. The live runner (run.mjs) turns each scenario into a real CLI + * invocation, verifies data flow, then tears the stack down. + * + * The matrix covers the two dimensions called out for AWS-specific coverage: + * - network topology: public endpoints vs. VPC-private endpoints + * - demo workload: launch the EC2 OTel-demo instance vs. skip it + * plus the managed-domain vs. serverless backend split. + */ + +/** + * The default scenario matrix. `enabledByDefault` scenarios run in a plain + * `run.mjs` invocation; the rest are opt-in (they cost more time/money or need + * extra inputs such as a VPC). A developer can also select scenarios by name. + * + * VPC scenarios require --vpc-id/--subnet-ids/--security-group-ids, supplied via + * env (see envToVpc). They are opt-in because they need pre-existing VPC infra. + */ +export const SCENARIOS = [ + { + name: 'managed-public-demo', + description: 'Managed domain, public endpoints, EC2 OTel-demo launched', + backend: 'managed', + vpc: false, + demo: true, + enabledByDefault: true, + verifyDataFlow: true, + }, + { + name: 'managed-public-nodemo', + description: 'Managed domain, public endpoints, no demo (synthetic OTLP push)', + backend: 'managed', + vpc: false, + demo: false, + enabledByDefault: true, + verifyDataFlow: true, + }, + { + name: 'serverless-public-nodemo', + description: 'Serverless (AOSS) collection, public endpoints, synthetic OTLP push', + backend: 'serverless', + vpc: false, + demo: false, + enabledByDefault: true, + verifyDataFlow: true, + }, + { + name: 'managed-vpc-demo', + description: 'Managed domain in a VPC (private endpoints), EC2 OTel-demo launched', + backend: 'managed', + vpc: true, + demo: true, + enabledByDefault: false, + // The EC2 demo launches INSIDE the configured VPC (see ec2-demo.mjs), so it + // can reach the VPC-private OSIS ingest endpoint. This is the scenario that + // actually proves end-to-end VPC data flow; verification queries the domain + // through the reachable-from-outside Application (OpenSearch UI) endpoint. + verifyDataFlow: true, + }, + { + name: 'managed-vpc-nodemo', + description: 'Managed domain in a VPC (private endpoints), no demo — create/destroy only', + backend: 'managed', + vpc: true, + demo: false, + enabledByDefault: false, + // A VPC-attached OSIS pipeline exposes a VPC-PRIVATE ingest endpoint (resolves + // to RFC-1918 addresses reachable only inside the VPC). The synthetic OTLP push + // originates from the developer's host, which has no route into the VPC, so it + // can never reach that endpoint — data-flow verification is not applicable here. + // This scenario therefore exercises create + the full teardown path only (the + // CLI still proves the VPC-attached pipeline reaches ACTIVE before exiting 0). + // Use managed-vpc-demo to verify actual VPC data flow (demo runs in-VPC). + verifyDataFlow: false, + }, +]; + +/** + * Look up a scenario by name. Throws with the list of valid names if unknown. + */ +export function getScenario(name) { + const s = SCENARIOS.find((x) => x.name === name); + if (!s) { + throw new Error(`Unknown scenario '${name}'. Valid: ${SCENARIOS.map((x) => x.name).join(', ')}`); + } + return s; +} + +/** + * Select the scenarios to run. + * - names: explicit list (highest precedence) + * - all: include opt-in scenarios too + * - otherwise: the default-enabled scenarios + */ +export function selectScenarios({ names, all } = {}) { + if (names?.length) return names.map(getScenario); + if (all) return [...SCENARIOS]; + return SCENARIOS.filter((s) => s.enabledByDefault); +} + +/** + * Derive VPC parameters from the environment. VPC scenarios are skipped (not + * failed) when these are absent, so the default matrix stays runnable anywhere. + * Returns { vpcId, subnetIds, securityGroupIds } or null when unset. + */ +export function envToVpc(env = process.env) { + const vpcId = env.E2E_VPC_ID || ''; + const subnetIds = (env.E2E_SUBNET_IDS || '').split(',').map((s) => s.trim()).filter(Boolean); + const securityGroupIds = (env.E2E_SECURITY_GROUP_IDS || '').split(',').map((s) => s.trim()).filter(Boolean); + if (!vpcId || !subnetIds.length || !securityGroupIds.length) return null; + return { vpcId, subnetIds, securityGroupIds }; +} + +/** + * Build a unique, DNS/OSIS-safe pipeline name for a scenario run. + * OSIS caps pipeline names at 28 chars, lowercase-alnum-hyphen, letter-first. + * `suffix` is typically a short timestamp/run id to avoid collisions. + */ +export function buildPipelineName(scenario, suffix) { + // Short, stable tag per scenario so the name stays within 28 chars. + const tagByName = { + 'managed-public-demo': 'mpd', + 'managed-public-nodemo': 'mpn', + 'serverless-public-nodemo': 'spn', + 'managed-vpc-demo': 'mvd', + 'managed-vpc-nodemo': 'mvn', + }; + const tag = tagByName[scenario.name] || scenario.name.replace(/[^a-z0-9]/g, '').slice(0, 6); + const raw = `e2e-${tag}-${suffix}`.toLowerCase(); + const cleaned = raw.replace(/[^a-z0-9-]/g, '').replace(/^-+/, '').slice(0, 28); + // Guarantee it starts with a letter (OSIS requirement). + return /^[a-z]/.test(cleaned) ? cleaned : `e${cleaned}`.slice(0, 28); +} + +/** + * Build the argv (after `node cli-installer.mjs`) for creating a scenario's stack. + * + * @param {object} scenario one of SCENARIOS + * @param {object} opts + * @param {string} opts.pipelineName + * @param {string} opts.region + * @param {object|null} [opts.vpc] { vpcId, subnetIds[], securityGroupIds[] } — required if scenario.vpc + * @returns {string[]} argv + */ +export function buildCreateArgs(scenario, opts) { + const { pipelineName, region, vpc } = opts; + if (!pipelineName) throw new Error('buildCreateArgs: pipelineName is required'); + if (!region) throw new Error('buildCreateArgs: region is required'); + + // Quick mode (the default — no --advanced) is what we want here: it auto-creates + // the whole stack (IAM OSI role, APS workspace, Application) named after the + // pipeline, while still honoring every explicit flag we pass below. Advanced mode + // only creates the resources you name explicitly, so passing --advanced with just + // --os-domain-name would create the domain but skip the OSI role, leaving OSIS to + // reject an empty PipelineRoleArn ("Cross-account pass role is not allowed"). The + // e2e always creates fresh resources, so quick mode is the correct, realistic path. + const args = ['--pipeline-name', pipelineName, '--region', region]; + + if (scenario.backend === 'serverless') { + args.push('--serverless', '--aoss-collection-name', pipelineName); + } else { + args.push('--managed', '--os-domain-name', pipelineName); + } + + if (scenario.vpc) { + if (!vpc?.vpcId || !vpc.subnetIds?.length || !vpc.securityGroupIds?.length) { + throw new Error(`buildCreateArgs: scenario '${scenario.name}' requires vpc {vpcId, subnetIds, securityGroupIds}`); + } + args.push( + '--vpc-id', vpc.vpcId, + '--subnet-ids', vpc.subnetIds.join(','), + '--security-group-ids', vpc.securityGroupIds.join(','), + ); + } + + if (!scenario.demo) args.push('--skip-demo'); + + return args; +} + +/** + * Build the argv for tearing a scenario's stack down. + */ +export function buildDestroyArgs(scenario, opts) { + const { pipelineName, region } = opts; + const args = ['destroy', '--pipeline-name', pipelineName, '--region', region]; + if (scenario.backend === 'serverless') args.push('--aoss-collection-name', pipelineName); + return args; +} diff --git a/aws/cli-installer/test/e2e/verify.mjs b/aws/cli-installer/test/e2e/verify.mjs new file mode 100644 index 00000000..e83f13e9 --- /dev/null +++ b/aws/cli-installer/test/e2e/verify.mjs @@ -0,0 +1,238 @@ +/** + * E2E data-flow verification. + * + * After a stack is created and telemetry is pushed (either by the EC2 OTel-demo + * or a synthetic OTLP push), these helpers confirm that documents actually + * landed in the expected OpenSearch indices by querying through the managed + * OpenSearch UI (Application) endpoint — the same SigV4 proxy path the installer + * uses, which works even for VPC-private domains from outside the VPC. + * + * Pure helpers (index expectations, count evaluation, OTLP payload building) are + * separated from the network calls so they can be unit-tested without AWS. + */ +import { createHash } from 'node:crypto'; +import { SignatureV4 } from '@aws-sdk/signature-v4'; +import { Sha256 } from '@aws-crypto/sha256-js'; +import { HttpRequest } from '@smithy/protocol-http'; +import { defaultProvider } from '@aws-sdk/credential-provider-node'; + +// ── Pure helpers ────────────────────────────────────────────────────────────── + +/** + * The index patterns the stack ingests into, matching render.mjs sink config and + * the index patterns created in opensearch-ui-init.mjs. Each is checked for a + * positive document count during verification. + * + * `signal` labels how the index is populated: 'logs' and 'traces' map directly + * to the OTLP signal that fills them; 'service-map' is derived asynchronously by + * the pipeline from spans, so it gets its own key and can be required separately. + */ +export const EXPECTED_INDICES = [ + { pattern: 'logs-otel-v1', signal: 'logs' }, + { pattern: 'otel-v1-apm-span', signal: 'traces' }, + { pattern: 'otel-v2-apm-service-map', signal: 'service-map' }, +]; + +/** + * Given a map of index-pattern -> observed count, decide pass/fail per index and + * overall. `require` narrows which signals must have data (e.g. a synthetic push + * that only sends logs+traces should not require the service-map index, which is + * derived asynchronously by the pipeline from spans). + * + * @returns {{ ok: boolean, results: Array<{pattern, signal, count, ok}> }} + */ +export function evaluateCounts(counts, { requireSignals } = {}) { + const results = EXPECTED_INDICES.map(({ pattern, signal }) => { + const count = Number(counts[pattern] ?? 0); + const required = requireSignals ? requireSignals.includes(signal) : true; + return { pattern, signal, count, required, ok: required ? count > 0 : true }; + }); + return { ok: results.every((r) => r.ok), results }; +} + +/** + * Parse a document count out of an OpenSearch `_count` response body, which may + * arrive as a parsed object or a raw JSON string (the console proxy returns text). + * Returns 0 when the shape is unexpected rather than throwing. + */ +export function parseCount(body) { + let obj = body; + if (typeof body === 'string') { + try { obj = JSON.parse(body); } catch { return 0; } + } + const n = obj?.count; + return Number.isFinite(n) ? n : 0; +} + +/** + * Build a minimal OTLP/HTTP JSON payload for a single signal, used by the + * no-demo scenarios to push synthetic telemetry directly at the OSIS ingest + * endpoint. `nowNanos` is injected (not read from the clock) so this stays pure + * and unit-testable. + * + * Supports 'logs' and 'traces'. The trace payload carries service.name so the + * pipeline can derive service-map edges. + */ +export function buildOtlpPayload(signal, { serviceName = 'e2e-synthetic', nowNanos, traceId, spanId } = {}) { + const ts = String(nowNanos ?? '0'); + const resource = { + attributes: [{ key: 'service.name', value: { stringValue: serviceName } }], + }; + + if (signal === 'logs') { + return { + resourceLogs: [{ + resource, + scopeLogs: [{ + scope: { name: 'e2e-verifier' }, + logRecords: [{ + timeUnixNano: ts, + observedTimeUnixNano: ts, + severityNumber: 9, + severityText: 'INFO', + body: { stringValue: 'e2e synthetic log' }, + traceId: traceId || '', + spanId: spanId || '', + }], + }], + }], + }; + } + + if (signal === 'traces') { + return { + resourceSpans: [{ + resource, + scopeSpans: [{ + scope: { name: 'e2e-verifier' }, + spans: [{ + traceId: traceId || '00000000000000000000000000000001', + spanId: spanId || '0000000000000001', + name: 'e2e-synthetic-span', + kind: 2, // SERVER + startTimeUnixNano: ts, + endTimeUnixNano: String(BigInt(ts || '0') + 1_000_000n), + attributes: [{ key: 'e2e', value: { boolValue: true } }], + status: { code: 1 }, + }], + }], + }], + }; + } + + throw new Error(`buildOtlpPayload: unsupported signal '${signal}'`); +} + +/** + * Build the OSIS OTLP ingest URL for a signal. + * OSIS exposes per-pipeline paths: https:////v1/{logs,traces,metrics} + */ +export function otlpUrl(ingestEndpoint, pipelineName, signal) { + const host = ingestEndpoint.replace(/^https?:\/\//, ''); + return `https://${host}/${pipelineName}/v1/${signal}`; +} + +// ── SigV4 request helper (shared by ingest + query) ───────────────────────────── + +async function sigv4Fetch({ method, url, body, service, region }) { + const isBodyless = method === 'GET' || method === 'DELETE'; + const bodyBytes = !isBodyless && body != null + ? (typeof body === 'string' ? body : JSON.stringify(body)) + : ''; + const parsed = new URL(url); + const query = {}; + parsed.searchParams.forEach((v, k) => { query[k] = v; }); + + const request = new HttpRequest({ + method, + protocol: parsed.protocol, + hostname: parsed.hostname, + port: parsed.port ? Number(parsed.port) : undefined, + path: parsed.pathname, + query, + headers: { + host: parsed.hostname, + 'Content-Type': 'application/json', + 'osd-xsrf': 'osd-fetch', + 'x-amz-content-sha256': createHash('sha256').update(bodyBytes).digest('hex'), + }, + body: bodyBytes || undefined, + }); + + const signer = new SignatureV4({ credentials: defaultProvider(), region, service, sha256: Sha256 }); + const signed = await signer.sign(request); + const resp = await fetch(url, { method, headers: signed.headers, body: isBodyless ? undefined : bodyBytes }); + const text = await resp.text(); + let data; try { data = JSON.parse(text); } catch { data = text; } + return { status: resp.status, data }; +} + +/** + * Push a synthetic OTLP payload to the OSIS ingest endpoint (service `osis`). + * Used by no-demo scenarios. Returns { status }. + */ +export async function pushOtlp({ ingestEndpoint, pipelineName, region, signal, payload }) { + const url = otlpUrl(ingestEndpoint, pipelineName, signal); + const { status } = await sigv4Fetch({ + method: 'POST', url, body: payload, service: 'osis', region, + }); + return { status }; +} + +// ── Query the domain through the OpenSearch UI (Application) endpoint ─────────── + +/** + * Discover the auto-created data-source id behind the OpenSearch UI, needed to + * proxy queries to the underlying domain/collection. + */ +export async function findDataSourceId({ appEndpoint, region }) { + const { status, data } = await sigv4Fetch({ + method: 'GET', + url: `${appEndpoint}/api/saved_objects/_find?type=data-source&per_page=10`, + service: 'opensearch', region, + }); + if (status !== 200) return null; + return data?.saved_objects?.[0]?.id || null; +} + +/** + * Get the document count for an index pattern by proxying an OpenSearch + * `_count` request through the UI console proxy. This is the reachable path for + * VPC-private domains (the UI proxies over the AWS-internal network). + */ +export async function countDocs({ appEndpoint, region, dataSourceId, indexPattern }) { + const path = encodeURIComponent(`/${indexPattern}*/_count`); + const url = `${appEndpoint}/api/console/proxy?path=${path}&method=GET` + + (dataSourceId ? `&dataSourceId=${dataSourceId}` : ''); + const { status, data } = await sigv4Fetch({ method: 'POST', url, service: 'opensearch', region }); + if (status !== 200) return { status, count: 0 }; + return { status, count: parseCount(data) }; +} + +/** + * Poll all expected indices until every required signal has a positive count or + * the deadline passes. Returns the final evaluateCounts() result plus timing. + * + * Data can take several minutes to land (EC2 demo bootstrap, or OSIS ingest + * warmup for a freshly-created VPC pipeline), so this polls patiently. + */ +export async function waitForData({ + appEndpoint, region, dataSourceId, requireSignals, + timeoutMs = 20 * 60_000, intervalMs = 30_000, log = () => {}, +}) { + const start = Date.now(); + let last; + while (Date.now() - start < timeoutMs) { + const counts = {}; + for (const { pattern } of EXPECTED_INDICES) { + const { count } = await countDocs({ appEndpoint, region, dataSourceId, indexPattern: pattern }); + counts[pattern] = count; + } + last = evaluateCounts(counts, { requireSignals }); + const summary = last.results.map((r) => `${r.pattern}=${r.count}${r.required ? '' : '(opt)'}`).join(' '); + log(`counts: ${summary} — ${last.ok ? 'OK' : 'waiting'}`); + if (last.ok) return { ...last, elapsedMs: Date.now() - start }; + await new Promise((r) => setTimeout(r, intervalMs)); + } + return { ...(last || { ok: false, results: [] }), elapsedMs: Date.now() - start, timedOut: true }; +} diff --git a/aws/cli-installer/test/unit.test.mjs b/aws/cli-installer/test/unit.test.mjs index 6684e27b..e76df85c 100644 --- a/aws/cli-installer/test/unit.test.mjs +++ b/aws/cli-installer/test/unit.test.mjs @@ -452,6 +452,10 @@ import { fgacPrincipals, validateVpcTopology, pipelineRoleArnError, + permissionsAllowPort, + permissionsAllowInternet, + analyzeSecurityGroupRules, + checkSecurityGroupRules, _withRetry, _isRoleNotPropagatedError, _isTransientHttpError, @@ -603,6 +607,163 @@ describe('validateVpcTopology', () => { }); }); +// ── Security-group rule analysis (best-effort data-path check) ──────────────── + +describe('permissionsAllowPort', () => { + const opts = { selfIds: ['sg-1'], vpcCidrs: ['172.30.0.0/16'] }; + + it('matches a tcp/443 range from a self-referencing group', () => { + const perms = [{ IpProtocol: 'tcp', FromPort: 443, ToPort: 443, UserIdGroupPairs: [{ GroupId: 'sg-1' }], IpRanges: [] }]; + assert.equal(permissionsAllowPort(perms, 443, opts), true); + }); + + it('matches all-protocols (-1) regardless of port fields', () => { + const perms = [{ IpProtocol: '-1', UserIdGroupPairs: [{ GroupId: 'sg-1' }], IpRanges: [] }]; + assert.equal(permissionsAllowPort(perms, 443, opts), true); + }); + + it('matches a 0.0.0.0/0 CIDR range covering the port', () => { + const perms = [{ IpProtocol: 'tcp', FromPort: 0, ToPort: 65535, UserIdGroupPairs: [], IpRanges: [{ CidrIp: '0.0.0.0/0' }] }]; + assert.equal(permissionsAllowPort(perms, 443, opts), true); + }); + + it('matches the VPC CIDR', () => { + const perms = [{ IpProtocol: 'tcp', FromPort: 443, ToPort: 443, IpRanges: [{ CidrIp: '172.30.0.0/16' }] }]; + assert.equal(permissionsAllowPort(perms, 443, opts), true); + }); + + it('does not match a foreign group or a narrower CIDR', () => { + const perms = [{ IpProtocol: 'tcp', FromPort: 443, ToPort: 443, UserIdGroupPairs: [{ GroupId: 'sg-other' }], IpRanges: [{ CidrIp: '10.0.0.5/32' }] }]; + assert.equal(permissionsAllowPort(perms, 443, opts), false); + }); + + it('does not match a rule whose port range excludes 443', () => { + const perms = [{ IpProtocol: 'tcp', FromPort: 80, ToPort: 80, IpRanges: [{ CidrIp: '0.0.0.0/0' }] }]; + assert.equal(permissionsAllowPort(perms, 443, opts), false); + }); +}); + +describe('permissionsAllowInternet', () => { + it('matches only a 0.0.0.0/0 range, not a self-ref or VPC CIDR', () => { + assert.equal(permissionsAllowInternet([{ IpProtocol: 'tcp', FromPort: 443, ToPort: 443, IpRanges: [{ CidrIp: '0.0.0.0/0' }] }], 443), true); + assert.equal(permissionsAllowInternet([{ IpProtocol: '-1', IpRanges: [{ CidrIp: '0.0.0.0/0' }] }], 443), true); + assert.equal(permissionsAllowInternet([{ IpProtocol: 'tcp', FromPort: 443, ToPort: 443, UserIdGroupPairs: [{ GroupId: 'sg-1' }] }], 443), false); + assert.equal(permissionsAllowInternet([{ IpProtocol: 'tcp', FromPort: 443, ToPort: 443, IpRanges: [{ CidrIp: '172.30.0.0/16' }] }], 443), false); + }); +}); + +describe('analyzeSecurityGroupRules', () => { + const vpcCidrs = ['172.30.0.0/16']; + + it('no warnings for a demo when egress allows internet and 443 is open in-VPC', () => { + const groups = [{ + GroupId: 'sg-1', + IpPermissions: [{ IpProtocol: 'tcp', FromPort: 443, ToPort: 443, UserIdGroupPairs: [{ GroupId: 'sg-1' }] }], + IpPermissionsEgress: [{ IpProtocol: '-1', IpRanges: [{ CidrIp: '0.0.0.0/0' }] }], + }]; + assert.deepEqual(analyzeSecurityGroupRules(groups, { groupIds: ['sg-1'], vpcCidrs, demo: true }), []); + }); + + it('warns about both intra-VPC and internet egress when egress is stripped (demo)', () => { + // The SA's failure: default allow-all egress removed, so the demo can't reach + // OSIS in-VPC OR bootstrap over the internet. + const groups = [{ + GroupId: 'sg-1', + IpPermissions: [{ IpProtocol: 'tcp', FromPort: 443, ToPort: 443, UserIdGroupPairs: [{ GroupId: 'sg-1' }] }], + IpPermissionsEgress: [], + }]; + const w = analyzeSecurityGroupRules(groups, { groupIds: ['sg-1'], vpcCidrs, demo: true }); + assert.equal(w.length, 2); + assert.ok(w.some((x) => /within the VPC/i.test(x))); + assert.ok(w.some((x) => /0\.0\.0\.0\/0/.test(x) && /internet/i.test(x))); + }); + + it('flags missing internet egress even when in-VPC egress is present (demo)', () => { + // Egress scoped to the VPC CIDR: telemetry to OSIS works, but the demo cannot + // pull images. This is the case the earlier version wrongly passed. + const groups = [{ + GroupId: 'sg-1', + IpPermissions: [{ IpProtocol: 'tcp', FromPort: 443, ToPort: 443, UserIdGroupPairs: [{ GroupId: 'sg-1' }] }], + IpPermissionsEgress: [{ IpProtocol: 'tcp', FromPort: 443, ToPort: 443, IpRanges: [{ CidrIp: '172.30.0.0/16' }] }], + }]; + const w = analyzeSecurityGroupRules(groups, { groupIds: ['sg-1'], vpcCidrs, demo: true }); + assert.equal(w.length, 1); + assert.ok(/internet/i.test(w[0]) && /0\.0\.0\.0\/0/.test(w[0])); + }); + + it('does NOT require internet egress when no demo launches (--skip-demo)', () => { + // No demo: only OSIS→domain in-VPC egress is needed. VPC-scoped egress suffices. + const groups = [{ + GroupId: 'sg-1', + IpPermissions: [{ IpProtocol: 'tcp', FromPort: 443, ToPort: 443, UserIdGroupPairs: [{ GroupId: 'sg-1' }] }], + IpPermissionsEgress: [{ IpProtocol: 'tcp', FromPort: 443, ToPort: 443, IpRanges: [{ CidrIp: '172.30.0.0/16' }] }], + }]; + assert.deepEqual(analyzeSecurityGroupRules(groups, { groupIds: ['sg-1'], vpcCidrs, demo: false }), []); + }); + + it('warns about missing intra-VPC ingress on 443', () => { + const groups = [{ + GroupId: 'sg-1', + IpPermissions: [{ IpProtocol: 'tcp', FromPort: 22, ToPort: 22, IpRanges: [{ CidrIp: '0.0.0.0/0' }] }], + IpPermissionsEgress: [{ IpProtocol: '-1', IpRanges: [{ CidrIp: '0.0.0.0/0' }] }], + }]; + const w = analyzeSecurityGroupRules(groups, { groupIds: ['sg-1'], vpcCidrs, demo: true }); + assert.equal(w.length, 1); + assert.ok(/ingress.*443/i.test(w[0])); + }); + + it('is satisfied when the union across multiple groups covers ingress and egress', () => { + const groups = [ + { GroupId: 'sg-in', IpPermissions: [{ IpProtocol: 'tcp', FromPort: 443, ToPort: 443, UserIdGroupPairs: [{ GroupId: 'sg-in' }] }], IpPermissionsEgress: [] }, + { GroupId: 'sg-out', IpPermissions: [], IpPermissionsEgress: [{ IpProtocol: 'tcp', FromPort: 443, ToPort: 443, IpRanges: [{ CidrIp: '0.0.0.0/0' }] }] }, + ]; + assert.deepEqual(analyzeSecurityGroupRules(groups, { groupIds: ['sg-in', 'sg-out'], vpcCidrs, demo: true }), []); + }); +}); + +describe('checkSecurityGroupRules', () => { + const cfg = { region: 'us-east-1', vpcId: 'vpc-aaa', securityGroupIds: ['sg-1'] }; + + it('returns [] when no VPC is configured', async () => { + assert.deepEqual(await checkSecurityGroupRules({ region: 'us-east-1', vpcId: '', securityGroupIds: [] }), []); + }); + + it('degrades to a single manual-verify warning when describe is unauthorized', async () => { + const w = await checkSecurityGroupRules(cfg, { + describeSecurityGroups: async () => { const e = new Error('UnauthorizedOperation'); e.name = 'UnauthorizedOperation'; throw e; }, + describeVpcs: async () => [{ VpcId: 'vpc-aaa', CidrBlock: '172.30.0.0/16' }], + }); + assert.equal(w.length, 1); + assert.ok(/lacks ec2:DescribeSecurityGroups/.test(w[0])); + }); + + it('flags a stripped egress rule end-to-end (demo default)', async () => { + const w = await checkSecurityGroupRules(cfg, { + describeSecurityGroups: async () => [{ + GroupId: 'sg-1', + IpPermissions: [{ IpProtocol: 'tcp', FromPort: 443, ToPort: 443, UserIdGroupPairs: [{ GroupId: 'sg-1' }] }], + IpPermissionsEgress: [], + }], + describeVpcs: async () => [{ VpcId: 'vpc-aaa', CidrBlockAssociationSet: [{ CidrBlock: '172.30.0.0/16' }] }], + }); + // demo defaults on (skipDemo unset): both in-VPC and internet egress missing. + assert.equal(w.length, 2); + assert.ok(w.some((x) => /internet/i.test(x))); + }); + + it('does not flag internet egress when skipDemo is set', async () => { + const w = await checkSecurityGroupRules({ ...cfg, skipDemo: true }, { + describeSecurityGroups: async () => [{ + GroupId: 'sg-1', + IpPermissions: [{ IpProtocol: 'tcp', FromPort: 443, ToPort: 443, UserIdGroupPairs: [{ GroupId: 'sg-1' }] }], + IpPermissionsEgress: [{ IpProtocol: 'tcp', FromPort: 443, ToPort: 443, IpRanges: [{ CidrIp: '172.30.0.0/16' }] }], + }], + describeVpcs: async () => [{ VpcId: 'vpc-aaa', CidrBlockAssociationSet: [{ CidrBlock: '172.30.0.0/16' }] }], + }); + assert.deepEqual(w, []); + }); +}); + // ── Creation ordering / race-condition guards ──────────────────────────────── describe('withRetry', () => {