diff --git a/aws/cli-installer/README.md b/aws/cli-installer/README.md index 1240f628..3830dc7e 100644 --- a/aws/cli-installer/README.md +++ b/aws/cli-installer/README.md @@ -70,11 +70,49 @@ node bin/launch-demo.mjs \ --region us-east-1 ``` +**Deploy into a VPC** (private endpoints): +```bash +node bin/cli-installer.mjs --advanced --managed \ + --pipeline-name obs-stack- \ + --region us-east-1 \ + --os-domain-name obs-stack- \ + --vpc-id vpc-xxxxxxxx \ + --subnet-ids subnet-aaaa,subnet-bbbb \ + --security-group-ids sg-xxxxxxxx +``` + +The domain, OSIS ingestion pipeline, and EC2 demo instance are all placed in the +selected VPC. Use this when your telemetry sources (EKS, ECS, EC2) run inside a VPC +and you want the OpenSearch stack in the same network boundary. + +- Pass **1-3 subnets** (each in a distinct AZ). With more than one subnet the domain + is created zone-aware; the pipeline attaches to the first two. +- Security groups must allow the intra-VPC traffic between the pipeline, domain, and + demo instance (a single group that allows traffic from itself works well). +- Omitting `--vpc-id` keeps the default behavior: public endpoints. + +> **You can run this from anywhere** — you do not need to be inside the VPC. A +> VPC-private domain's endpoint isn't reachable from your machine, but the managed +> OpenSearch UI (Application) endpoint is public and proxies to the domain over the +> AWS-internal network. For VPC domains the installer: +> - sets your IAM principal as the domain master, +> - authorizes the OpenSearch UI service (`application.opensearchservice.amazonaws.com`) +> to reach the domain through its VPC endpoint, and +> - performs all FGAC role mapping and UI setup through that reachable Application +> endpoint. +> +> So no bastion or VPN is required. (If you deploy a VPC domain by other means, run +> `aws opensearch authorize-vpc-endpoint-access --domain-name --service +> application.opensearchservice.amazonaws.com` yourself, or the UI cannot connect.) + **Interactive mode** (TUI wizard): ```bash node bin/cli-installer.mjs ``` +Advanced mode adds a **Network topology** step where you pick public endpoints or a +VPC, then select subnets and security groups from your account. + ## Destroy ```bash @@ -96,6 +134,7 @@ Deletes: EC2 instance, OpenSearch Application, Connected Data Source, OSIS pipel - **Index pattern fields need manual refresh** — After data starts flowing, go to Management → Index Patterns → select pattern → click 🔄 to pick up new fields. - **Demo data takes 10-15 minutes** — The EC2 instance needs time to bootstrap Docker, pull images, and start sending telemetry. - **Idempotent but not updateable** — Running twice safely no-ops, but won't update existing resources with new config. +- **VPC mode maps FGAC through the OpenSearch UI** — for VPC-private domains your IAM principal is the domain master and role mapping runs through the managed OpenSearch UI (Application) endpoint, so the CLI does not need to be inside the VPC. VPC options apply only to newly created domains, not reused endpoints. ## Development diff --git a/aws/cli-installer/package-lock.json b/aws/cli-installer/package-lock.json index ca530c75..4f1696f7 100644 --- a/aws/cli-installer/package-lock.json +++ b/aws/cli-installer/package-lock.json @@ -1,12 +1,12 @@ { "name": "@opensearch-project/observability-stack", - "version": "0.1.2", + "version": "0.2.0", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "@opensearch-project/observability-stack", - "version": "0.1.2", + "version": "0.2.0", "license": "Apache-2.0", "dependencies": { "@aws-crypto/sha256-js": "^5.2.0", @@ -21,6 +21,7 @@ "@aws-sdk/client-secrets-manager": "^3.1022.0", "@aws-sdk/client-ssm": "^3.1021.0", "@aws-sdk/client-sts": "^3.750.0", + "@aws-sdk/credential-provider-node": "^3.750.0", "@aws-sdk/signature-v4": "^3.370.0", "@inquirer/prompts": "^7.0.0", "@smithy/protocol-http": "^5.3.12", diff --git a/aws/cli-installer/package.json b/aws/cli-installer/package.json index 79a8f96e..64d0952a 100644 --- a/aws/cli-installer/package.json +++ b/aws/cli-installer/package.json @@ -30,6 +30,7 @@ "@aws-sdk/client-secrets-manager": "^3.1022.0", "@aws-sdk/client-ssm": "^3.1021.0", "@aws-sdk/client-sts": "^3.750.0", + "@aws-sdk/credential-provider-node": "^3.750.0", "@aws-sdk/signature-v4": "^3.370.0", "@inquirer/prompts": "^7.0.0", "@smithy/protocol-http": "^5.3.12", diff --git a/aws/cli-installer/src/aws.mjs b/aws/cli-installer/src/aws.mjs index 8a367446..ece81b3b 100644 --- a/aws/cli-installer/src/aws.mjs +++ b/aws/cli-installer/src/aws.mjs @@ -12,6 +12,7 @@ import { GetApplicationCommand, UpdateApplicationCommand, ListApplicationsCommand, + AuthorizeVpcEndpointAccessCommand, } from '@aws-sdk/client-opensearch'; import { IAMClient, @@ -55,8 +56,12 @@ import { createSpinner, createAsciiAnimation, } from './ui.mjs'; +import { SignatureV4 } from '@aws-sdk/signature-v4'; +import { Sha256 } from '@aws-crypto/sha256-js'; +import { HttpRequest } from '@smithy/protocol-http'; +import { defaultProvider } from '@aws-sdk/credential-provider-node'; import chalk from 'chalk'; -import { randomBytes } from 'node:crypto'; +import { randomBytes, createHash } from 'node:crypto'; import { SecretsManagerClient, CreateSecretCommand, @@ -195,8 +200,42 @@ export async function createOpenSearch(cfg) { return createManagedDomain(cfg); } +/** + * Extract the reachable endpoint from a DomainStatus, supporting both public + * (top-level Endpoint) and VPC domains (Endpoints.vpc map). + */ +function domainEndpoint(status) { + if (!status) return ''; + if (status.Endpoint) return status.Endpoint; + return status.Endpoints?.vpc || ''; +} + +// Managed OpenSearch UI reaches a VPC-private domain over the domain's VPC +// endpoint, which the domain owner must authorize for the UI service principal. +// Idempotent: a re-authorize just no-ops. +const OPENSEARCH_UI_SERVICE = 'application.opensearchservice.amazonaws.com'; + +async function authorizeOpenSearchUiVpcAccess(cfg, client) { + try { + await client.send(new AuthorizeVpcEndpointAccessCommand({ + DomainName: cfg.osDomainName, + Service: OPENSEARCH_UI_SERVICE, + })); + printSuccess('Authorized OpenSearch UI to reach the VPC-private domain'); + } catch (err) { + // Already authorized is fine; anything else is a warning, not fatal. + if (/already|Conflict|LimitExceeded/i.test(err.message)) { + printInfo('OpenSearch UI VPC access already authorized'); + } else { + printWarning(`Could not authorize OpenSearch UI VPC access: ${err.message}`); + printInfo(`Authorize it manually: aws opensearch authorize-vpc-endpoint-access --domain-name ${cfg.osDomainName} --service ${OPENSEARCH_UI_SERVICE} --region ${cfg.region}`); + } + } +} + async function createManagedDomain(cfg) { - printStep(`Creating OpenSearch domain '${cfg.osDomainName}'...`); + const inVpc = Boolean(cfg.vpcId); + printStep(`Creating OpenSearch domain '${cfg.osDomainName}'${inVpc ? ' (VPC)' : ''}...`); console.error(); const client = new OpenSearchClient({ region: cfg.region }); @@ -204,13 +243,18 @@ async function createManagedDomain(cfg) { // Check if domain already exists try { const desc = await client.send(new DescribeDomainCommand({ DomainName: cfg.osDomainName })); - const endpoint = desc.DomainStatus?.Endpoint; - if (endpoint) { + const status = desc.DomainStatus || {}; + const endpoint = domainEndpoint(status); + // Only short-circuit if the pre-existing domain is fully active. A domain + // still Processing (e.g. a prior interrupted run) must fall through to the + // wait loop, or downstream security-API calls race an unready cluster. + if (endpoint && !status.Processing && !status.UpgradeProcessing) { cfg.opensearchEndpoint = `https://${endpoint}`; printSuccess(`Domain '${cfg.osDomainName}' already exists: ${cfg.opensearchEndpoint}`); + if (inVpc) await authorizeOpenSearchUiVpcAccess(cfg, client); return; } - printSuccess(`Domain '${cfg.osDomainName}' already exists — waiting for endpoint`); + printSuccess(`Domain '${cfg.osDomainName}' already exists — waiting for it to become active`); } catch (err) { if (err.name !== 'ResourceNotFoundException') throw err; @@ -227,36 +271,80 @@ async function createManagedDomain(cfg) { }], }); + // Build cluster config; enable zone awareness when spanning multiple VPC subnets (AZs). + const clusterConfig = { + InstanceType: cfg.osInstanceType, + InstanceCount: cfg.osInstanceCount, + }; + if (inVpc && cfg.subnetIds.length > 1) { + // Zone awareness supports 2 or 3 AZs, and the data node count must be a + // multiple of the AZ count. Round the requested count up to the next multiple. + const azCount = Math.min(cfg.subnetIds.length, 3); + const nodeCount = Math.max(azCount, Math.ceil(cfg.osInstanceCount / azCount) * azCount); + if (nodeCount !== cfg.osInstanceCount) { + printInfo(`Zone-aware domain across ${azCount} AZs requires the data node count to be a multiple of ${azCount}; using ${nodeCount} data nodes.`); + cfg.osInstanceCount = nodeCount; + } + clusterConfig.InstanceCount = nodeCount; + clusterConfig.ZoneAwarenessEnabled = true; + clusterConfig.ZoneAwarenessConfig = { AvailabilityZoneCount: azCount }; + } + + // Master user: for VPC-private domains the domain's Security API is only + // reachable from inside the VPC, so an internal (username/password) master + // can't be used to bootstrap role mappings from outside. Instead, make the + // caller's IAM principal the master — it can then drive role mapping through + // the reachable managed OpenSearch UI (which proxies to the domain over the + // AWS-internal path) via SigV4, no in-VPC network access required. Public + // domains keep the internal-database master (username/password). + const iamMaster = inVpc && Boolean(cfg.callerPrincipal?.arn); + const advancedSecurity = iamMaster + ? { + Enabled: true, + InternalUserDatabaseEnabled: false, + MasterUserOptions: { MasterUserARN: cfg.callerPrincipal.arn }, + } + : { + Enabled: true, + InternalUserDatabaseEnabled: true, + MasterUserOptions: { + MasterUserName: cfg.opensearchUser || 'admin', + MasterUserPassword: (cfg._masterPassword = generatePassword()), + }, + }; + try { - cfg._masterPassword = generatePassword(); await client.send(new CreateDomainCommand({ DomainName: cfg.osDomainName, EngineVersion: cfg.osEngineVersion, - ClusterConfig: { - InstanceType: cfg.osInstanceType, - InstanceCount: cfg.osInstanceCount, - }, + ClusterConfig: clusterConfig, EBSOptions: { EBSEnabled: true, VolumeType: 'gp3', VolumeSize: cfg.osVolumeSize, }, + // VPCOptions places the domain inside the selected subnets/SGs (private endpoint). + // Omitting it leaves the domain on a public endpoint (default behavior). + ...(inVpc ? { + VPCOptions: { + SubnetIds: cfg.subnetIds, + SecurityGroupIds: cfg.securityGroupIds, + }, + } : {}), NodeToNodeEncryptionOptions: { Enabled: true }, EncryptionAtRestOptions: { Enabled: true }, DomainEndpointOptions: { EnforceHTTPS: true }, - AdvancedSecurityOptions: { - Enabled: true, - InternalUserDatabaseEnabled: true, - MasterUserOptions: { - MasterUserName: cfg.opensearchUser || 'admin', - MasterUserPassword: cfg._masterPassword, - }, - }, + AdvancedSecurityOptions: advancedSecurity, AccessPolicies: accessPolicy, TagList: stackTags(cfg.pipelineName), })); - printSuccess('Domain creation initiated — waiting for endpoint'); - await storeMasterPassword(cfg.region, cfg.pipelineName, cfg._masterPassword); + printSuccess(`Domain creation initiated${inVpc ? ` in VPC ${cfg.vpcId}` : ''} — waiting for endpoint`); + if (iamMaster) { + cfg.iamMasterArn = cfg.callerPrincipal.arn; + printInfo(`Master user: IAM principal ${cfg.iamMasterArn} (role mapping via OpenSearch UI)`); + } else { + await storeMasterPassword(cfg.region, cfg.pipelineName, cfg._masterPassword); + } } catch (createErr) { printError('Failed to create OpenSearch domain'); console.error(); @@ -284,7 +372,7 @@ async function createManagedDomain(cfg) { try { const desc = await client.send(new DescribeDomainCommand({ DomainName: cfg.osDomainName })); const ds = desc.DomainStatus || {}; - const endpoint = ds.Endpoint; + const endpoint = domainEndpoint(ds); // Feed real stage progress into the owl animation try { @@ -294,10 +382,21 @@ async function createManagedDomain(cfg) { anim.setDomainStatus(current?.Description || current?.Name || 'Initializing...'); } catch { /* change progress may not be available yet */ } - if (endpoint) { + // Gate on the endpoint being present AND the domain no longer processing. + // The endpoint URL is published while the cluster is still initializing, so + // returning on endpoint alone races the immediately-following security-API + // calls (FGAC mapping / UI→domain connection), which then hit a cluster that + // is not yet serving. Waiting for Processing to clear removes that race. + const active = !ds.Processing && !ds.UpgradeProcessing; + if (endpoint && active) { cfg.opensearchEndpoint = `https://${endpoint}`; anim.stop(); spinner.succeed(`Domain ready: ${cfg.opensearchEndpoint} (${fmtElapsed(Math.round((Date.now() - start) / 1000))})`); + // For VPC-private domains, authorize the managed OpenSearch UI service to + // reach the domain through its VPC endpoint. Without this the UI cannot + // connect to the domain ("No living connections"), so FGAC mapping and UI + // setup — which we route through the UI — would fail. + if (inVpc) await authorizeOpenSearchUiVpcAccess(cfg, client); return; } } catch { /* keep polling */ } @@ -492,10 +591,40 @@ export async function createAossDataAccessPolicy(cfg) { // ── FGAC role mapping for managed domains ──────────────────────────────── +// Roles to map for full OpenSearch UI + PPL access. +const FGAC_ROLES = ['all_access', 'security_manager']; + +/** + * Backend roles and users to add to the domain's FGAC role mappings: the OSI + * pipeline role (so ingestion can write) plus the caller's principal (so the + * caller can use the OpenSearch UI). Returns { backendRoles, users }. + */ +export function fgacPrincipals(cfg) { + const backendRoles = [cfg.iamRoleArn]; + const users = []; + const p = cfg.callerPrincipal; + if (p && p.arn !== cfg.iamRoleArn) { + if (p.type === 'role') backendRoles.push(p.arn); + else users.push(p.arn); + } + return { backendRoles, users }; +} + export async function mapOsiRoleInDomain(cfg) { if (cfg.opensearchType === 'serverless') return; if (!cfg.opensearchEndpoint || !cfg.iamRoleArn) return; + // VPC-private domains: the domain's Security API is only reachable from inside + // the VPC, so we can't map roles by calling the domain directly from here. + // Instead the caller is the IAM master, and role mapping is done through the + // reachable managed OpenSearch UI once the Application exists (see + // mapOsiRoleViaOpenSearchUI, called from executePipeline after the app is up). + if (cfg.vpcId) { + cfg.deferFgacToUi = true; + printInfo('VPC domain — FGAC role mapping will run through the OpenSearch UI after the Application is created.'); + return; + } + printStep('Mapping roles in OpenSearch FGAC...'); // Retrieve master password — from flag (reuse) or Secrets Manager (created by CLI) @@ -513,26 +642,16 @@ export async function mapOsiRoleInDomain(cfg) { const url = `${cfg.opensearchEndpoint}/_plugins/_security/api/rolesmapping`; const auth = Buffer.from(`${cfg.opensearchUser || 'admin'}:${masterPass}`).toString('base64'); - // Map both the OSI pipeline role and the caller's principal (for OpenSearch UI access) - const callerPrincipal = cfg.callerPrincipal; // { arn, type: 'role'|'user' } - const newBackendRoles = [cfg.iamRoleArn]; - const newUsers = []; - if (callerPrincipal && callerPrincipal.arn !== cfg.iamRoleArn) { - if (callerPrincipal.type === 'role') { - newBackendRoles.push(callerPrincipal.arn); - } else { - newUsers.push(callerPrincipal.arn); - } - } - - // Map to both all_access and security_manager for full permissions (including PPL) - const rolesToMap = ['all_access', 'security_manager']; - - try { - const headers = { 'Content-Type': 'application/json', 'Authorization': `Basic ${auth}` }; + const { backendRoles: newBackendRoles, users: newUsers } = fgacPrincipals(cfg); + const headers = { 'Content-Type': 'application/json', 'Authorization': `Basic ${auth}` }; - for (const role of rolesToMap) { - const roleUrl = `${url}/${role}`; + // Map one role, retrying transient failures. The security plugin can briefly + // return 5xx/connection errors right after the cluster becomes active, and a + // silent miss here leaves the OSI role unmapped — the pipeline then goes ACTIVE + // but can't write. So retry, and treat a persistent failure as fatal. + async function mapRole(role) { + const roleUrl = `${url}/${role}`; + await withRetry(async () => { const getResp = await fetch(roleUrl, { headers }); let existingBackendRoles = []; let existingUsers = []; @@ -540,30 +659,160 @@ export async function mapOsiRoleInDomain(cfg) { const data = await getResp.json(); existingBackendRoles = data?.[role]?.backend_roles || []; existingUsers = data?.[role]?.users || []; + } else if (getResp.status >= 500) { + throw new Error(`security API GET ${role} returned ${getResp.status} (cluster warming up)`); } const mergedBackendRoles = [...new Set([...existingBackendRoles, ...newBackendRoles])]; const mergedUsers = [...new Set([...existingUsers, ...newUsers])]; const ops = [{ op: 'add', path: '/backend_roles', value: mergedBackendRoles }]; - if (newUsers.length) { - ops.push({ op: 'add', path: '/users', value: mergedUsers }); - } - - const resp = await fetch(roleUrl, { - method: 'PATCH', - headers, - body: JSON.stringify(ops), - }); + if (newUsers.length) ops.push({ op: 'add', path: '/users', value: mergedUsers }); + const resp = await fetch(roleUrl, { method: 'PATCH', headers, body: JSON.stringify(ops) }); if (!resp.ok) { const body = await resp.text(); - printWarning(`FGAC mapping for ${role} returned ${resp.status}: ${body}`); + // 5xx and 401/403 right after provisioning are transient; retry. A stable + // 4xx (e.g. malformed) would exhaust retries and surface below. + throw new Error(`security API PATCH ${role} returned ${resp.status}: ${body}`); } - } + }, { + shouldRetry: (e) => isTransientHttpError(e) || /returned (401|403|5\d\d)/.test(e.message), + onRetry: (e, i) => printInfo(`FGAC mapping for ${role} not ready yet (attempt ${i + 1}) — retrying`), + }); + } + + try { + for (const role of FGAC_ROLES) await mapRole(role); printSuccess('Roles mapped to all_access and security_manager in OpenSearch'); } catch (err) { - printWarning(`Could not map roles in FGAC: ${err.message}`); - printInfo('You may need to manually map the IAM role in OpenSearch UI → Security → Roles'); + printError(`Could not map the OSI role in OpenSearch FGAC: ${err.message}`); + printInfo('The pipeline cannot write to OpenSearch until this role is mapped.'); + printInfo('Map it manually in OpenSearch UI → Security → Roles, or re-run the installer.'); + throw new Error('FGAC role mapping failed — pipeline would not be able to write to OpenSearch'); + } +} + +// ── SigV4 request against the managed OpenSearch UI Application endpoint ────── +// The managed UI proxies the domain's Security API over the AWS-internal path, +// so this reaches a VPC-private domain from anywhere the app endpoint resolves. + +async function osuiSecurityRequest(method, url, body) { + const isGet = method === 'GET' || method === 'DELETE'; + const bodyBytes = (!isGet && body) ? JSON.stringify(body) : ''; + const bodyHash = createHash('sha256').update(bodyBytes).digest('hex'); + const parsed = new URL(url); + const query = {}; + parsed.searchParams.forEach((v, k) => { query[k] = v; }); + + const request = new HttpRequest({ + method, + protocol: parsed.protocol, + hostname: parsed.hostname, + port: parsed.port ? Number(parsed.port) : undefined, + path: parsed.pathname, + query, + headers: { + host: parsed.hostname, + 'Content-Type': 'application/json', + 'osd-xsrf': 'osd-fetch', + 'x-amz-content-sha256': bodyHash, + }, + body: bodyBytes || undefined, + }); + + const signer = new SignatureV4({ + credentials: defaultProvider(), + region: parsed.hostname.split('.')[1] || 'us-east-1', + service: 'opensearch', + sha256: Sha256, + }); + const signed = await signer.sign(request); + const resp = await fetch(url, { method, headers: signed.headers, body: isGet ? undefined : bodyBytes }); + const text = await resp.text(); + let data; try { data = JSON.parse(text); } catch { data = text; } + return { status: resp.status, data }; +} + +/** + * Discover the data-source id the managed OpenSearch UI created for the domain. + * The Security API proxy is keyed by this id (?dataSourceId=...). + */ +async function findAppDataSourceId(appEndpoint) { + for (let attempt = 0; attempt < 12; attempt++) { + const r = await osuiSecurityRequest('GET', `${appEndpoint}/api/saved_objects/_find?type=data-source&per_page=10`); + const id = r.data?.saved_objects?.[0]?.id; + if (id) return id; + await new Promise((res) => setTimeout(res, 10_000)); + } + return null; +} + +/** + * Map the OSI pipeline role (and caller principal) into the domain's FGAC roles + * through the reachable managed OpenSearch UI. Used for VPC-private domains, + * where the domain's own Security API is not reachable from this host. + */ +export async function mapOsiRoleViaOpenSearchUI(cfg) { + if (!cfg.deferFgacToUi) return; + const appEndpoint = cfg.appEndpoint; + if (!appEndpoint) { + printWarning('No OpenSearch UI endpoint — cannot map FGAC roles for the VPC domain.'); + printInfo('Map the OSI role manually in OpenSearch UI → Security → Roles once the UI is reachable.'); + return; + } + + printStep('Mapping roles in OpenSearch FGAC (via OpenSearch UI)...'); + + const dsId = await findAppDataSourceId(appEndpoint); + if (!dsId) { + printWarning('OpenSearch UI has not connected the domain data source yet — skipping FGAC mapping.'); + printInfo('Re-run the installer, or map the OSI role manually in OpenSearch UI → Security → Roles.'); + return; + } + + const { backendRoles: newBackendRoles, users: newUsers } = fgacPrincipals(cfg); + + // A VPC-private domain returns "No Living connections" through the UI until the + // UI→domain VPC endpoint connection is live, which lags the data-source object + // by a bit. So retry each role until the proxy actually reaches the domain. + const looksUnreachable = (data) => /no living connections|data source error|not ready|unavailable/i.test( + typeof data === 'string' ? data : JSON.stringify(data || ''), + ); + + async function mapRoleViaUi(role) { + const base = `${appEndpoint}/api/v1/configuration/rolesmapping/${role}?dataSourceId=${dsId}`; + await withRetry(async () => { + const getResp = await osuiSecurityRequest('GET', base); + if (getResp.status >= 500 || looksUnreachable(getResp.data)) { + throw new Error(`UI security API GET ${role}: ${getResp.status} ${JSON.stringify(getResp.data).slice(0, 160)}`); + } + const cur = (getResp.status === 200 && typeof getResp.data === 'object') ? getResp.data : {}; + const mergedBackendRoles = [...new Set([...(cur.backend_roles || []), ...newBackendRoles])]; + const mergedUsers = [...new Set([...(cur.users || []), ...newUsers])]; + + // The UI security API replaces the mapping wholesale, so send the merged set. + const resp = await osuiSecurityRequest('POST', base, { + backend_roles: mergedBackendRoles, + hosts: cur.hosts || [], + users: mergedUsers, + }); + if (resp.status !== 200 || looksUnreachable(resp.data)) { + throw new Error(`UI security API POST ${role}: ${resp.status} ${JSON.stringify(resp.data).slice(0, 160)}`); + } + }, { + shouldRetry: (e) => isTransientHttpError(e) || looksUnreachable(e.message) || /: (5\d\d|4\d\d) /.test(e.message), + onRetry: (e, i) => printInfo(`OpenSearch UI not connected to the VPC domain yet (attempt ${i + 1}) — retrying`), + }); + } + + try { + for (const role of FGAC_ROLES) await mapRoleViaUi(role); + printSuccess('Roles mapped to all_access and security_manager via OpenSearch UI'); + } catch (err) { + printError(`Could not map the OSI role via OpenSearch UI: ${err.message}`); + printInfo('The pipeline cannot write to the VPC-private domain until this role is mapped.'); + printInfo('Map the OSI role manually in OpenSearch UI → Security → Roles, or re-run the installer.'); + throw new Error('FGAC role mapping via OpenSearch UI failed — pipeline would not be able to write to OpenSearch'); } } @@ -750,15 +999,37 @@ export async function createOsiPipeline(cfg, pipelineYaml) { if (!skipCreate) { try { - await client.send(new CreatePipelineCommand({ - PipelineName: cfg.pipelineName, - MinUnits: cfg.minOcu, - MaxUnits: cfg.maxOcu, - PipelineConfigurationBody: pipelineYaml, - PipelineRoleArn: cfg.iamRoleArn, - Tags: stackTags(cfg.pipelineName), - })); - printSuccess(`Pipeline '${cfg.pipelineName}' creation initiated`); + // When the domain lives in a VPC, attach the pipeline to the same network so it + // can reach the private domain endpoint. OSIS accepts at most 2 subnets; pick the + // first two provided (each in a distinct AZ). The ingestion endpoint becomes + // VPC-private, which is what in-VPC workloads (EKS/ECS) expect. + const inVpc = Boolean(cfg.vpcId); + const vpcOptions = inVpc ? { + VpcOptions: { + SubnetIds: cfg.subnetIds.slice(0, 2), + SecurityGroupIds: cfg.securityGroupIds, + }, + } : {}; + + // OSIS validates the pipeline role's assume-role trust synchronously. When + // the role was just created, IAM may not have propagated yet, so retry on + // role-not-found / cannot-assume errors instead of failing the whole run. + await withRetry( + () => client.send(new CreatePipelineCommand({ + PipelineName: cfg.pipelineName, + MinUnits: cfg.minOcu, + MaxUnits: cfg.maxOcu, + PipelineConfigurationBody: pipelineYaml, + PipelineRoleArn: cfg.iamRoleArn, + ...vpcOptions, + Tags: stackTags(cfg.pipelineName), + })), + { + shouldRetry: isRoleNotPropagatedError, + onRetry: (e, i) => printInfo(`Pipeline role not propagated yet (attempt ${i + 1}) — retrying`), + }, + ); + printSuccess(`Pipeline '${cfg.pipelineName}' creation initiated${inVpc ? ` (VPC-attached)` : ''}`); } catch (err) { printError('Failed to create OSI pipeline'); console.error(); @@ -936,16 +1207,24 @@ export async function createConnectedDataSource(cfg) { const workspaceArn = `arn:aws:aps:${cfg.region}:${cfg.accountId}:workspace/${cfg.apsWorkspaceId}`; try { - const result = await client.send(new AddDirectQueryDataSourceCommand({ - DataSourceName: dataSourceName, - DataSourceType: { - Prometheus: { - RoleArn: cfg.connectedDataSourceRoleArn, - WorkspaceArn: workspaceArn, + // The direct-query data source assumes connectedDataSourceRoleArn, which may + // have just been created; retry while IAM propagation catches up. + const result = await withRetry( + () => client.send(new AddDirectQueryDataSourceCommand({ + DataSourceName: dataSourceName, + DataSourceType: { + Prometheus: { + RoleArn: cfg.connectedDataSourceRoleArn, + WorkspaceArn: workspaceArn, + }, }, + Description: `Prometheus data source for ${cfg.pipelineName} observability stack`, + })), + { + shouldRetry: isRoleNotPropagatedError, + onRetry: (e, i) => printInfo(`Connected Data Source role not propagated yet (attempt ${i + 1}) — retrying`), }, - Description: `Prometheus data source for ${cfg.pipelineName} observability stack`, - })); + ); cfg.connectedDataSourceArn = result.DataSourceArn; printSuccess(`Connected Data Source created: ${cfg.connectedDataSourceArn}`); await tagResource(cfg.region, cfg.connectedDataSourceArn, cfg.pipelineName); @@ -1035,15 +1314,31 @@ export async function createOpenSearchApplication(cfg) { } /** - * Fetch the application endpoint via GetApplicationCommand. + * Fetch the application endpoint via GetApplicationCommand, waiting for the app + * to reach ACTIVE with a populated endpoint. CreateApplication returns before the + * endpoint is provisioned, so a single read races an empty value — which would + * skip FGAC role mapping and UI setup for VPC domains. Poll until it appears. */ async function fetchAppEndpoint(client, cfg) { if (!cfg.appId) return; - try { - const resp = await client.send(new GetApplicationCommand({ id: cfg.appId })); - cfg.appEndpoint = resp.endpoint || ''; - // endpoint logged by setupDashboards - } catch { /* best effort */ } + const maxWait = 300_000; // 5 min + const interval = 5_000; + const start = Date.now(); + while (Date.now() - start < maxWait) { + try { + const resp = await client.send(new GetApplicationCommand({ id: cfg.appId })); + if (resp.endpoint) { + cfg.appEndpoint = resp.endpoint; + return; // endpoint logged by setupDashboards + } + if (resp.status && !['CREATING', 'UPDATING', 'ACTIVE'].includes(resp.status)) { + printWarning(`OpenSearch Application status is ${resp.status} — endpoint may not become available`); + return; + } + } catch { /* keep polling */ } + await sleep(interval); + } + printWarning('Timed out waiting for the OpenSearch Application endpoint'); } /** @@ -1200,6 +1495,188 @@ export async function listApplications(region) { })); } +// ── VPC / subnet / security group listing (for interactive VPC selection) ──── + +function nameTag(tags) { + return (tags || []).find((t) => t.Key === 'Name')?.Value || ''; +} + +/** + * List VPCs in the given region. + * Returns [{ id, cidr, isDefault, name }]. + */ +export async function listVpcs(region) { + const { EC2Client, DescribeVpcsCommand } = await import('@aws-sdk/client-ec2'); + const client = new EC2Client({ region }); + const resp = await client.send(new DescribeVpcsCommand({})); + return (resp.Vpcs || []).map((v) => ({ + id: v.VpcId, + cidr: v.CidrBlock || '', + isDefault: Boolean(v.IsDefault), + name: nameTag(v.Tags), + })); +} + +/** + * List subnets for a VPC. + * Returns [{ id, az, cidr, name, mapPublicIp }]. + */ +export async function listSubnets(region, vpcId) { + const { EC2Client, DescribeSubnetsCommand } = await import('@aws-sdk/client-ec2'); + const client = new EC2Client({ region }); + const resp = await client.send(new DescribeSubnetsCommand({ + Filters: [{ Name: 'vpc-id', Values: [vpcId] }], + })); + return (resp.Subnets || []).map((s) => ({ + id: s.SubnetId, + az: s.AvailabilityZone || '', + cidr: s.CidrBlock || '', + name: nameTag(s.Tags), + mapPublicIp: Boolean(s.MapPublicIpOnLaunch), + })); +} + +/** + * List security groups for a VPC. + * Returns [{ id, name, description }]. + */ +export async function listSecurityGroups(region, vpcId) { + const { EC2Client, DescribeSecurityGroupsCommand } = await import('@aws-sdk/client-ec2'); + const client = new EC2Client({ region }); + const resp = await client.send(new DescribeSecurityGroupsCommand({ + Filters: [{ Name: 'vpc-id', Values: [vpcId] }], + })); + return (resp.SecurityGroups || []).map((g) => ({ + id: g.GroupId, + name: g.GroupName || '', + description: g.Description || '', + })); +} + +/** + * Validate the VPC topology against live EC2 state so the run fails fast, before + * any OpenSearch/OSIS resources are created. The syntactic checks in + * validateConfig() only confirm the IDs are well-formed; this confirms they + * actually exist, belong together, and satisfy OpenSearch's zone-awareness rules. + * + * Catches (each of which otherwise surfaces minutes into domain/pipeline creation): + * - VPC does not exist / wrong region. + * - A subnet or security group is not a member of the given VPC. OpenSearch's + * CreateDomain rejects a subnet/SG that lives in a different VPC. + * - Two subnets share an Availability Zone. createOpenSearch() derives the + * zone-awareness AZ count from the subnet count (min(subnetIds.length, 3)), + * so duplicate AZs make CreateDomain fail with a ValidationException. + * + * @param {object} cfg the resolved config (needs region, vpcId, subnetIds, securityGroupIds) + * @param {object} [deps] optional injected EC2 accessors for testing + * @returns {Promise} error strings (empty = valid) + */ +export async function validateVpcTopology(cfg, deps = {}) { + // Only relevant when a VPC deployment was requested. Well-formedness is assumed + // to have been checked by validateConfig() already. + if (!cfg.vpcId) return []; + + const describeVpcs = deps.describeVpcs || (async (region, ids) => { + const { EC2Client, DescribeVpcsCommand } = await import('@aws-sdk/client-ec2'); + const client = new EC2Client({ region }); + return (await client.send(new DescribeVpcsCommand({ VpcIds: ids }))).Vpcs || []; + }); + const describeSubnets = deps.describeSubnets || (async (region, ids) => { + const { EC2Client, DescribeSubnetsCommand } = await import('@aws-sdk/client-ec2'); + const client = new EC2Client({ region }); + return (await client.send(new DescribeSubnetsCommand({ SubnetIds: ids }))).Subnets || []; + }); + const describeSecurityGroups = deps.describeSecurityGroups || (async (region, ids) => { + const { EC2Client, DescribeSecurityGroupsCommand } = await import('@aws-sdk/client-ec2'); + const client = new EC2Client({ region }); + return (await client.send(new DescribeSecurityGroupsCommand({ GroupIds: ids }))).SecurityGroups || []; + }); + + const errors = []; + const region = cfg.region; + const subnetIds = cfg.subnetIds || []; + const securityGroupIds = cfg.securityGroupIds || []; + + // 1. VPC exists. A missing/invalid VPC throws InvalidVpcID.NotFound — translate + // that into a clean error rather than an SDK stack trace. + try { + const vpcs = await describeVpcs(region, [cfg.vpcId]); + if (!vpcs.length) { + errors.push(`VPC ${cfg.vpcId} was not found in ${region}. Check the ID and region.`); + return errors; // Nothing else can be validated without the VPC. + } + } catch (err) { + if (/InvalidVpcID\.NotFound|does not exist/i.test(err.message || '')) { + errors.push(`VPC ${cfg.vpcId} was not found in ${region}. Check the ID and region.`); + } else { + errors.push(`Could not verify VPC ${cfg.vpcId}: ${err.message}`); + } + return errors; + } + + // 2. Subnets: each must exist and belong to the VPC; collect their AZs. + if (subnetIds.length) { + try { + const subnets = await describeSubnets(region, subnetIds); + const found = new Map(subnets.map((s) => [s.SubnetId, s])); + for (const id of subnetIds) { + const s = found.get(id); + if (!s) { + errors.push(`Subnet ${id} was not found in ${region}.`); + } else if (s.VpcId !== cfg.vpcId) { + errors.push(`Subnet ${id} belongs to VPC ${s.VpcId}, not ${cfg.vpcId}. All subnets must be in the target VPC.`); + } + } + // Zone-awareness: subnets must be in distinct AZs. createOpenSearch() + // enables zone awareness with AvailabilityZoneCount = min(subnetIds, 3) + // and places one node group per AZ, so two subnets in the same AZ make + // CreateDomain fail. Only meaningful with more than one subnet. + const inVpc = subnets.filter((s) => s.VpcId === cfg.vpcId); + if (inVpc.length > 1) { + const azSeen = new Map(); + for (const s of inVpc) { + const az = s.AvailabilityZone; + if (azSeen.has(az)) { + errors.push(`Subnets ${azSeen.get(az)} and ${s.SubnetId} are both in ${az}. OpenSearch requires each subnet to be in a distinct Availability Zone for a zone-aware domain.`); + } else { + azSeen.set(az, s.SubnetId); + } + } + } + } catch (err) { + if (/InvalidSubnetID\.NotFound|does not exist/i.test(err.message || '')) { + errors.push(`One or more subnets were not found in ${region}: ${subnetIds.join(', ')}.`); + } else { + errors.push(`Could not verify subnets: ${err.message}`); + } + } + } + + // 3. Security groups: each must exist and belong to the VPC. + if (securityGroupIds.length) { + try { + const groups = await describeSecurityGroups(region, securityGroupIds); + const found = new Map(groups.map((g) => [g.GroupId, g])); + for (const id of securityGroupIds) { + const g = found.get(id); + if (!g) { + errors.push(`Security group ${id} was not found in ${region}.`); + } else if (g.VpcId !== cfg.vpcId) { + errors.push(`Security group ${id} belongs to VPC ${g.VpcId}, not ${cfg.vpcId}. All security groups must be in the target VPC.`); + } + } + } catch (err) { + if (/InvalidGroup\.NotFound|does not exist/i.test(err.message || '')) { + errors.push(`One or more security groups were not found in ${region}: ${securityGroupIds.join(', ')}.`); + } else { + errors.push(`Could not verify security groups: ${err.message}`); + } + } + } + + return errors; +} + // ── Pipeline listing / describe / update ───────────────────────────────────── /** @@ -1501,6 +1978,54 @@ function sleep(ms) { return new Promise((resolve) => setTimeout(resolve, ms)); } +/** + * Retry an async operation on transient failures with exponential backoff. + * `shouldRetry(err)` decides whether an error is worth retrying (default: any). + * Returns the operation's result, or rethrows the last error once `attempts` + * is exhausted. `onRetry(err, attempt)` runs between tries for progress output. + */ +async function withRetry(fn, { attempts = 6, delayMs = 5000, backoff = 1.6, maxDelayMs = 30_000, shouldRetry = () => true, onRetry } = {}) { + let lastErr; + for (let i = 0; i < attempts; i++) { + try { + return await fn(i); + } catch (err) { + lastErr = err; + if (i === attempts - 1 || !shouldRetry(err)) throw err; + if (onRetry) onRetry(err, i); + await sleep(Math.min(maxDelayMs, Math.round(delayMs * backoff ** i))); + } + } + throw lastErr; +} + +/** + * True when an error is a freshly-created IAM role that hasn't propagated yet. + * OSIS/OpenSearch validate assume-role synchronously and reject with these + * shapes until the role and its trust policy are globally consistent. + */ +function isRoleNotPropagatedError(err) { + const msg = err?.message || ''; + const name = err?.name || ''; + return ( + /cannot be assumed|not authorized to perform: sts:AssumeRole|unable to assume|does not have permission to assume|role .*(does not exist|not found)|Invalid .*RoleArn|no such entity/i.test(msg) || + name === 'ValidationException' && /role/i.test(msg) + ); +} + +/** + * True when an HTTP/network error against a just-provisioned OpenSearch domain + * or the managed UI proxy is transient (cluster still warming up, VPC endpoint + * connection not yet live). These clear on their own within a minute or two. + */ +function isTransientHttpError(err) { + const msg = err?.message || String(err || ''); + return /ECONNREFUSED|ECONNRESET|ETIMEDOUT|EAI_AGAIN|socket hang up|network|fetch failed|terminated|502|503|504|timeout/i.test(msg); +} + +// Exported for unit tests. +export { withRetry as _withRetry, isRoleNotPropagatedError as _isRoleNotPropagatedError, isTransientHttpError as _isTransientHttpError }; + function fmtElapsed(totalSec) { if (totalSec < 60) return `${totalSec}s`; const m = Math.floor(totalSec / 60); diff --git a/aws/cli-installer/src/cli.mjs b/aws/cli-installer/src/cli.mjs index 5afd6d41..68d727d4 100644 --- a/aws/cli-installer/src/cli.mjs +++ b/aws/cli-installer/src/cli.mjs @@ -53,6 +53,12 @@ export function parseCli(argv) { .option('--os-engine-version ', 'Engine version', DEFAULTS.osEngineVersion) .option('--managed', 'Target is OpenSearch managed domain'); + // Network topology — deploy into a VPC instead of public endpoints + program + .option('--vpc-id ', 'Deploy the domain, pipeline, and demo into this VPC (private endpoints)') + .option('--subnet-ids ', 'Comma-separated subnet IDs for VPC deployment') + .option('--security-group-ids ', 'Comma-separated security group IDs for VPC deployment'); + // IAM program .option('--iam-role-arn ', 'Reuse an existing IAM role') @@ -104,6 +110,15 @@ function parseDestroyArgs(argv) { return { _command: 'destroy', ...program.opts() }; } +/** + * Parse a comma-separated list of IDs into a trimmed, de-duplicated array. + * Returns [] for empty/undefined input. + */ +function parseIdList(value) { + if (!value) return []; + return [...new Set(value.split(',').map((s) => s.trim()).filter(Boolean))]; +} + /** * Convert commander opts to our normalized config shape. */ @@ -143,6 +158,9 @@ function optsToConfig(opts) { osInstanceCount: Number(opts.osInstanceCount), osVolumeSize: Number(opts.osVolumeSize), osEngineVersion: opts.osEngineVersion, + vpcId: opts.vpcId || '', + subnetIds: parseIdList(opts.subnetIds), + securityGroupIds: parseIdList(opts.securityGroupIds), iamAction, iamRoleArn: opts.iamRoleArn || '', iamRoleName: opts.iamRoleName || '', @@ -232,6 +250,13 @@ export function validateConfig(cfg) { } if (!cfg.region) errors.push('--region is required'); + // OpenSearch backend must be resolved to either create or reuse. In advanced + // mode with no OpenSearch flags, osAction stays empty and the domain step is + // silently skipped — the run then fails deep in pipeline creation with an empty + // endpoint. Catch it up front so the CLI fails fast with a clear message. + if (!cfg.osAction) { + errors.push('No OpenSearch backend specified. Pass --os-domain-name (create a managed domain), --aoss-collection-name (create a serverless collection), or --opensearch-endpoint (reuse an existing one). Or run with --quick to auto-create defaults.'); + } if (cfg.osAction === 'reuse' && !cfg.opensearchEndpoint) { errors.push('--opensearch-endpoint required when reusing OpenSearch'); } @@ -262,5 +287,31 @@ export function validateConfig(cfg) { errors.push('Prometheus URL must start with http:// or https://'); } + // VPC deployment: all three (vpc/subnets/SGs) must be present together and well-formed. + const wantsVpc = cfg.vpcId || cfg.subnetIds?.length || cfg.securityGroupIds?.length; + if (wantsVpc) { + if (!cfg.vpcId) errors.push('--vpc-id is required when configuring subnets or security groups'); + if (!cfg.subnetIds?.length) errors.push('--subnet-ids is required for VPC deployment (at least one subnet)'); + if (!cfg.securityGroupIds?.length) errors.push('--security-group-ids is required for VPC deployment (at least one security group)'); + + if (cfg.vpcId && !/^vpc-[0-9a-f]+$/.test(cfg.vpcId)) { + errors.push(`--vpc-id must look like vpc-xxxxxxxx (got ${cfg.vpcId})`); + } + for (const id of cfg.subnetIds || []) { + if (!/^subnet-[0-9a-f]+$/.test(id)) errors.push(`Invalid subnet ID: ${id} (expected subnet-xxxxxxxx)`); + } + for (const id of cfg.securityGroupIds || []) { + if (!/^sg-[0-9a-f]+$/.test(id)) errors.push(`Invalid security group ID: ${id} (expected sg-xxxxxxxx)`); + } + // OpenSearch domains support at most 3 subnets (one per AZ, zone-awareness limit). + if (cfg.subnetIds?.length > 3) { + errors.push(`--subnet-ids accepts at most 3 subnets for an OpenSearch domain (got ${cfg.subnetIds.length})`); + } + // A VPC domain cannot be reused via a public endpoint reference; VPC only applies to newly created domains. + if (cfg.osAction === 'reuse') { + errors.push('VPC options apply only when creating a new OpenSearch domain; omit --vpc-id when reusing an existing endpoint'); + } + } + return errors; } diff --git a/aws/cli-installer/src/commands/create.mjs b/aws/cli-installer/src/commands/create.mjs index 781a2ad4..ac2a50db 100644 --- a/aws/cli-installer/src/commands/create.mjs +++ b/aws/cli-installer/src/commands/create.mjs @@ -46,7 +46,7 @@ function connector(width, specs) { return theme.muted(arr.join('').trimEnd()); } -function renderArchitectureDiagram(cfg) { +export function renderArchitectureDiagram(cfg) { const osLabel = cfg.opensearchType === 'serverless' ? 'AOSS Collection' : 'OpenSearch'; const pathLabel = `/${cfg.pipelineName}/v1/*`; const m = theme.muted; @@ -55,14 +55,23 @@ function renderArchitectureDiagram(cfg) { const h = theme.highlight; const sp = (n) => ' '.repeat(Math.max(0, n)); + // VPC mode tags the boxes that actually live inside the VPC (EC2, the OSI + // ingest endpoint, and the OpenSearch domain). Prometheus, the Connected Data + // Source, and the UI are regional managed services reached over the + // AWS-internal path, so they stay untagged. The tag is short enough to fit the + // boxes' existing minimum widths, so it adds no width and the column math below + // is unchanged from the public render. + const inVpc = Boolean(cfg.vpcId); + const vpcTag = inVpc ? ' ' + theme.success('[vpc]') : ''; + // ── Define all boxes ────────────────────────────────────────────────── - const otlp = box([a('OSI Endpoint'), m(pathLabel)], 21); + const otlp = box([a('OSI Endpoint') + vpcTag, m(pathLabel)], 21); const logs = box([h('Logs')], 9); const traces = box([h('Traces')], 9); const metrics = box([h('Metrics')], 9); const raw = box([m('Raw'), m('Traces')], 9); const svc = box([m('Service'), m('Map')], 9); - const os = box([p(osLabel), m('logs, traces, svc-map')]); + const os = box([p(osLabel) + vpcTag, m('logs, traces, svc-map')]); const prom = box([p('AWS Prometheus'), m('metrics, svc-map')]); const dash = box([p('OpenSearch UI'), m('Observability workspace')]); @@ -98,7 +107,7 @@ function renderArchitectureDiagram(cfg) { const out = ['']; // EC2 Demo box (above OTLP) - const ec2 = box([p('EC2 Instance'), m('OTel Demo + Agents')], 21); + const ec2 = box([p('EC2 Instance') + vpcTag, m('OTel Demo + Agents')], 21); const ec2Off = Math.max(0, C_OTLP - ec2.mid); out.push(sp(ec2Off) + ec2.top); for (const l of ec2.lines) out.push(sp(ec2Off) + l); @@ -182,6 +191,17 @@ function renderArchitectureDiagram(cfg) { out.push(sp(dashOff) + dash.top); for (const l of dash.lines) out.push(sp(dashOff) + l); out.push(sp(dashOff) + dash.bot); + + // Network-topology legend — only in VPC mode. Placed after the diagram (not + // before) so it reads as a caption on what was just drawn and isn't scrolled + // past. Surfaces the IDs being deployed into and which boxes are private. + if (inVpc) { + out.push(''); + out.push(a('Network topology')); + out.push(m(`VPC ${cfg.vpcId} · subnets ${(cfg.subnetIds || []).join(', ')} · security groups ${(cfg.securityGroupIds || []).join(', ')}`)); + out.push(m(`${theme.success('[vpc]')} boxes run inside the VPC; Prometheus, Connected Data Source, and the UI are regional (AWS-internal path).`)); + } + out.push(''); return out; diff --git a/aws/cli-installer/src/config.mjs b/aws/cli-installer/src/config.mjs index a94b745b..49dd5047 100644 --- a/aws/cli-installer/src/config.mjs +++ b/aws/cli-installer/src/config.mjs @@ -34,6 +34,15 @@ export function createDefaultConfig() { osInstanceCount: DEFAULTS.osInstanceCount, osVolumeSize: DEFAULTS.osVolumeSize, osEngineVersion: DEFAULTS.osEngineVersion, + // Network topology (empty = public endpoints, current default behavior) + vpcId: '', + subnetIds: [], + securityGroupIds: [], + // For VPC-private domains: the domain master is the caller's IAM principal + // (set at create time) and FGAC role mapping is deferred to run through the + // reachable managed OpenSearch UI once the Application exists. + iamMasterArn: '', + deferFgacToUi: false, iamAction: '', iamRoleArn: '', iamRoleName: '', diff --git a/aws/cli-installer/src/destroy.mjs b/aws/cli-installer/src/destroy.mjs index 3fd840b1..bf0a56ef 100644 --- a/aws/cli-installer/src/destroy.mjs +++ b/aws/cli-installer/src/destroy.mjs @@ -15,12 +15,22 @@ async function cleanupFgacRoles(region, pipelineName, opensearchPassword, os) { const app = (ApplicationSummaries || []).find(a => a.name === pipelineName); if (!app) return; - const { dataSources } = await os.send(new GetApplicationCommand({ id: app.id })); + const { dataSources, endpoint: appEndpoint } = await os.send(new GetApplicationCommand({ id: app.id })); const domainArn = (dataSources || []).find(d => d.dataSourceArn?.includes(':domain/'))?.dataSourceArn; if (!domainArn) return; const domainName = domainArn.split('/').pop(); const { DomainStatus } = await os.send(new DescribeDomainCommand({ DomainName: domainName })); + const isVpc = Boolean(DomainStatus?.VPCOptions?.VPCId); + + // VPC-private domain: its Security API is unreachable from here, but the + // managed OpenSearch UI proxies to it. Clean up role mappings through the UI. + if (isVpc) { + if (!appEndpoint) return; + await cleanupFgacRolesViaUi(appEndpoint, pipelineName); + return; + } + if (!DomainStatus?.Endpoint) return; // Get password from Secrets Manager or flag @@ -65,6 +75,50 @@ async function cleanupFgacRoles(region, pipelineName, opensearchPassword, os) { } } +// Remove this stack's backend roles from all_access via the managed OpenSearch +// UI (SigV4), used for VPC-private domains that aren't reachable directly. +async function cleanupFgacRolesViaUi(appEndpoint, pipelineName) { + const { createHash } = await import('node:crypto'); + const { SignatureV4 } = await import('@aws-sdk/signature-v4'); + const { Sha256 } = await import('@aws-crypto/sha256-js'); + const { HttpRequest } = await import('@smithy/protocol-http'); + const { defaultProvider } = await import('@aws-sdk/credential-provider-node'); + + async function req(method, url, body) { + const isGet = method === 'GET'; + const bodyBytes = (!isGet && body) ? JSON.stringify(body) : ''; + const parsed = new URL(url); + const query = {}; parsed.searchParams.forEach((v, k) => { query[k] = v; }); + const request = new HttpRequest({ + method, protocol: parsed.protocol, hostname: parsed.hostname, path: parsed.pathname, query, + headers: { host: parsed.hostname, 'Content-Type': 'application/json', 'osd-xsrf': 'osd-fetch', + 'x-amz-content-sha256': createHash('sha256').update(bodyBytes).digest('hex') }, + body: bodyBytes || undefined, + }); + const signer = new SignatureV4({ credentials: defaultProvider(), + region: parsed.hostname.split('.')[1] || 'us-east-1', service: 'opensearch', sha256: Sha256 }); + const signed = await signer.sign(request); + const resp = await fetch(url, { method, headers: signed.headers, body: isGet ? undefined : bodyBytes }); + const text = await resp.text(); + let data; try { data = JSON.parse(text); } catch { data = text; } + return { status: resp.status, data }; + } + + const find = await req('GET', `${appEndpoint}/api/saved_objects/_find?type=data-source&per_page=10`); + const dsId = find.data?.saved_objects?.[0]?.id; + if (!dsId) return; + + const base = `${appEndpoint}/api/v1/configuration/rolesmapping/all_access?dataSourceId=${dsId}`; + const cur = await req('GET', base); + if (cur.status !== 200 || typeof cur.data !== 'object') return; + const existing = cur.data.backend_roles || []; + const filtered = existing.filter(r => !r.includes(pipelineName)); + if (filtered.length !== existing.length) { + await req('POST', base, { backend_roles: filtered, hosts: cur.data.hosts || [], users: cur.data.users || [] }); + printSuccess('FGAC backend role mappings cleaned up (via OpenSearch UI)'); + } +} + export async function destroy(cfg) { const { pipelineName, region } = cfg; if (!pipelineName) throw new Error('--pipeline-name is required'); diff --git a/aws/cli-installer/src/ec2-demo.mjs b/aws/cli-installer/src/ec2-demo.mjs index 56fc7c32..efdeb6e1 100644 --- a/aws/cli-installer/src/ec2-demo.mjs +++ b/aws/cli-installer/src/ec2-demo.mjs @@ -300,26 +300,41 @@ export async function launchDemoInstance(cfg) { const iam = new IAMClient({ region: cfg.region }); const ssm = new SSMClient({ region: cfg.region }); + // When a VPC was selected for the stack, place the demo instance in the same + // network so it can reach a VPC-private OSIS ingestion endpoint. Otherwise fall + // back to the account's default VPC subnet (current behavior). + const inVpc = Boolean(cfg.vpcId); + const spinner = createSpinner('Looking up AMI and subnet...'); - const [ami, subnet] = await Promise.all([getLatestAL2023Ami(ssm), getDefaultVpcSubnet(ec2, INSTANCE_TYPE)]); + const amiPromise = getLatestAL2023Ami(ssm); + const subnetId = inVpc ? cfg.subnetIds[0] : (await getDefaultVpcSubnet(ec2, INSTANCE_TYPE)).SubnetId; + const ami = await amiPromise; spinner.stop(`AMI: ${ami}`); - const sgSpinner = createSpinner('Creating security group...'); - const sgId = await createDemoSecurityGroup(ec2, cfg); - sgSpinner.stop(`Security group: ${sgId}`); + // In VPC mode, reuse the caller-provided security groups (they should already + // allow the intra-VPC traffic the domain/pipeline need). Otherwise create a + // dedicated demo SG in the default VPC. + let sgIds; + if (inVpc) { + sgIds = cfg.securityGroupIds; + } else { + const sgSpinner = createSpinner('Creating security group...'); + sgIds = [await createDemoSecurityGroup(ec2, cfg)]; + sgSpinner.stop(`Security group: ${sgIds[0]}`); + } const profileSpinner = createSpinner('Creating instance profile...'); const profileName = await createDemoInstanceProfile(iam, cfg); profileSpinner.stop(`Instance profile: ${profileName}`); - const launchSpinner = createSpinner(`Launching ${INSTANCE_TYPE} instance...`); + const launchSpinner = createSpinner(`Launching ${INSTANCE_TYPE} instance${inVpc ? ` in ${subnetId}` : ''}...`); const { Instances } = await ec2.send(new RunInstancesCommand({ ImageId: ami, InstanceType: INSTANCE_TYPE, MinCount: 1, MaxCount: 1, - SubnetId: subnet.SubnetId, - SecurityGroupIds: [sgId], + SubnetId: subnetId, + SecurityGroupIds: sgIds, IamInstanceProfile: { Name: profileName }, UserData: buildUserData(cfg), TagSpecifications: tagSpec('instance', cfg.pipelineName), @@ -338,7 +353,7 @@ export async function launchDemoInstance(cfg) { printInfo(`View init logs: aws ssm start-session --target ${instanceId} --region ${cfg.region}`); cfg.demoInstanceId = instanceId; - cfg.demoSecurityGroupId = sgId; + cfg.demoSecurityGroupId = sgIds[0]; return instanceId; } diff --git a/aws/cli-installer/src/interactive.mjs b/aws/cli-installer/src/interactive.mjs index d80b676b..01522d54 100644 --- a/aws/cli-installer/src/interactive.mjs +++ b/aws/cli-installer/src/interactive.mjs @@ -1,10 +1,10 @@ import { - printHeader, printStep, printInfo, printSubStep, - createSpinner, theme, GoBack, eSelect, eInput, + printHeader, printStep, printInfo, printSubStep, printWarning, + createSpinner, theme, GoBack, eSelect, eInput, eCheckbox, saveCursor, clearFromCursor, } from './ui.mjs'; import { createDefaultConfig, DEFAULTS, DEFAULT_REGION } from './config.mjs'; -import { listDomains, listCollections, listWorkspaces, listApplications } from './aws.mjs'; +import { listDomains, listCollections, listWorkspaces, listApplications, listVpcs, listSubnets, listSecurityGroups } from './aws.mjs'; const CUSTOM_INPUT = Symbol('custom'); @@ -271,6 +271,103 @@ async function stepOpenSearch(cfg) { } } +async function stepVpc(cfg) { + if (cfg.mode !== 'advanced') return 'skip'; + // VPC options only apply when creating a new domain (not reusing an endpoint). + if (cfg.osAction === 'reuse') return 'skip'; + + printStep('Network topology'); + printInfo('Deploy the domain, ingestion pipeline, and demo into a VPC for private endpoints,'); + printInfo('or use public endpoints (default).'); + console.error(); + + while (true) { + const netChoice = await eSelect({ + message: 'Deployment network', + choices: [ + { name: `Public endpoints ${theme.muted('— default, reachable over the internet')}`, value: 'public' }, + { name: `VPC ${theme.muted('— private endpoints inside a VPC (EKS/ECS workloads)')}`, value: 'vpc' }, + ], + default: cfg.vpcId ? 'vpc' : 'public', + }); + if (netChoice === GoBack) return GoBack; + + if (netChoice === 'public') { + cfg.vpcId = ''; + cfg.subnetIds = []; + cfg.securityGroupIds = []; + return; + } + + // VPC selection + const vpcs = await fetchWithSpinner('Loading VPCs', () => listVpcs(cfg.region)); + if (!vpcs.length) { + printWarning('No VPCs found in this region — falling back to public endpoints.'); + cfg.vpcId = ''; + return; + } + const vpcChoices = vpcs.map((v) => ({ + name: `${v.id} ${theme.muted(`— ${v.cidr}${v.name ? ` (${v.name})` : ''}${v.isDefault ? ' [default]' : ''}`)}`, + value: v.id, + })); + const selectedVpc = await eSelect({ message: 'Select VPC', choices: vpcChoices }); + if (selectedVpc === GoBack) continue; + cfg.vpcId = selectedVpc; + + // Subnet selection (multi-select, at most 3 for the domain) + const subnets = await fetchWithSpinner('Loading subnets', () => listSubnets(cfg.region, cfg.vpcId)); + if (!subnets.length) { + printWarning('No subnets found in that VPC — pick another VPC or use public endpoints.'); + continue; + } + const subnetChoices = subnets.map((s) => ({ + name: `${s.id} ${theme.muted(`— ${s.az} ${s.cidr}${s.name ? ` (${s.name})` : ''}`)}`, + value: s.id, + })); + const selectedSubnets = await eCheckbox({ + message: 'Select subnets (1-3, ideally one per AZ)', + choices: subnetChoices, + required: true, + }); + if (selectedSubnets === GoBack) continue; + if (!selectedSubnets.length) { + printWarning('Select at least one subnet.'); + continue; + } + if (selectedSubnets.length > 3) { + printWarning('An OpenSearch domain supports at most 3 subnets — select 3 or fewer.'); + continue; + } + cfg.subnetIds = selectedSubnets; + + // Security group selection (multi-select) + const sgs = await fetchWithSpinner('Loading security groups', () => listSecurityGroups(cfg.region, cfg.vpcId)); + if (!sgs.length) { + printWarning('No security groups found in that VPC — pick another VPC or use public endpoints.'); + continue; + } + const sgChoices = sgs.map((g) => ({ + name: `${g.id} ${theme.muted(`— ${g.name}${g.description ? ` (${g.description})` : ''}`)}`, + value: g.id, + })); + const selectedSgs = await eCheckbox({ + message: 'Select security groups', + choices: sgChoices, + required: true, + }); + if (selectedSgs === GoBack) continue; + if (!selectedSgs.length) { + printWarning('Select at least one security group.'); + continue; + } + cfg.securityGroupIds = selectedSgs; + + printInfo('The pipeline and demo instance will be placed in this VPC too.'); + printInfo('Your IAM principal will be the domain master; FGAC mapping and UI setup run through the managed OpenSearch UI, so you can run this from anywhere.'); + return; + } +} + async function stepIam(cfg) { if (cfg.mode !== 'advanced') return 'skip'; @@ -539,7 +636,7 @@ export async function runCreateWizard(session = null) { if (!session) printHeader(); - const steps = [stepMode, stepCore, stepOpenSearch, stepIam, stepAps, stepConnectedDataSourceRole, stepConnectedDataSource, stepApp, stepTuning, stepDemo]; + const steps = [stepMode, stepCore, stepOpenSearch, stepVpc, stepIam, stepAps, stepConnectedDataSourceRole, stepConnectedDataSource, stepApp, stepTuning, stepDemo]; const visited = []; let i = 0; diff --git a/aws/cli-installer/src/main.mjs b/aws/cli-installer/src/main.mjs index b705cd82..3a62f0a4 100644 --- a/aws/cli-installer/src/main.mjs +++ b/aws/cli-installer/src/main.mjs @@ -8,15 +8,18 @@ import { createApsWorkspace, createOsiPipeline, mapOsiRoleInDomain, + mapOsiRoleViaOpenSearchUI, createAossDataAccessPolicy, setupDashboards, createConnectedDataSourceRole, createConnectedDataSource, createOpenSearchApplication, + validateVpcTopology, } from './aws.mjs'; import { printError, printSuccess, + printStep, printPanel, printBox, STAR, @@ -91,6 +94,22 @@ export async function run() { */ export async function executePipeline(cfg) { await checkRequirements(cfg); + + // Live VPC validation — verify the VPC/subnets/SGs exist, belong together, and + // satisfy zone-awareness before creating any OpenSearch resources. Fails fast + // so we don't leave a half-built stack when the network inputs are wrong. + if (cfg.vpcId) { + printStep('Validating VPC network configuration...'); + const vpcErrors = await validateVpcTopology(cfg); + if (vpcErrors.length) { + console.error(); + for (const e of vpcErrors) printError(e); + throw new Error('VPC configuration is invalid; no resources were created.'); + } + printSuccess('VPC, subnets, and security groups validated'); + console.error(); + } + printSummary(cfg); console.error(); @@ -143,6 +162,14 @@ export async function executePipeline(cfg) { console.error(); } + // For VPC-private domains, FGAC role mapping is deferred until the managed + // OpenSearch UI (Application) exists, since the UI proxies to the domain over + // the AWS-internal path — no in-VPC network access needed from this host. + if (cfg.deferFgacToUi) { + await mapOsiRoleViaOpenSearchUI(cfg); + console.error(); + } + // Generate pipeline YAML const pipelineYaml = renderPipeline(cfg); @@ -181,7 +208,9 @@ export async function executePipeline(cfg) { `${theme.label(pad('OSI Pipeline Role:'))} ${cfg.iamRoleArn}`, `${theme.label(pad('OpenSearch:'))} ${link(cfg.opensearchEndpoint)}`, ...(cfg.opensearchType !== 'serverless' ? [ - `${theme.label(pad('OpenSearch Master Password:'))} Secrets Manager: observability-stack/${cfg.pipelineName}/master-password`, + cfg.iamMasterArn + ? `${theme.label(pad('OpenSearch Master:'))} IAM principal ${cfg.iamMasterArn}` + : `${theme.label(pad('OpenSearch Master Password:'))} Secrets Manager: observability-stack/${cfg.pipelineName}/master-password`, ] : []), `${theme.label(pad('OpenSearch UI:'))} ${link(cfg.dashboardsUrl)}`, `${theme.label(pad('Prometheus:'))} ${link(cfg.prometheusUrl)}`, @@ -234,6 +263,13 @@ function printSummary(cfg) { osEntries.push(['Instance count', String(cfg.osInstanceCount)]); osEntries.push(['Volume size', `${cfg.osVolumeSize} GB`]); osEntries.push(['Engine version', cfg.osEngineVersion]); + if (cfg.vpcId) { + osEntries.push(['Network', `VPC ${cfg.vpcId}`]); + osEntries.push(['Subnets', cfg.subnetIds.join(', ')]); + osEntries.push(['Security groups', cfg.securityGroupIds.join(', ')]); + } else { + osEntries.push(['Network', 'public endpoint']); + } } // IAM diff --git a/aws/cli-installer/src/ui.mjs b/aws/cli-installer/src/ui.mjs index dfcb9ffb..7464cf37 100644 --- a/aws/cli-installer/src/ui.mjs +++ b/aws/cli-installer/src/ui.mjs @@ -1,7 +1,7 @@ import chalk from 'chalk'; import ora from 'ora'; import readline from 'node:readline'; -import { search, input, confirm } from '@inquirer/prompts'; +import { search, input, confirm, checkbox } from '@inquirer/prompts'; // ── Theme colors ───────────────────────────────────────────────────────────── @@ -291,6 +291,52 @@ export function eSelect(opts) { ); } +// ── Multi-select checkbox ───────────────────────────────────────────────────── + +/** + * Checkbox (multi-select) prompt with Escape-to-go-back. + * Space to toggle, Enter to confirm, Esc to go back. + * Returns an array of selected values, or GoBack on Escape. + */ +export function eCheckbox(opts) { + if (!_keypressInit && process.stdin.isTTY) { + readline.emitKeypressEvents(process.stdin); + _keypressInit = true; + } + + const promise = checkbox({ + message: opts.message || 'Select', + choices: opts.choices || [], + required: opts.required ?? false, + theme: _selectKeyTheme, + }); + + let escaped = false; + const onKeypress = (_ch, key) => { + if (key?.name === 'escape') { + escaped = true; + promise.cancel(); + } + }; + process.stdin.on('keypress', onKeypress); + const cleanup = () => process.stdin.removeListener('keypress', onKeypress); + + return promise.then( + (val) => { cleanup(); return val; }, + (err) => { + cleanup(); + if (escaped) return GoBack; + if (err.name === 'ExitPromptError') { + console.error(); + console.error(` ${theme.muted('Goodbye.')}`); + console.error(); + process.exit(0); + } + throw err; + }, + ); +} + // ── Pipeline status colorizer ──────────────────────────────────────────────── const STATUS_COLORS = { diff --git a/aws/cli-installer/test/unit.test.mjs b/aws/cli-installer/test/unit.test.mjs index 9fd8e532..504fa2a7 100644 --- a/aws/cli-installer/test/unit.test.mjs +++ b/aws/cli-installer/test/unit.test.mjs @@ -297,3 +297,384 @@ describe('renderPipeline', () => { assert.ok(yaml.startsWith("version: '2'")); }); }); + +// ── VPC / network topology tests ────────────────────────────────────────────── + +import { validateConfig } from '../src/cli.mjs'; + +function baseCfg(overrides = {}) { + return { + pipelineName: 'obs-stack-test', + region: 'us-east-1', + osAction: 'create', + osDomainName: 'obs-stack-test', + iamAction: 'create', + apsAction: 'create', + dashboardsAction: 'create', + vpcId: '', + subnetIds: [], + securityGroupIds: [], + ...overrides, + }; +} + +describe('validateConfig — VPC options', () => { + it('passes with no VPC options (public default)', () => { + assert.deepEqual(validateConfig(baseCfg()), []); + }); + + it('passes with a complete VPC config', () => { + const errors = validateConfig(baseCfg({ + vpcId: 'vpc-0a1b2c3d4e5f60718', + subnetIds: ['subnet-0aaaa1111bbbb2222', 'subnet-0cccc3333dddd4444'], + securityGroupIds: ['sg-0eeee5555ffff6666'], + })); + assert.deepEqual(errors, []); + }); + + it('requires subnets and SGs when a VPC is given', () => { + const errors = validateConfig(baseCfg({ vpcId: 'vpc-0a1b2c3d4e5f60718' })); + assert.ok(errors.some((e) => e.includes('--subnet-ids is required'))); + assert.ok(errors.some((e) => e.includes('--security-group-ids is required'))); + }); + + it('requires a VPC when only subnets are given', () => { + const errors = validateConfig(baseCfg({ subnetIds: ['subnet-0aaaa1111bbbb2222'] })); + assert.ok(errors.some((e) => e.includes('--vpc-id is required'))); + }); + + it('rejects malformed IDs', () => { + const errors = validateConfig(baseCfg({ + vpcId: 'notavpc', + subnetIds: ['sub-xyz'], + securityGroupIds: ['group-1'], + })); + assert.ok(errors.some((e) => e.includes('--vpc-id must look like'))); + assert.ok(errors.some((e) => e.includes('Invalid subnet ID'))); + assert.ok(errors.some((e) => e.includes('Invalid security group ID'))); + }); + + it('rejects more than 3 subnets', () => { + const errors = validateConfig(baseCfg({ + vpcId: 'vpc-0a1b2c3d4e5f60718', + subnetIds: ['subnet-1', 'subnet-2', 'subnet-3', 'subnet-4'], + securityGroupIds: ['sg-0eeee5555ffff6666'], + })); + assert.ok(errors.some((e) => e.includes('at most 3 subnets'))); + }); + + it('fails fast when VPC flags are given but no OpenSearch backend is chosen', () => { + // Advanced mode with only VPC flags: osAction stays empty. Regression guard — + // this used to skip domain creation and fail deep in pipeline creation. + const errors = validateConfig(baseCfg({ + osAction: '', + osDomainName: '', + vpcId: 'vpc-0a1b2c3d4e5f60718', + subnetIds: ['subnet-0aaaa1111bbbb2222', 'subnet-0cccc3333dddd4444'], + securityGroupIds: ['sg-0eeee5555ffff6666'], + })); + assert.ok(errors.some((e) => e.includes('No OpenSearch backend specified'))); + }); + + it('rejects VPC options when reusing an existing domain', () => { + const errors = validateConfig(baseCfg({ + osAction: 'reuse', + opensearchEndpoint: 'https://search-foo-abc.us-east-1.es.amazonaws.com', + vpcId: 'vpc-0a1b2c3d4e5f60718', + subnetIds: ['subnet-0aaaa1111bbbb2222'], + securityGroupIds: ['sg-0eeee5555ffff6666'], + })); + assert.ok(errors.some((e) => e.includes('VPC options apply only when creating'))); + }); +}); + +import { + fgacPrincipals, + validateVpcTopology, + _withRetry, + _isRoleNotPropagatedError, + _isTransientHttpError, +} from '../src/aws.mjs'; + +// ── Live VPC topology validation (EC2 API path) ─────────────────────────────── +// validateVpcTopology takes injected EC2 accessors so we can exercise every +// branch without real AWS calls. + +describe('validateVpcTopology', () => { + const cfg = { + region: 'us-east-1', + vpcId: 'vpc-aaa', + subnetIds: ['subnet-1', 'subnet-2'], + securityGroupIds: ['sg-1'], + }; + + // Happy-path accessors: everything exists, in the target VPC, distinct AZs. + function goodDeps() { + return { + describeVpcs: async () => [{ VpcId: 'vpc-aaa' }], + describeSubnets: async () => [ + { SubnetId: 'subnet-1', VpcId: 'vpc-aaa', AvailabilityZone: 'us-east-1a' }, + { SubnetId: 'subnet-2', VpcId: 'vpc-aaa', AvailabilityZone: 'us-east-1b' }, + ], + describeSecurityGroups: async () => [{ GroupId: 'sg-1', VpcId: 'vpc-aaa' }], + }; + } + + it('returns [] with no VPC configured (skips EC2 calls)', async () => { + let called = false; + const errors = await validateVpcTopology( + { region: 'us-east-1', vpcId: '', subnetIds: [], securityGroupIds: [] }, + { describeVpcs: async () => { called = true; return []; } }, + ); + assert.deepEqual(errors, []); + assert.equal(called, false); + }); + + it('passes for a valid VPC topology', async () => { + assert.deepEqual(await validateVpcTopology(cfg, goodDeps()), []); + }); + + it('reports a missing VPC and stops', async () => { + const errors = await validateVpcTopology(cfg, { + ...goodDeps(), + describeVpcs: async () => [], + }); + assert.equal(errors.length, 1); + assert.match(errors[0], /VPC vpc-aaa was not found/); + }); + + it('translates InvalidVpcID.NotFound into a clean error', async () => { + const errors = await validateVpcTopology(cfg, { + ...goodDeps(), + describeVpcs: async () => { throw new Error('InvalidVpcID.NotFound: The vpc ID does not exist'); }, + }); + assert.match(errors[0], /VPC vpc-aaa was not found/); + }); + + it('flags a subnet that belongs to a different VPC', async () => { + const errors = await validateVpcTopology(cfg, { + ...goodDeps(), + describeSubnets: async () => [ + { SubnetId: 'subnet-1', VpcId: 'vpc-aaa', AvailabilityZone: 'us-east-1a' }, + { SubnetId: 'subnet-2', VpcId: 'vpc-other', AvailabilityZone: 'us-east-1b' }, + ], + }); + assert.ok(errors.some((e) => /Subnet subnet-2 belongs to VPC vpc-other/.test(e))); + }); + + it('flags a subnet that does not exist', async () => { + const errors = await validateVpcTopology(cfg, { + ...goodDeps(), + describeSubnets: async () => [ + { SubnetId: 'subnet-1', VpcId: 'vpc-aaa', AvailabilityZone: 'us-east-1a' }, + ], + }); + assert.ok(errors.some((e) => /Subnet subnet-2 was not found/.test(e))); + }); + + it('flags two subnets sharing an Availability Zone (zone-awareness edge case)', async () => { + const errors = await validateVpcTopology(cfg, { + ...goodDeps(), + describeSubnets: async () => [ + { SubnetId: 'subnet-1', VpcId: 'vpc-aaa', AvailabilityZone: 'us-east-1a' }, + { SubnetId: 'subnet-2', VpcId: 'vpc-aaa', AvailabilityZone: 'us-east-1a' }, + ], + }); + assert.ok(errors.some((e) => /both in us-east-1a/.test(e) && /distinct Availability Zone/.test(e))); + }); + + it('does not flag AZ collisions for a single-subnet domain', async () => { + const single = { ...cfg, subnetIds: ['subnet-1'] }; + const errors = await validateVpcTopology(single, { + ...goodDeps(), + describeSubnets: async () => [ + { SubnetId: 'subnet-1', VpcId: 'vpc-aaa', AvailabilityZone: 'us-east-1a' }, + ], + }); + assert.deepEqual(errors, []); + }); + + it('flags a security group that belongs to a different VPC', async () => { + const errors = await validateVpcTopology(cfg, { + ...goodDeps(), + describeSecurityGroups: async () => [{ GroupId: 'sg-1', VpcId: 'vpc-other' }], + }); + assert.ok(errors.some((e) => /Security group sg-1 belongs to VPC vpc-other/.test(e))); + }); + + it('flags a security group that does not exist', async () => { + const errors = await validateVpcTopology(cfg, { + ...goodDeps(), + describeSecurityGroups: async () => [], + }); + assert.ok(errors.some((e) => /Security group sg-1 was not found/.test(e))); + }); + + it('accumulates multiple independent problems in one pass', async () => { + const errors = await validateVpcTopology(cfg, { + describeVpcs: async () => [{ VpcId: 'vpc-aaa' }], + describeSubnets: async () => [ + { SubnetId: 'subnet-1', VpcId: 'vpc-aaa', AvailabilityZone: 'us-east-1a' }, + { SubnetId: 'subnet-2', VpcId: 'vpc-aaa', AvailabilityZone: 'us-east-1a' }, + ], + describeSecurityGroups: async () => [{ GroupId: 'sg-1', VpcId: 'vpc-other' }], + }); + assert.ok(errors.some((e) => /both in us-east-1a/.test(e))); + assert.ok(errors.some((e) => /Security group sg-1 belongs to VPC vpc-other/.test(e))); + }); +}); + +// ── Creation ordering / race-condition guards ──────────────────────────────── + +describe('withRetry', () => { + const fast = { delayMs: 1, backoff: 1, maxDelayMs: 1 }; + + it('returns the result on first success without retrying', async () => { + let calls = 0; + const out = await _withRetry(async () => { calls++; return 'ok'; }, fast); + assert.equal(out, 'ok'); + assert.equal(calls, 1); + }); + + it('retries transient failures then succeeds', async () => { + let calls = 0; + const out = await _withRetry(async () => { + calls++; + if (calls < 3) throw new Error('ECONNREFUSED'); + return 'ok'; + }, { ...fast, attempts: 5, shouldRetry: _isTransientHttpError }); + assert.equal(out, 'ok'); + assert.equal(calls, 3); + }); + + it('stops immediately when shouldRetry returns false', async () => { + let calls = 0; + await assert.rejects( + _withRetry(async () => { calls++; throw new Error('nope'); }, { ...fast, shouldRetry: () => false }), + /nope/, + ); + assert.equal(calls, 1); + }); + + it('rethrows the last error after exhausting attempts', async () => { + let calls = 0; + await assert.rejects( + _withRetry(async () => { calls++; throw new Error('still failing'); }, { ...fast, attempts: 3 }), + /still failing/, + ); + assert.equal(calls, 3); + }); +}); + +describe('isRoleNotPropagatedError', () => { + it('matches OSIS/OpenSearch assume-role propagation errors', () => { + for (const msg of [ + 'The role arn:aws:iam::123:role/foo cannot be assumed', + 'is not authorized to perform: sts:AssumeRole', + 'role arn:aws:iam::123:role/foo does not exist', + 'Invalid PipelineRoleArn', + ]) { + assert.ok(_isRoleNotPropagatedError(new Error(msg)), `expected retry for: ${msg}`); + } + }); + + it('does not match unrelated errors', () => { + assert.ok(!_isRoleNotPropagatedError(new Error('AccessDeniedException: es:CreateDomain'))); + assert.ok(!_isRoleNotPropagatedError(new Error('quota exceeded'))); + }); +}); + +describe('isTransientHttpError', () => { + it('matches connection and gateway errors', () => { + for (const msg of ['ECONNREFUSED', 'socket hang up', 'fetch failed', 'returned 503', 'gateway timeout']) { + assert.ok(_isTransientHttpError(new Error(msg)), `expected transient for: ${msg}`); + } + }); + + it('does not match a plain 400 / auth error', () => { + assert.ok(!_isTransientHttpError(new Error('400 bad request: malformed body'))); + assert.ok(!_isTransientHttpError(new Error('ValidationException'))); + }); +}); + +describe('fgacPrincipals — FGAC role/user set', () => { + const osiRole = 'arn:aws:iam::123456789012:role/obs-stack-test-osi-role'; + + it('always includes the OSI pipeline role as a backend role', () => { + const { backendRoles, users } = fgacPrincipals({ iamRoleArn: osiRole }); + assert.deepEqual(backendRoles, [osiRole]); + assert.deepEqual(users, []); + }); + + it('adds a role-type caller principal as a backend role', () => { + const caller = { arn: 'arn:aws:iam::123456789012:role/Admin', type: 'role' }; + const { backendRoles, users } = fgacPrincipals({ iamRoleArn: osiRole, callerPrincipal: caller }); + assert.deepEqual(backendRoles, [osiRole, caller.arn]); + assert.deepEqual(users, []); + }); + + it('adds a user-type caller principal as a user, not a backend role', () => { + const caller = { arn: 'arn:aws:iam::123456789012:user/kyle', type: 'user' }; + const { backendRoles, users } = fgacPrincipals({ iamRoleArn: osiRole, callerPrincipal: caller }); + assert.deepEqual(backendRoles, [osiRole]); + assert.deepEqual(users, [caller.arn]); + }); + + it('does not duplicate the caller when it equals the OSI role', () => { + const caller = { arn: osiRole, type: 'role' }; + const { backendRoles, users } = fgacPrincipals({ iamRoleArn: osiRole, callerPrincipal: caller }); + assert.deepEqual(backendRoles, [osiRole]); + assert.deepEqual(users, []); + }); +}); + +// ── Pre-deploy architecture diagram (VPC annotations) ───────────────────────── + +import { renderArchitectureDiagram } from '../src/commands/create.mjs'; + +describe('renderArchitectureDiagram — VPC annotations', () => { + const strip = (s) => s.replace(/\x1B\[[0-9;]*m/g, ''); + const pub = () => ({ pipelineName: 'obs-stack-test', opensearchType: 'managed' }); + const vpc = () => ({ + ...pub(), + vpcId: 'vpc-0a1b2c3d', + subnetIds: ['subnet-aaaa', 'subnet-bbbb'], + securityGroupIds: ['sg-0eeee'], + }); + // Box rows carry the widths that the positional math depends on. + const boxRowWidths = (lines) => + lines.map(strip).filter((l) => /[┌└│]/.test(l)).map((l) => l.trimEnd().length); + + it('public render has no network header and no [vpc] tags', () => { + const text = renderArchitectureDiagram(pub()).map(strip).join('\n'); + assert.ok(!text.includes('[vpc]')); + assert.ok(!text.includes('Network topology')); + }); + + it('VPC render shows a header with the VPC, subnet, and SG IDs', () => { + const text = renderArchitectureDiagram(vpc()).map(strip).join('\n'); + assert.ok(text.includes('Network topology')); + assert.ok(text.includes('vpc-0a1b2c3d')); + assert.ok(text.includes('subnet-aaaa') && text.includes('subnet-bbbb')); + assert.ok(text.includes('sg-0eeee')); + }); + + it('tags exactly the private boxes (EC2, OSI endpoint, OpenSearch)', () => { + const lines = renderArchitectureDiagram(vpc()).map(strip); + const tagged = lines.filter((l) => l.includes('[vpc]') && /│/.test(l)); + // Three in-VPC boxes; Prometheus/CDS/UI are regional and stay untagged. + assert.equal(tagged.length, 3); + assert.ok(tagged.some((l) => l.includes('EC2 Instance'))); + assert.ok(tagged.some((l) => l.includes('OSI Endpoint'))); + assert.ok(tagged.some((l) => l.includes('OpenSearch') && !l.includes('OpenSearch UI'))); + // Regional services are never tagged. AWS Prometheus shares a row with the + // OpenSearch box, so check that no [vpc] appears within the Prometheus cell + // (the text at or after "AWS Prometheus"), not merely on the same line. + const promCell = (l) => l.slice(l.indexOf('AWS Prometheus')); + assert.ok(!lines.some((l) => l.includes('AWS Prometheus') && promCell(l).includes('[vpc]'))); + }); + + it('adds no width to any box vs. the public render (column math unchanged)', () => { + assert.deepEqual(boxRowWidths(renderArchitectureDiagram(vpc())), boxRowWidths(renderArchitectureDiagram(pub()))); + }); +});