diff --git a/.github/workflows/PR-test.yml b/.github/workflows/PR-test.yml index ba2a541025..3c5dd910b6 100644 --- a/.github/workflows/PR-test.yml +++ b/.github/workflows/PR-test.yml @@ -161,6 +161,9 @@ jobs: needs: [ CheckLabel, GenerateTestMatrix, OutputEnvVariables, StartLocalStack, BuildAndUpload ] if: needs.CheckLabel.outputs.should_run == 'true' uses: ./.github/workflows/ec2-integration-test.yml + permissions: + id-token: write + contents: read with: build_id: ${{ github.sha }} test_dir: terraform/ec2/linux @@ -181,6 +184,9 @@ jobs: needs: [ CheckLabel, GenerateTestMatrix, OutputEnvVariables, StartLocalStack, BuildAndUpload ] if: needs.CheckLabel.outputs.should_run == 'true' uses: ./.github/workflows/ec2-integration-test.yml + permissions: + id-token: write + contents: read with: build_id: ${{ github.sha }} test_dir: terraform/ec2/linux diff --git a/.github/workflows/ec2-integration-test.yml b/.github/workflows/ec2-integration-test.yml index 8ddd9c1530..db45a9a497 100644 --- a/.github/workflows/ec2-integration-test.yml +++ b/.github/workflows/ec2-integration-test.yml @@ -89,6 +89,8 @@ jobs: # nick-fields/retry@v2 starts at base dir - name: Terraform apply + id: terraform_apply + continue-on-error: true uses: nick-fields/retry@v2 with: max_attempts: 2 @@ -131,6 +133,21 @@ jobs: else terraform destroy -var="region=${{ inputs.region }}" -var="ami=${{ matrix.arrays.ami }}" -auto-approve && exit 1 fi + + - name: "[WIP] ✅ Passed" + if: steps.terraform_apply.outcome == 'success' && matrix.arrays.wip + run: echo "::notice::WIP test passed" + + - name: "[WIP] ⚠️ Failed (Overruled)" + if: steps.terraform_apply.outcome == 'failure' && matrix.arrays.wip + run: | + echo "::warning::WIP test failed but allowed to continue" + echo "### ⚠️ WIP Test Failed (Overruled)" >> $GITHUB_STEP_SUMMARY + + - name: Fail if not WIP + if: steps.terraform_apply.outcome == 'failure' && !matrix.arrays.wip + run: exit 1 + #This is here just in case workflow cancel - name: Terraform destroy if: ${{ cancelled() || failure() }} diff --git a/.github/workflows/test-artifacts.yml b/.github/workflows/test-artifacts.yml index 695a6ba5c9..f3a68ca97b 100644 --- a/.github/workflows/test-artifacts.yml +++ b/.github/workflows/test-artifacts.yml @@ -3,7 +3,7 @@ name: Test Artifacts env: - PRIVATE_KEY: ${{ secrets.AWS_PRIVATE_KEY }} + PRIVATE_KEY: ${{ secrets.AWS_PRIVATE_KEY }} TERRAFORM_AWS_ASSUME_ROLE: ${{ vars.TERRAFORM_AWS_ASSUME_ROLE }} TERRAFORM_AWS_ASSUME_ROLE_DURATION: 14400 # 4 hours S3_INTEGRATION_BUCKET: ${{ vars.S3_INTEGRATION_BUCKET }} @@ -39,6 +39,10 @@ on: description: 'Override for the GitHub test repository branch to use (default is main)' type: string +permissions: + id-token: write + contents: read + concurrency: group: ${{ github.workflow }}-${{ inputs.build_id }} cancel-in-progress: true @@ -57,66 +61,22 @@ jobs: CWA_GITHUB_TEST_REPO_NAME: ${{ steps.set-outputs.outputs.CWA_GITHUB_TEST_REPO_NAME }} CWA_GITHUB_TEST_REPO_URL: ${{ steps.set-outputs.outputs.CWA_GITHUB_TEST_REPO_URL }} CWA_GITHUB_TEST_REPO_BRANCH: ${{ steps.set-outputs.outputs.CWA_GITHUB_TEST_REPO_BRANCH }} - CWA_COMMIT_DATE: ${{ steps.get-commit-date.outputs.commit_date }} steps: - name: SetOutputs id: set-outputs - env: - TEST_REPO_BRANCH: ${{ inputs.test_repo_branch }} run: | - CWA_GITHUB_TEST_REPO_BRANCH="$TEST_REPO_BRANCH" + CWA_GITHUB_TEST_REPO_BRANCH=${{ inputs.test_repo_branch }} echo "CWA_GITHUB_TEST_REPO_NAME=${{ env.CWA_GITHUB_TEST_REPO_NAME }}" >> "$GITHUB_OUTPUT" echo "CWA_GITHUB_TEST_REPO_URL=${{ env.CWA_GITHUB_TEST_REPO_URL }}" >> "$GITHUB_OUTPUT" echo "CWA_GITHUB_TEST_REPO_BRANCH=${CWA_GITHUB_TEST_REPO_BRANCH:-${{ env.CWA_GITHUB_TEST_REPO_BRANCH }}}" >> "$GITHUB_OUTPUT" - - name: Checkout agent repository for commit date - uses: actions/checkout@v3 - with: - repository: aws/amazon-cloudwatch-agent - fetch-depth: 0 - path: agent-repo - - - name: Get commit date - id: get-commit-date - env: - BUILD_ID: ${{ inputs.build_id }} - run: | - cd agent-repo # Navigate to agent repo checkout - echo "Extracting commit date from agent repository..." - - # Get commit date as Unix timestamp, fallback to 0 for easier backfilling - if [[ "$BUILD_ID" =~ ^[0-9a-f]{40}$ ]]; then - # Full SHA - get date from git log - echo "Full SHA detected: $BUILD_ID" - COMMIT_DATE=$(git log -1 --format=%ct "$BUILD_ID" 2>/dev/null || echo "0") - elif [[ "$BUILD_ID" =~ ^[0-9]+\.[0-9]+\.[0-9a-f]+$ ]]; then - # Version format like 1.300057.1b1168 - extract SHA and get date - SHA_PART=$(echo "$BUILD_ID" | sed 's/.*\.//') - echo "Version format detected, extracted SHA: $SHA_PART" - COMMIT_DATE=$(git log -1 --format=%ct --grep="$SHA_PART" 2>/dev/null || echo "0") - else - # Fallback to 0 for easier backfilling of historical data - echo "Unknown format, using 0 for easier backfilling" - COMMIT_DATE="0" - fi - - echo "commit_date=${COMMIT_DATE}" >> $GITHUB_OUTPUT - echo "Retrieved commit date: ${COMMIT_DATE} for build_id: $BUILD_ID" - - if [[ "$COMMIT_DATE" != "0" ]]; then - echo "SUCCESS: Found commit date ${COMMIT_DATE}" - else - echo "Using 0 - can be backfilled later with historical data" - fi - - name: Echo test variables run: | echo "build_id: ${{ inputs.build_id }}" echo "CWA_GITHUB_TEST_REPO_NAME: ${{ steps.set-outputs.outputs.CWA_GITHUB_TEST_REPO_NAME }}" echo "CWA_GITHUB_TEST_REPO_URL: ${{ steps.set-outputs.outputs.CWA_GITHUB_TEST_REPO_URL }}" echo "CWA_GITHUB_TEST_REPO_BRANCH: ${{ steps.set-outputs.outputs.CWA_GITHUB_TEST_REPO_BRANCH }}" - echo "CWA_COMMIT_DATE: ${{ steps.get-commit-date.outputs.commit_date }}" - uses: actions/checkout@v3 with: @@ -126,30 +86,14 @@ jobs: - name: Set up Go 1.x uses: actions/setup-go@v4 with: - go-version: ~1.25 + go-version: ~1.22.2 GenerateTestMatrix: name: 'GenerateTestMatrix' needs: [OutputEnvVariables] runs-on: ubuntu-latest outputs: - ec2_gpu_matrix: ${{ steps.set-matrix.outputs.ec2_gpu_matrix }} ec2_linux_matrix: ${{ steps.set-matrix.outputs.ec2_linux_matrix }} - ec2_linux_onprem_matrix: ${{ steps.set-matrix.outputs.ec2_linux_onprem_matrix }} - ec2_selinux_matrix: ${{ steps.set-matrix.outputs.ec2_selinux_matrix }} - ec2_windows_matrix: ${{ steps.set-matrix.outputs.ec2_windows_matrix }} - ec2_mac_matrix: ${{ steps.set-matrix.outputs.ec2_mac_matrix }} - ec2_performance_matrix: ${{steps.set-matrix.outputs.ec2_performance_matrix}} - ec2_windows_performance_matrix: ${{steps.set-matrix.outputs.ec2_windows_performance_matrix}} - ec2_stress_matrix: ${{steps.set-matrix.outputs.ec2_stress_matrix}} - ec2_windows_stress_matrix: ${{steps.set-matrix.outputs.ec2_windows_stress_matrix}} - ecs_ec2_launch_daemon_matrix: ${{ steps.set-matrix.outputs.ecs_ec2_launch_daemon_matrix }} - ecs_fargate_matrix: ${{ steps.set-matrix.outputs.ecs_fargate_matrix }} - eks_daemon_matrix: ${{ steps.set-matrix.outputs.eks_daemon_matrix }} - eks_deployment_matrix: ${{ steps.set-matrix.outputs.eks_deployment_matrix }} - ec2_linux_itar_matrix: ${{ steps.set-matrix.outputs.ec2_linux_itar_matrix }} - ec2_linux_china_matrix: ${{ steps.set-matrix.outputs.ec2_linux_china_matrix }} - eks_addon_matrix: ${{ steps.set-matrix.outputs.eks_addon_matrix }} steps: - uses: actions/checkout@v3 with: @@ -159,82 +103,17 @@ jobs: - name: Set up Go 1.x uses: actions/setup-go@v4 with: - go-version: ~1.25 + go-version: ~1.22.2 - name: Generate matrix id: set-matrix run: | go run --tags=generator generator/test_case_generator.go - echo "ec2_gpu_matrix=$(echo $(cat generator/resources/ec2_gpu_complete_test_matrix.json))" >> "$GITHUB_OUTPUT" - echo "eks_addon_matrix=$(echo $(cat generator/resources/eks_addon_complete_test_matrix.json))" >> "$GITHUB_OUTPUT" echo "ec2_linux_matrix=$(echo $(cat generator/resources/ec2_linux_complete_test_matrix.json))" >> "$GITHUB_OUTPUT" - echo "ec2_linux_onprem_matrix=$(echo $(cat generator/resources/ec2_linux_onprem_complete_test_matrix.json))" >> "$GITHUB_OUTPUT" - echo "ec2_selinux_matrix=$(echo $(cat generator/resources/ec2_selinux_complete_test_matrix.json))" >> "$GITHUB_OUTPUT" - echo "ec2_windows_matrix=$(echo $(cat generator/resources/ec2_windows_complete_test_matrix.json))" >> "$GITHUB_OUTPUT" - echo "ec2_mac_matrix=$(echo $(cat generator/resources/ec2_mac_complete_test_matrix.json))" >> "$GITHUB_OUTPUT" - echo "ec2_performance_matrix=$(echo $(cat generator/resources/ec2_performance_complete_test_matrix.json))" >> "$GITHUB_OUTPUT" - echo "ec2_windows_performance_matrix=$(echo $(cat generator/resources/ec2_windows_performance_complete_test_matrix.json))" >> "$GITHUB_OUTPUT" - echo "ec2_stress_matrix=$(echo $(cat generator/resources/ec2_stress_complete_test_matrix.json))" >> "$GITHUB_OUTPUT" - echo "ec2_windows_stress_matrix=$(echo $(cat generator/resources/ec2_windows_stress_complete_test_matrix.json))" >> "$GITHUB_OUTPUT" - echo "ecs_ec2_launch_daemon_matrix=$(echo $(cat generator/resources/ecs_ec2_daemon_complete_test_matrix.json))" >> "$GITHUB_OUTPUT" - echo "ecs_fargate_matrix=$(echo $(cat generator/resources/ecs_fargate_complete_test_matrix.json))" >> "$GITHUB_OUTPUT" - echo "eks_daemon_matrix=$(echo $(cat generator/resources/eks_daemon_complete_test_matrix.json))" >> "$GITHUB_OUTPUT" - echo "eks_deployment_matrix=$(echo $(cat generator/resources/eks_deployment_complete_test_matrix.json))" >> "$GITHUB_OUTPUT" - echo "ec2_linux_itar_matrix=$(echo $(cat generator/resources/ec2_linux_itar_complete_test_matrix.json))" >> "$GITHUB_OUTPUT" - echo "ec2_linux_china_matrix=$(echo $(cat generator/resources/ec2_linux_china_complete_test_matrix.json))" >> "$GITHUB_OUTPUT" - name: Echo test plan matrix run: | - echo "ec2_gpu_matrix: ${{ steps.set-matrix.outputs.ec2_gpu_matrix }}" - echo "eks_addon_matrix: ${{ steps.set-matrix.outputs.eks_addon_matrix }}" echo "ec2_linux_matrix: ${{ steps.set-matrix.outputs.ec2_linux_matrix }}" - echo "ec2_linux_onprem_matrix: ${{ steps.set-matrix.outputs.ec2_linux_onprem_matrix }}" - echo "ec2_selinux_matrix: ${{ steps.set-matrix.outputs.ec2_selinux_matrix }}" - echo "ec2_windows_matrix: ${{ steps.set-matrix.outputs.ec2_windows_matrix }}" - echo "ec2_mac_matrix: ${{ steps.set-matrix.outputs.ec2_mac_matrix }}" - echo "ec2_performance_matrix: ${{ steps.set-matrix.outputs.ec2_performance_matrix}}" - echo "ec2_windows_performance_matrix: ${{ steps.set-matrix.outputs.ec2_windows_performance_matrix}}" - echo "ec2_stress_matrix: ${{ steps.set-matrix.outputs.ec2_stress_matrix}}" - echo "ec2_windows_stress_matrix: ${{ steps.set-matrix.outputs.ec2_windows_stress_matrix}}" - echo "ecs_ec2_launch_daemon_matrix: ${{ steps.set-matrix.outputs.ecs_ec2_launch_daemon_matrix }}" - echo "ecs_fargate_matrix: ${{ steps.set-matrix.outputs.ecs_fargate_matrix }}" - echo "eks_daemon_matrix: ${{ steps.set-matrix.outputs.eks_daemon_matrix }}" - echo "eks_deployment_matrix: ${{ steps.set-matrix.outputs.eks_deployment_matrix }}" - echo "ec2_linux_itar_matrix: ${{ steps.set-matrix.outputs.ec2_linux_itar_matrix }}" - echo "ec2_linux_china_matrix: ${{ steps.set-matrix.outputs.ec2_linux_china_matrix }}" - - CloudformationTest: - needs: [OutputEnvVariables, GenerateTestMatrix] - name: 'CFTest' - runs-on: ubuntu-latest - strategy: - fail-fast: false - permissions: - id-token: write - contents: read - steps: - - uses: actions/checkout@v3 - with: - repository: ${{ needs.OutputEnvVariables.outputs.CWA_GITHUB_TEST_REPO_NAME }} - ref: ${{ needs.OutputEnvVariables.outputs.CWA_GITHUB_TEST_REPO_BRANCH }} - path: test - - - name: Set up Go 1.x - uses: actions/setup-go@v2 - with: - go-version: ~1.25 - - - name: Configure AWS Credentials - uses: aws-actions/configure-aws-credentials@v4 - with: - role-to-assume: ${{ env.TERRAFORM_AWS_ASSUME_ROLE }} - aws-region: us-west-2 - role-duration-seconds: ${{ env.TERRAFORM_AWS_ASSUME_ROLE_DURATION }} - - - name: Test cf - run: | - cd test/test/cloudformation - go test -timeout 1h -package_path=s3://${S3_INTEGRATION_BUCKET}/integration-test/binary/${{ inputs.build_id }}/linux/amd64/amazon-cloudwatch-agent.rpm -iam_role=${CF_IAM_ROLE} -key_name=${CF_KEY_NAME} -metric_name=mem_used_percent StartLocalStack: name: 'StartLocalStack' @@ -253,178 +132,6 @@ jobs: github_sha: ${{ inputs.build_id }} s3_integration_bucket: ${{ vars.S3_INTEGRATION_BUCKET }} - StartLocalStackITAR: - name: 'StartLocalStackITAR' - needs: [OutputEnvVariables] - uses: ./.github/workflows/start-localstack.yml - secrets: - AWS_PRIVATE_KEY: ${{ secrets.AWS_PRIVATE_KEY_ITAR }} - KEY_NAME: ${{ secrets.KEY_NAME }} - permissions: - id-token: write - contents: read - with: - region: us-gov-east-1 - test_repo_name: ${{ needs.OutputEnvVariables.outputs.CWA_GITHUB_TEST_REPO_NAME }} - test_repo_branch: ${{ needs.OutputEnvVariables.outputs.CWA_GITHUB_TEST_REPO_BRANCH }} - terraform_assume_role: ${{ vars.TERRAFORM_AWS_ASSUME_ROLE_ITAR }} - test_repo_url: ${{ needs.OutputEnvVariables.outputs.CWA_GITHUB_TEST_REPO_URL }} - github_sha: ${{ inputs.build_id }} - s3_integration_bucket: ${{ vars.S3_INTEGRATION_BUCKET_ITAR }} - - StartLocalStackCN: - name: 'StartLocalStackCN' - needs: [ OutputEnvVariables, UploadDependenciesCN ] - uses: ./.github/workflows/start-localstack.yml - secrets: - AWS_PRIVATE_KEY: ${{ secrets.AWS_PRIVATE_KEY_CN }} - KEY_NAME: ${{ secrets.KEY_NAME }} - permissions: - id-token: write - contents: read - with: - region: cn-north-1 - test_repo_name: ${{ needs.OutputEnvVariables.outputs.CWA_GITHUB_TEST_REPO_NAME }} - test_repo_branch: ${{ needs.OutputEnvVariables.outputs.CWA_GITHUB_TEST_REPO_BRANCH }} - terraform_assume_role: ${{ vars.TERRAFORM_AWS_ASSUME_ROLE_CN }} - test_repo_url: ${{ needs.OutputEnvVariables.outputs.CWA_GITHUB_TEST_REPO_URL }} - github_sha: ${{ inputs.build_id }} - s3_integration_bucket: ${{ vars.S3_INTEGRATION_BUCKET_CN }} - - UploadDependenciesCN: - name: 'UploadDependenciesCN' - needs: [ OutputEnvVariables ] - uses: ./.github/workflows/upload-dependencies.yml - secrets: inherit - permissions: - id-token: write - contents: read - with: - region: cn-north-1 - test_repo_name: ${{ needs.OutputEnvVariables.outputs.CWA_GITHUB_TEST_REPO_NAME }} - test_repo_branch: ${{ needs.OutputEnvVariables.outputs.CWA_GITHUB_TEST_REPO_BRANCH }} - github_sha: ${{github.sha}} - s3_integration_bucket: ${{ vars.S3_INTEGRATION_BUCKET_CN }} - terraform_assume_role: ${{ vars.TERRAFORM_AWS_ASSUME_ROLE_CN }} - - EC2NvidiaGPUIntegrationTest: - needs: [ StartLocalStack, GenerateTestMatrix, OutputEnvVariables ] - name: ${{matrix.arrays.testName}} - runs-on: ubuntu-latest - strategy: - fail-fast: false - matrix: - arrays: ${{ fromJson(needs.GenerateTestMatrix.outputs.ec2_gpu_matrix) }} - permissions: - id-token: write - contents: read - steps: - - uses: actions/checkout@v3 - with: - repository: ${{ needs.OutputEnvVariables.outputs.CWA_GITHUB_TEST_REPO_NAME }} - ref: ${{ needs.OutputEnvVariables.outputs.CWA_GITHUB_TEST_REPO_BRANCH }} - - - name: Configure AWS Credentials - uses: aws-actions/configure-aws-credentials@v4 - with: - role-to-assume: ${{ env.TERRAFORM_AWS_ASSUME_ROLE }} - aws-region: us-west-2 - role-duration-seconds: ${{ env.TERRAFORM_AWS_ASSUME_ROLE_DURATION }} - - - name: Echo Test Info - run: echo run on ec2 instance os ${{ matrix.arrays.os }} arc ${{ matrix.arrays.arc }} test dir ${{ matrix.arrays.test_dir }} - - - name: Install Terraform - uses: hashicorp/setup-terraform@v3 - with: - terraform_version: 1.12.0 - - - name: Verify Terraform version - run: terraform --version - - # nick-fields/retry@v2 starts at base dir - - name: Terraform apply - if: ${{ matrix.arrays.family == 'linux' }} - uses: nick-fields/retry@v2 - with: - max_attempts: 3 - timeout_minutes: 30 - retry_wait_seconds: 5 - command: | - if [ "${{ matrix.arrays.terraform_dir }}" != "" ]; then - cd "${{ matrix.arrays.terraform_dir }}" - else - cd terraform/ec2/linux - fi - - terraform init - if terraform apply --auto-approve \ - -var="ssh_key_value=${PRIVATE_KEY}" \ - -var="github_test_repo=${{env.CWA_GITHUB_TEST_REPO_URL}}" \ - -var="test_name=${{ matrix.arrays.os }}" \ - -var="cwa_github_sha=${{ inputs.build_id }}" \ - -var="install_agent=${{ matrix.arrays.installAgentCommand }}" \ - -var="github_test_repo_branch=${{ needs.OutputEnvVariables.outputs.CWA_GITHUB_TEST_REPO_BRANCH }}" \ - -var="ec2_instance_type=${{ matrix.arrays.instanceType }}" \ - -var="user=${{ matrix.arrays.username }}" \ - -var="ami=${{ matrix.arrays.ami }}" \ - -var="ca_cert_path=${{ matrix.arrays.caCertPath }}" \ - -var="arc=${{ matrix.arrays.arc }}" \ - -var="binary_name=${{ matrix.arrays.binaryName }}" \ - -var="local_stack_host_name=${{ needs.StartLocalStack.outputs.local_stack_host_name }}" \ - -var="s3_bucket=${S3_INTEGRATION_BUCKET}" \ - -var="ssh_key_name=${KEY_NAME}" \ - -var="test_dir=${{ matrix.arrays.test_dir }}" ; then terraform destroy -auto-approve - else - terraform destroy -auto-approve && exit 1 - fi - - name: Terraform apply - if: ${{ matrix.arrays.family == 'window' }} - uses: nick-fields/retry@v2 - with: - max_attempts: 3 - timeout_minutes: 30 - retry_wait_seconds: 5 - command: | - if [ "${{ matrix.arrays.terraform_dir }}" != "" ]; then - cd "${{ matrix.arrays.terraform_dir }}" - else - cd terraform/ec2/win - fi - - terraform init - if terraform apply --auto-approve \ - -var="ssh_key_value=${PRIVATE_KEY}" \ - -var="ssh_key_name=${KEY_NAME}" \ - -var="github_repo=${{env.CWA_GITHUB_TEST_REPO_URL}}" \ - -var="cwa_github_sha=${{ inputs.build_id }}" \ - -var="ami=${{ matrix.arrays.ami }}" \ - -var="test_dir=${{ matrix.arrays.test_dir }}" \ - -var="ec2_instance_type=${{ matrix.arrays.instanceType }}" \ - -var="github_test_repo=${{env.CWA_GITHUB_TEST_REPO_URL}}" \ - -var="github_test_repo_branch=${{ needs.OutputEnvVariables.outputs.CWA_GITHUB_TEST_REPO_BRANCH }}" \ - -var="s3_bucket=${S3_INTEGRATION_BUCKET}" ; then terraform destroy -auto-approve - else - terraform destroy -auto-approve && exit 1 - fi - #This is here just in case workflow cancel - - name: Terraform destroy - if: ${{ cancelled() || failure() }} - uses: nick-fields/retry@v2 - with: - max_attempts: 3 - timeout_minutes: 8 - retry_wait_seconds: 5 - command: | - if [ "${{ matrix.arrays.terraform_dir }}" != "" ]; then - cd "${{ matrix.arrays.terraform_dir }}" - elif if "${{ matrix.arrays.os }}" == window; then - cd terraform/ec2/win - else - cd terraform/ec2/linux - fi - terraform destroy --auto-approve - EC2LinuxIntegrationTest: needs: [ StartLocalStack, GenerateTestMatrix, OutputEnvVariables ] name: 'EC2Linux' @@ -442,247 +149,8 @@ jobs: terraform_assume_role: ${{ vars.TERRAFORM_AWS_ASSUME_ROLE }} s3_integration_bucket: ${{ vars.S3_INTEGRATION_BUCKET }} is_selinux_test: false - is_onprem_test: false - secrets: inherit - - LinuxOnPremIntegrationTest: - needs: [GenerateTestMatrix, OutputEnvVariables] - name: 'OnpremLinux' - uses: ./.github/workflows/ec2-integration-test.yml - with: - build_id: ${{ inputs.build_id }} - test_dir: terraform/ec2/linux - job_id: linux-onprem-integration-test - test_props: ${{needs.GenerateTestMatrix.outputs.ec2_linux_onprem_matrix}} - test_repo_name: ${{ needs.OutputEnvVariables.outputs.CWA_GITHUB_TEST_REPO_NAME }} - test_repo_url: ${{ needs.OutputEnvVariables.outputs.CWA_GITHUB_TEST_REPO_URL }} - test_repo_branch: ${{ needs.OutputEnvVariables.outputs.CWA_GITHUB_TEST_REPO_BRANCH }} - localstack_host: ${{needs.StartLocalStack.outputs.local_stack_host_name}} - region: us-west-2 - terraform_assume_role: ${{ vars.TERRAFORM_AWS_ASSUME_ROLE }} - s3_integration_bucket: ${{ vars.S3_INTEGRATION_BUCKET }} - is_selinux_test: false - is_onprem_test: true secrets: inherit - - EC2LinuxIntegrationTestITAR: - needs: [ StartLocalStackITAR, GenerateTestMatrix, OutputEnvVariables ] - name: 'EC2LinuxITAR' - uses: ./.github/workflows/ec2-integration-test.yml - with: - build_id: ${{ inputs.build_id }} - test_dir: terraform/ec2/linux - job_id: ec2-linux-integration-test - test_props: ${{needs.GenerateTestMatrix.outputs.ec2_linux_itar_matrix}} - test_repo_name: ${{ needs.OutputEnvVariables.outputs.CWA_GITHUB_TEST_REPO_NAME }} - test_repo_url: ${{ needs.OutputEnvVariables.outputs.CWA_GITHUB_TEST_REPO_URL }} - test_repo_branch: ${{ needs.OutputEnvVariables.outputs.CWA_GITHUB_TEST_REPO_BRANCH }} - localstack_host: ${{needs.StartLocalStackITAR.outputs.local_stack_host_name}} - region: us-gov-east-1 - terraform_assume_role: ${{ vars.TERRAFORM_AWS_ASSUME_ROLE_ITAR }} - s3_integration_bucket: ${{ vars.S3_INTEGRATION_BUCKET_ITAR }} - secrets: - AWS_PRIVATE_KEY: ${{ secrets.AWS_PRIVATE_KEY_ITAR }} - KEY_NAME: ${{ secrets.KEY_NAME }} - - EC2LinuxIntegrationTestCN: - needs: [ StartLocalStackCN, GenerateTestMatrix, OutputEnvVariables ] - name: 'EC2LinuxCN' - uses: ./.github/workflows/ec2-integration-test.yml - with: - build_id: ${{ inputs.build_id }} - test_dir: terraform/ec2/linux - job_id: ec2-linux-integration-test - test_props: ${{needs.GenerateTestMatrix.outputs.ec2_linux_china_matrix}} - test_repo_name: ${{ needs.OutputEnvVariables.outputs.CWA_GITHUB_TEST_REPO_NAME }} - test_repo_url: ${{ needs.OutputEnvVariables.outputs.CWA_GITHUB_TEST_REPO_URL }} - test_repo_branch: ${{ needs.OutputEnvVariables.outputs.CWA_GITHUB_TEST_REPO_BRANCH }} - localstack_host: ${{needs.StartLocalStackCN.outputs.local_stack_host_name}} - region: cn-north-1 - terraform_assume_role: ${{ vars.TERRAFORM_AWS_ASSUME_ROLE_CN }} - s3_integration_bucket: ${{ vars.S3_INTEGRATION_BUCKET_CN }} - secrets: - AWS_PRIVATE_KEY: ${{ secrets.AWS_PRIVATE_KEY_CN }} - KEY_NAME: ${{ secrets.KEY_NAME }} - - EC2SELinuxIntegrationTest: - needs: [ StartLocalStack, GenerateTestMatrix, OutputEnvVariables ] - name: 'EC2SELinux' - uses: ./.github/workflows/ec2-integration-test.yml - with: - build_id: ${{ inputs.build_id }} - test_dir: terraform/ec2/linux - job_id: ec2-linux-integration-test - test_props: ${{needs.GenerateTestMatrix.outputs.ec2_selinux_matrix}} - test_repo_name: ${{ needs.OutputEnvVariables.outputs.CWA_GITHUB_TEST_REPO_NAME }} - test_repo_url: ${{ needs.OutputEnvVariables.outputs.CWA_GITHUB_TEST_REPO_URL }} - test_repo_branch: ${{ needs.OutputEnvVariables.outputs.CWA_GITHUB_TEST_REPO_BRANCH }} - localstack_host: ${{needs.StartLocalStack.outputs.local_stack_host_name}} - region: us-west-2 - terraform_assume_role: ${{ vars.TERRAFORM_AWS_ASSUME_ROLE }} - s3_integration_bucket: ${{ vars.S3_INTEGRATION_BUCKET }} - is_selinux_test: true - - EC2WinIntegrationTest: - needs: [OutputEnvVariables, GenerateTestMatrix] - name: ${{matrix.arrays.testName}} - runs-on: ubuntu-latest - strategy: - fail-fast: false - matrix: - arrays: ${{ fromJson(needs.GenerateTestMatrix.outputs.ec2_windows_matrix) }} - permissions: - id-token: write - contents: read - steps: - - uses: actions/checkout@v3 - with: - repository: ${{ needs.OutputEnvVariables.outputs.CWA_GITHUB_TEST_REPO_NAME }} - ref: ${{ needs.OutputEnvVariables.outputs.CWA_GITHUB_TEST_REPO_BRANCH }} - - - name: Configure AWS Credentials - uses: aws-actions/configure-aws-credentials@v4 - with: - role-to-assume: ${{ env.TERRAFORM_AWS_ASSUME_ROLE }} - aws-region: us-west-2 - role-duration-seconds: ${{ env.TERRAFORM_AWS_ASSUME_ROLE_DURATION }} - - - name: Echo Test Info - run: echo run on ec2 instance os ${{ matrix.arrays.os }} use ssm ${{ matrix.arrays.useSSM }} test ${{ matrix.arrays.test_dir }} - - - name: Install Terraform - uses: hashicorp/setup-terraform@v3 - with: - terraform_version: 1.12.0 - - - name: Verify Terraform version - run: terraform --version - - # nick-fields/retry@v2 starts at base dir - - name: Terraform apply - uses: nick-fields/retry@v2 - with: - max_attempts: 3 - timeout_minutes: 60 - retry_wait_seconds: 5 - command: | - if [ "${{ matrix.arrays.terraform_dir }}" != "" ]; then - cd "${{ matrix.arrays.terraform_dir }}" - else - cd terraform/ec2/win - fi - - terraform init - if terraform apply --auto-approve \ - -var="ec2_instance_type=${{ matrix.arrays.instanceType }}" \ - -var="ssh_key_value=${PRIVATE_KEY}" \ - -var="ssh_key_name=${KEY_NAME}" \ - -var="test_name=${{ matrix.arrays.os }}" \ - -var="cwa_github_sha=${{ inputs.build_id }}" \ - -var="test_dir=${{ matrix.arrays.test_dir }}" \ - -var="ami=${{ matrix.arrays.ami }}" \ - -var="use_ssm=${{ matrix.arrays.useSSM }}" \ - -var="s3_bucket=${S3_INTEGRATION_BUCKET}" ; then - terraform destroy -auto-approve - else - terraform destroy -auto-approve && exit 1 - fi - #This is here just in case workflow cancel - - name: Terraform destroy - if: ${{ cancelled() || failure() }} - uses: nick-fields/retry@v2 - with: - max_attempts: 3 - timeout_minutes: 8 - retry_wait_seconds: 5 - command: | - if [ "${{ matrix.arrays.terraform_dir }}" != "" ]; then - cd "${{ matrix.arrays.terraform_dir }}" - else - cd terraform/ec2/win - fi - terraform destroy --auto-approve - EC2DarwinIntegrationTest: - needs: [GenerateTestMatrix, OutputEnvVariables] - name: ${{matrix.arrays.testName}} - runs-on: ubuntu-latest - strategy: - fail-fast: false - matrix: - arrays: ${{ fromJson(needs.GenerateTestMatrix.outputs.ec2_mac_matrix) }} - permissions: - id-token: write - contents: read - steps: - - uses: actions/checkout@v3 - with: - repository: ${{ needs.OutputEnvVariables.outputs.CWA_GITHUB_TEST_REPO_NAME }} - ref: ${{ needs.OutputEnvVariables.outputs.CWA_GITHUB_TEST_REPO_BRANCH }} - - - name: Configure AWS Credentials - uses: aws-actions/configure-aws-credentials@v4 - with: - role-to-assume: ${{ env.TERRAFORM_AWS_ASSUME_ROLE }} - aws-region: us-west-2 - role-duration-seconds: ${{ env.TERRAFORM_AWS_ASSUME_ROLE_DURATION }} - - - name: Echo OS - run: echo run on ec2 instance os ${{ matrix.arrays.os }} - - - name: Install Terraform - uses: hashicorp/setup-terraform@v3 - with: - terraform_version: 1.12.0 - - - name: Verify Terraform version - run: terraform --version - - # nick-fields/retry@v2 starts at base dir - - name: Terraform apply - uses: nick-fields/retry@v2 - with: - max_attempts: 3 - timeout_minutes: 60 - retry_wait_seconds: 5 - command: | - if [ "${{ matrix.arrays.terraform_dir }}" != "" ]; then - cd "${{ matrix.arrays.terraform_dir }}" - else - cd terraform/ec2/mac - fi - - terraform init - if terraform apply --auto-approve \ - -var="ssh_key_value=${PRIVATE_KEY}" \ - -var="ssh_key_name=${KEY_NAME}" \ - -var="arc=${{ matrix.arrays.arc }}" \ - -var="ec2_instance_type=${{ matrix.arrays.instanceType }}" \ - -var="cwa_github_sha=${{ inputs.build_id }}" \ - -var="ami=${{ matrix.arrays.ami }}" \ - -var="test_dir=${{ matrix.arrays.test_dir }}" \ - -var="license_manager_arn=${{ env.LICENSE_MANAGER_ARN }}" \ - -var="s3_bucket=${S3_INTEGRATION_BUCKET}"; then - terraform destroy -auto-approve - else - terraform destroy -auto-approve && exit 1 - fi - #This is here just in case workflow cancel - - name: Terraform destroy - if: ${{ cancelled() || failure() }} - uses: nick-fields/retry@v2 - with: - max_attempts: 3 - timeout_minutes: 8 - retry_wait_seconds: 5 - command: | - if [ "${{ matrix.arrays.terraform_dir }}" != "" ]; then - cd "${{ matrix.arrays.terraform_dir }}" - else - cd terraform/ec2/mac - fi - terraform destroy --auto-approve - StopLocalStack: name: 'StopLocalStack' if: ${{ always() && needs.StartLocalStack.result == 'success' }} @@ -699,679 +167,3 @@ jobs: terraform_assume_role: ${{ vars.TERRAFORM_AWS_ASSUME_ROLE }} github_sha: ${{inputs.build_id}} s3_integration_bucket: ${{ vars.S3_INTEGRATION_BUCKET }} - - StopLocalStackITAR: - name: 'StopLocalStackITAR' - if: ${{ always() && needs.StartLocalStackITAR.result == 'success' }} - needs: [ StartLocalStackITAR, EC2LinuxIntegrationTestITAR, OutputEnvVariables ] - uses: ./.github/workflows/stop-localstack.yml - secrets: - AWS_PRIVATE_KEY: ${{ secrets.AWS_PRIVATE_KEY_ITAR }} - KEY_NAME: ${{ secrets.KEY_NAME }} - permissions: - id-token: write - contents: read - with: - region: us-gov-east-1 - test_repo_name: ${{ needs.OutputEnvVariables.outputs.CWA_GITHUB_TEST_REPO_NAME }} - test_repo_branch: ${{ needs.OutputEnvVariables.outputs.CWA_GITHUB_TEST_REPO_BRANCH }} - terraform_assume_role: ${{ vars.TERRAFORM_AWS_ASSUME_ROLE_ITAR }} - github_sha: ${{inputs.build_id}} - s3_integration_bucket: ${{ vars.S3_INTEGRATION_BUCKET_ITAR }} - - StopLocalStackCN: - name: 'StopLocalStackCN' - if: ${{ always() && needs.StartLocalStackCN.result == 'success' }} - needs: [ StartLocalStackCN, EC2LinuxIntegrationTestCN, OutputEnvVariables] - uses: ./.github/workflows/stop-localstack.yml - secrets: - AWS_PRIVATE_KEY: ${{ secrets.AWS_PRIVATE_KEY_CN }} - KEY_NAME: ${{ secrets.KEY_NAME }} - permissions: - id-token: write - contents: read - with: - region: cn-north-1 - test_repo_name: ${{ needs.OutputEnvVariables.outputs.CWA_GITHUB_TEST_REPO_NAME }} - test_repo_branch: ${{ needs.OutputEnvVariables.outputs.CWA_GITHUB_TEST_REPO_BRANCH }} - terraform_assume_role: ${{ vars.TERRAFORM_AWS_ASSUME_ROLE_CN }} - github_sha: ${{inputs.build_id}} - s3_integration_bucket: ${{ vars.S3_INTEGRATION_BUCKET_CN }} - - ECSEC2IntegrationTest: - name: ${{matrix.arrays.testName}} - runs-on: ubuntu-latest - needs: [ GenerateTestMatrix, OutputEnvVariables ] - strategy: - fail-fast: false - matrix: - arrays: ${{ fromJson(needs.GenerateTestMatrix.outputs.ecs_ec2_launch_daemon_matrix) }} - permissions: - id-token: write - contents: read - steps: - - uses: actions/checkout@v3 - with: - repository: ${{ needs.OutputEnvVariables.outputs.CWA_GITHUB_TEST_REPO_NAME }} - ref: ${{ needs.OutputEnvVariables.outputs.CWA_GITHUB_TEST_REPO_BRANCH }} - - - name: Configure AWS Credentials - uses: aws-actions/configure-aws-credentials@v4 - with: - role-to-assume: ${{ env.TERRAFORM_AWS_ASSUME_ROLE }} - aws-region: us-west-2 - role-duration-seconds: ${{ env.TERRAFORM_AWS_ASSUME_ROLE_DURATION }} - - - name: Login ECR - id: login-ecr - uses: aws-actions/amazon-ecr-login@v2 - - - name: Install Terraform - uses: hashicorp/setup-terraform@v3 - with: - terraform_version: 1.12.0 - - - name: Verify Terraform version - run: terraform --version - - - name: Terraform apply - uses: nick-fields/retry@v2 - with: - max_attempts: 3 - timeout_minutes: 30 - retry_wait_seconds: 5 - command: | - if [ "${{ matrix.arrays.terraform_dir }}" != "" ]; then - cd "${{ matrix.arrays.terraform_dir }}" - else - cd terraform/ecs_ec2/daemon - fi - - terraform init - if terraform apply --auto-approve\ - -var="test_dir=${{ matrix.arrays.test_dir }}"\ - -var="ec2_instance_type=${{ matrix.arrays.instanceType }}" \ - -var="cwagent_image_repo=${{ steps.login-ecr.outputs.registry }}/${{ env.ECR_INTEGRATION_TEST_REPO }}"\ - -var="cwagent_image_tag=${{ inputs.build_id }}"\ - -var="ec2_instance_type=${{ matrix.arrays.instanceType }}" \ - -var="metadataEnabled=${{ matrix.arrays.metadataEnabled }}" \ - -var="ami=${{ matrix.arrays.ami }}" ; then - terraform destroy -auto-approve - else - terraform destroy -auto-approve && exit 1 - fi - - - name: Terraform destroy - if: ${{ cancelled() || failure() }} - uses: nick-fields/retry@v2 - with: - max_attempts: 3 - timeout_minutes: 8 - retry_wait_seconds: 5 - command: | - if [ "${{ matrix.arrays.terraform_dir }}" != "" ]; then - cd "${{ matrix.arrays.terraform_dir }}" - else - cd terraform/ecs_ec2/daemon - fi - terraform destroy --auto-approve - - ECSFargateIntegrationTest: - name: ${{matrix.arrays.testName}} - runs-on: ubuntu-latest - needs: [ GenerateTestMatrix, OutputEnvVariables ] - strategy: - fail-fast: false - matrix: - arrays: ${{ fromJson(needs.GenerateTestMatrix.outputs.ecs_fargate_matrix) }} - permissions: - id-token: write - contents: read - steps: - - uses: actions/checkout@v3 - with: - repository: ${{ needs.OutputEnvVariables.outputs.CWA_GITHUB_TEST_REPO_NAME }} - ref: ${{ needs.OutputEnvVariables.outputs.CWA_GITHUB_TEST_REPO_BRANCH }} - - - name: Configure AWS Credentials - uses: aws-actions/configure-aws-credentials@v4 - with: - role-to-assume: ${{ env.TERRAFORM_AWS_ASSUME_ROLE }} - aws-region: us-west-2 - role-duration-seconds: ${{ env.TERRAFORM_AWS_ASSUME_ROLE_DURATION }} - - - name: Login ECR - id: login-ecr - uses: aws-actions/amazon-ecr-login@v2 - - - name: Install Terraform - uses: hashicorp/setup-terraform@v3 - with: - terraform_version: 1.12.0 - - - name: Verify Terraform version - run: terraform --version - - - name: Terraform apply - uses: nick-fields/retry@v2 - with: - max_attempts: 3 - timeout_minutes: 30 - retry_wait_seconds: 5 - command: | - if [ "${{ matrix.arrays.terraform_dir }}" != "" ]; then - cd "${{ matrix.arrays.terraform_dir }}" - else - cd terraform/ecs_fargate/linux - fi - - terraform init - if terraform apply --auto-approve\ - -var="test_dir=${{ matrix.arrays.test_dir }}"\ - -var="cwagent_image_repo=${{ steps.login-ecr.outputs.registry }}/${{ env.ECR_INTEGRATION_TEST_REPO }}"\ - -var="cwagent_image_tag=${{ inputs.build_id }}"; then - terraform destroy -auto-approve - else - terraform destroy -auto-approve && exit 1 - fi - - - name: Terraform destroy - if: ${{ cancelled() || failure() }} - uses: nick-fields/retry@v2 - with: - max_attempts: 3 - timeout_minutes: 8 - retry_wait_seconds: 5 - command: | - if [ "${{ matrix.arrays.terraform_dir }}" != "" ]; then - cd "${{ matrix.arrays.terraform_dir }}" - else - cd terraform/ecs_fargate/linux - fi - terraform destroy --auto-approve - - EKSIntegrationTest: - name: ${{matrix.arrays.testName}} - runs-on: ubuntu-latest - needs: [ GenerateTestMatrix, OutputEnvVariables ] - strategy: - fail-fast: false - matrix: - arrays: ${{ fromJson(needs.GenerateTestMatrix.outputs.eks_daemon_matrix) }} - permissions: - id-token: write - contents: read - steps: - - uses: actions/checkout@v3 - with: - repository: ${{ needs.OutputEnvVariables.outputs.CWA_GITHUB_TEST_REPO_NAME }} - ref: ${{ needs.OutputEnvVariables.outputs.CWA_GITHUB_TEST_REPO_BRANCH }} - - - name: Configure AWS Credentials - uses: aws-actions/configure-aws-credentials@v4 - with: - role-to-assume: ${{ env.TERRAFORM_AWS_ASSUME_ROLE }} - aws-region: us-west-2 - role-duration-seconds: ${{ env.TERRAFORM_AWS_ASSUME_ROLE_DURATION }} - - - name: Login ECR - id: login-ecr - uses: aws-actions/amazon-ecr-login@v2 - - - name: Install Terraform - uses: hashicorp/setup-terraform@v3 - with: - terraform_version: 1.12.0 - - - name: Verify Terraform version - run: terraform --version - - - name: Terraform apply - uses: nick-fields/retry@v2 - with: - max_attempts: 2 - timeout_minutes: 90 # EKS takes about 20 minutes to spin up a cluster and service on the cluster - retry_wait_seconds: 5 - command: | - if [ "${{ matrix.arrays.terraform_dir }}" != "" ]; then - cd "${{ matrix.arrays.terraform_dir }}" - else - cd terraform/eks/daemon - fi - - terraform init - if terraform apply --auto-approve \ - -var="test_dir=${{ matrix.arrays.test_dir }}"\ - -var="cwagent_image_repo=${{ steps.login-ecr.outputs.registry }}/${{ env.ECR_INTEGRATION_TEST_REPO }}" \ - -var="cwagent_image_tag=${{ inputs.build_id }}" \ - -var="ami_type=${{ matrix.arrays.ami }}" \ - -var="instance_type=${{ matrix.arrays.instanceType }}" \ - -var="k8s_version=${{ matrix.arrays.k8sVersion }}"; then - terraform destroy -auto-approve - else - terraform destroy -auto-approve && exit 1 - fi - - - name: Terraform destroy - if: ${{ cancelled() || failure() }} - uses: nick-fields/retry@v2 - with: - max_attempts: 3 - timeout_minutes: 8 - retry_wait_seconds: 5 - command: | - if [ "${{ matrix.arrays.terraform_dir }}" != "" ]; then - cd "${{ matrix.arrays.terraform_dir }}" - else - cd terraform/eks/daemon - fi - terraform destroy --auto-approve - - EKSPrometheusIntegrationTest: - name: ${{matrix.arrays.testName}} - runs-on: ubuntu-latest - needs: [ GenerateTestMatrix, OutputEnvVariables ] - strategy: - fail-fast: false - matrix: - arrays: ${{ fromJson(needs.GenerateTestMatrix.outputs.eks_deployment_matrix) }} - permissions: - id-token: write - contents: read - steps: - - uses: actions/checkout@v3 - with: - repository: ${{ needs.OutputEnvVariables.outputs.CWA_GITHUB_TEST_REPO_NAME }} - ref: ${{ needs.OutputEnvVariables.outputs.CWA_GITHUB_TEST_REPO_BRANCH }} - - - name: Configure AWS Credentials - uses: aws-actions/configure-aws-credentials@v4 - with: - role-to-assume: ${{ env.TERRAFORM_AWS_ASSUME_ROLE }} - aws-region: us-west-2 - role-duration-seconds: ${{ env.TERRAFORM_AWS_ASSUME_ROLE_DURATION }} - - - name: Login ECR - id: login-ecr - uses: aws-actions/amazon-ecr-login@v2 - - - name: Install Terraform - uses: hashicorp/setup-terraform@v3 - with: - terraform_version: 1.12.0 - - - name: Verify Terraform version - run: terraform --version - - - name: Terraform apply - uses: nick-fields/retry@v2 - with: - max_attempts: 3 - timeout_minutes: 60 # EKS takes about 20 minutes to spin up a cluster and service on the cluster - retry_wait_seconds: 5 - command: | - if [ "${{ matrix.arrays.terraform_dir }}" != "" ]; then - cd "${{ matrix.arrays.terraform_dir }}" - else - cd terraform/eks/deployment - fi - - terraform init - if terraform apply --auto-approve \ - -var="test_dir=${{ matrix.arrays.test_dir }}"\ - -var="cwagent_image_repo=${{ steps.login-ecr.outputs.registry }}/${{ env.ECR_INTEGRATION_TEST_REPO }}" \ - -var="cwagent_image_tag=${{ inputs.build_id }}" \ - -var="k8s_version=${{ matrix.arrays.k8sVersion }}"; then - terraform destroy -auto-approve - else - terraform destroy -auto-approve && exit 1 - fi - - name: Terraform destroy - if: ${{ cancelled() || failure() }} - uses: nick-fields/retry@v2 - with: - max_attempts: 3 - timeout_minutes: 8 - retry_wait_seconds: 5 - command: | - if [ "${{ matrix.arrays.terraform_dir }}" != "" ]; then - cd "${{ matrix.arrays.terraform_dir }}" - else - cd terraform/eks/deployment - fi - terraform destroy --auto-approve - - PerformanceTrackingTest: - name: ${{matrix.arrays.testName}} - needs: [ GenerateTestMatrix, OutputEnvVariables ] - runs-on: ubuntu-latest - strategy: - fail-fast: false - matrix: - arrays: ${{ fromJson(needs.GenerateTestMatrix.outputs.ec2_performance_matrix) }} - permissions: - id-token: write - contents: read - steps: - - uses: actions/checkout@v3 - with: - repository: ${{ needs.OutputEnvVariables.outputs.CWA_GITHUB_TEST_REPO_NAME }} - ref: ${{ needs.OutputEnvVariables.outputs.CWA_GITHUB_TEST_REPO_BRANCH }} - - - name: Configure AWS Credentials - uses: aws-actions/configure-aws-credentials@v4 - with: - role-to-assume: ${{ env.TERRAFORM_AWS_ASSUME_ROLE }} - aws-region: us-west-2 - role-duration-seconds: ${{ env.TERRAFORM_AWS_ASSUME_ROLE_DURATION }} - - - name: Install Terraform - uses: hashicorp/setup-terraform@v3 - with: - terraform_version: 1.12.0 - - - name: Verify Terraform version - run: terraform --version - - - name: Terraform apply - uses: nick-fields/retry@v2 - with: - max_attempts: 1 - timeout_minutes: 60 - retry_wait_seconds: 5 - command: | - cd terraform/performance - terraform init - if terraform apply --auto-approve \ - -var="ssh_key_value=${PRIVATE_KEY}" \ - -var="cwa_github_sha=${{ inputs.build_id }}" \ - -var="cwa_github_sha_date=${{ needs.OutputEnvVariables.outputs.CWA_COMMIT_DATE }}" \ - -var="ami=${{ matrix.arrays.ami }}" \ - -var="arc=${{ matrix.arrays.arc }}" \ - -var="s3_bucket=${S3_INTEGRATION_BUCKET}" \ - -var="ssh_key_name=${KEY_NAME}" \ - -var="values_per_minute=${{ matrix.arrays.values_per_minute}}"\ - -var="family=${{ matrix.arrays.family}}"\ - -var="test_dir=${{ matrix.arrays.test_dir }}" ; then terraform destroy -auto-approve - else - terraform destroy -auto-approve && exit 1 - fi - - - name: Terraform destroy - if: ${{ cancelled() || failure() }} - uses: nick-fields/retry@v2 - with: - max_attempts: 3 - timeout_minutes: 8 - retry_wait_seconds: 5 - command: cd terraform/performance && terraform destroy --auto-approve - - EC2WinPerformanceTest: - name: ${{matrix.arrays.testName}} - needs: [ GenerateTestMatrix, OutputEnvVariables ] - runs-on: ubuntu-latest - strategy: - fail-fast: false - matrix: - arrays: ${{ fromJson(needs.GenerateTestMatrix.outputs.ec2_windows_performance_matrix) }} - permissions: - id-token: write - contents: read - steps: - - uses: actions/checkout@v3 - with: - repository: ${{ needs.OutputEnvVariables.outputs.CWA_GITHUB_TEST_REPO_NAME }} - ref: ${{ needs.OutputEnvVariables.outputs.CWA_GITHUB_TEST_REPO_BRANCH }} - - - name: Configure AWS Credentials - uses: aws-actions/configure-aws-credentials@v4 - with: - role-to-assume: ${{ env.TERRAFORM_AWS_ASSUME_ROLE }} - aws-region: us-west-2 - role-duration-seconds: ${{ env.TERRAFORM_AWS_ASSUME_ROLE_DURATION }} - - - name: Install Terraform - uses: hashicorp/setup-terraform@v3 - with: - terraform_version: 1.12.0 - - - name: Verify Terraform version - run: terraform --version - - - name: Terraform apply - uses: nick-fields/retry@v2 - with: - max_attempts: 1 - timeout_minutes: 60 - retry_wait_seconds: 5 - command: | - cd terraform/performance - terraform init - if terraform apply --auto-approve \ - -var="ssh_key_value=${PRIVATE_KEY}" \ - -var="cwa_github_sha=${{ inputs.build_id }}" \ - -var="cwa_github_sha_date=${{ needs.OutputEnvVariables.outputs.CWA_COMMIT_DATE }}" \ - -var="ami=${{ matrix.arrays.ami }}" \ - -var="arc=${{ matrix.arrays.arc }}" \ - -var="s3_bucket=${S3_INTEGRATION_BUCKET}" \ - -var="ssh_key_name=${KEY_NAME}" \ - -var="values_per_minute=${{ matrix.arrays.values_per_minute}}"\ - -var="family=${{ matrix.arrays.family}}"\ - -var="test_dir=${{ matrix.arrays.test_dir }}" ; then terraform destroy -auto-approve - else - terraform destroy -auto-approve && exit 1 - fi - - - name: Terraform destroy - if: ${{ cancelled() || failure() }} - uses: nick-fields/retry@v2 - with: - max_attempts: 3 - timeout_minutes: 8 - retry_wait_seconds: 5 - command: cd terraform/performance && terraform destroy --auto-approve - - StressTrackingTest: - name: ${{matrix.arrays.testName}} - needs: [GenerateTestMatrix, OutputEnvVariables ] - runs-on: ubuntu-latest - strategy: - fail-fast: false - matrix: - arrays: ${{ fromJson(needs.GenerateTestMatrix.outputs.ec2_stress_matrix) }} - permissions: - id-token: write - contents: read - steps: - - uses: actions/checkout@v3 - with: - repository: ${{ needs.OutputEnvVariables.outputs.CWA_GITHUB_TEST_REPO_NAME }} - ref: ${{ needs.OutputEnvVariables.outputs.CWA_GITHUB_TEST_REPO_BRANCH }} - - - name: Configure AWS Credentials - uses: aws-actions/configure-aws-credentials@v4 - with: - role-to-assume: ${{ env.TERRAFORM_AWS_ASSUME_ROLE }} - aws-region: us-west-2 - role-duration-seconds: ${{ env.TERRAFORM_AWS_ASSUME_ROLE_DURATION }} - - - name: Install Terraform - uses: hashicorp/setup-terraform@v3 - with: - terraform_version: 1.12.0 - - - name: Verify Terraform version - run: terraform --version - - - name: Echo Test Info - run: echo run on ec2 instance os ${{ matrix.arrays.os }} arc ${{ matrix.arrays.arc }} test dir ${{ matrix.arrays.test_dir }} values per minute ${{ matrix.arrays.values_per_minute }} - - - name: Terraform apply - uses: nick-fields/retry@v2 - with: - max_attempts: 1 - timeout_minutes: 60 - retry_wait_seconds: 5 - command: | - cd terraform/stress - terraform init - if terraform apply --auto-approve \ - -var="ssh_key_value=${PRIVATE_KEY}" \ - -var="cwa_github_sha=${{ inputs.build_id }}" \ - -var="ami=${{ matrix.arrays.ami }}" \ - -var="arc=${{ matrix.arrays.arc }}" \ - -var="s3_bucket=${S3_INTEGRATION_BUCKET}" \ - -var="ssh_key_name=${KEY_NAME}" \ - -var="values_per_minute=${{ matrix.arrays.values_per_minute}}"\ - -var="test_dir=${{ matrix.arrays.test_dir }}" ; then terraform destroy -auto-approve - else - terraform destroy -auto-approve && exit 1 - fi - - - name: Terraform destroy - if: ${{ cancelled() || failure() }} - uses: nick-fields/retry@v2 - with: - max_attempts: 3 - timeout_minutes: 8 - retry_wait_seconds: 5 - command: cd terraform/stress && terraform destroy --auto-approve - - EC2WinStressTrackingTest: - name: ${{matrix.arrays.testName}} - needs: [GenerateTestMatrix, OutputEnvVariables] - runs-on: ubuntu-latest - strategy: - fail-fast: false - matrix: - arrays: ${{ fromJson(needs.GenerateTestMatrix.outputs.ec2_windows_stress_matrix) }} - permissions: - id-token: write - contents: read - steps: - - uses: actions/checkout@v3 - with: - repository: ${{ needs.OutputEnvVariables.outputs.CWA_GITHUB_TEST_REPO_NAME }} - ref: ${{ needs.OutputEnvVariables.outputs.CWA_GITHUB_TEST_REPO_BRANCH }} - - - name: Configure AWS Credentials - uses: aws-actions/configure-aws-credentials@v4 - with: - role-to-assume: ${{ env.TERRAFORM_AWS_ASSUME_ROLE }} - aws-region: us-west-2 - role-duration-seconds: ${{ env.TERRAFORM_AWS_ASSUME_ROLE_DURATION }} - - - name: Install Terraform - uses: hashicorp/setup-terraform@v3 - with: - terraform_version: 1.12.0 - - - name: Verify Terraform version - run: terraform --version - - - name: Echo Test Info - run: echo run on ec2 instance os ${{ matrix.arrays.os }} arc ${{ matrix.arrays.arc }} test dir ${{ matrix.arrays.test_dir }} values per minute ${{ matrix.arrays.values_per_minute }} - - - name: Terraform apply - uses: nick-fields/retry@v2 - with: - max_attempts: 1 - timeout_minutes: 60 - retry_wait_seconds: 5 - command: | - cd terraform/stress - terraform init - if terraform apply --auto-approve \ - -var="ssh_key_value=${PRIVATE_KEY}" \ - -var="cwa_github_sha=${{ inputs.build_id }}" \ - -var="ami=${{ matrix.arrays.ami }}" \ - -var="arc=${{ matrix.arrays.arc }}" \ - -var="s3_bucket=${S3_INTEGRATION_BUCKET}" \ - -var="ssh_key_name=${KEY_NAME}" \ - -var="values_per_minute=${{ matrix.arrays.values_per_minute}}"\ - -var="family=${{ matrix.arrays.family}}"\ - -var="test_dir=${{ matrix.arrays.test_dir }}" ; then terraform destroy -auto-approve - else - terraform destroy -auto-approve && exit 1 - fi - - - name: Terraform destroy - if: ${{ cancelled() || failure() }} - uses: nick-fields/retry@v2 - with: - max_attempts: 3 - timeout_minutes: 8 - retry_wait_seconds: 5 - command: cd terraform/stress && terraform destroy --auto-approve - - GPUEndToEndTest: - name: ${{matrix.arrays.testName}} - needs: [ GenerateTestMatrix, OutputEnvVariables ] - runs-on: ubuntu-latest - strategy: - fail-fast: false - matrix: - arrays: ${{ fromJson(needs.GenerateTestMatrix.outputs.eks_addon_matrix) }} - permissions: - id-token: write - contents: read - steps: - - uses: actions/checkout@v3 - with: - repository: ${{ needs.OutputEnvVariables.outputs.CWA_GITHUB_TEST_REPO_NAME }} - ref: ${{ needs.OutputEnvVariables.outputs.CWA_GITHUB_TEST_REPO_BRANCH }} - - - name: Configure AWS Credentials - uses: aws-actions/configure-aws-credentials@v4 - with: - role-to-assume: ${{ env.TERRAFORM_AWS_ASSUME_ROLE }} - aws-region: us-west-2 - role-duration-seconds: ${{ env.TERRAFORM_AWS_ASSUME_ROLE_DURATION }} - - - name: Login ECR - id: login-ecr - uses: aws-actions/amazon-ecr-login@v2 - - - name: Install Terraform - uses: hashicorp/setup-terraform@v3 - with: - terraform_version: 1.12.0 - - - name: Verify Terraform version - run: terraform --version - - - name: Terraform apply and setup - run: | - if [ "${{ matrix.arrays.terraform_dir }}" != "" ]; then - cd "${{ matrix.arrays.terraform_dir }}" - else - cd terraform/eks/addon/gpu - fi - - terraform init - if terraform apply --auto-approve \ - -var="test_dir=${{ matrix.arrays.test_dir }}" \ - -var="cwagent_image_repo=${{ steps.login-ecr.outputs.registry }}/${{ env.ECR_INTEGRATION_TEST_REPO }}" \ - -var="cwagent_image_tag=${{ inputs.build_id }}" \ - -var="ami_type=${{ matrix.arrays.ami }}" \ - -var="instance_type=${{ matrix.arrays.instanceType }}" \ - -var="k8s_version=${{ matrix.arrays.k8sVersion }}"; then - echo "Terraform apply successful." - else - terraform destroy -auto-approve && exit 1 - fi - - - name: Terraform destroy - if: always() - uses: nick-fields/retry@v2 - with: - max_attempts: 3 - timeout_minutes: 8 - retry_wait_seconds: 5 - command: | - if [ "${{ matrix.arrays.terraform_dir }}" != "" ]; then - cd "${{ matrix.arrays.terraform_dir }}" - else - cd terraform/eks/addon/gpu - fi - terraform destroy -auto-approve diff --git a/.github/workflows/wd-integration-test.yml b/.github/workflows/wd-integration-test.yml index 1a1fd0ceb7..b3b35bf202 100644 --- a/.github/workflows/wd-integration-test.yml +++ b/.github/workflows/wd-integration-test.yml @@ -83,7 +83,7 @@ jobs: EC2LinuxWorkloadDiscoveryIntegrationTest: needs: [ GenerateTestMatrix ] - name: ${{matrix.arrays.testName}} + name: ${{ matrix.arrays.wip && '[WIP] ' || '' }}${{matrix.arrays.testName}} runs-on: ubuntu-latest strategy: fail-fast: false @@ -117,6 +117,8 @@ jobs: run: terraform --version - name: Terraform apply + id: terraform_apply + continue-on-error: true uses: nick-fields/retry@v2 with: max_attempts: 3 @@ -151,6 +153,20 @@ jobs: terraform destroy -auto-approve && exit 1 fi + - name: "[WIP] ✅ Passed" + if: steps.terraform_apply.outcome == 'success' && matrix.arrays.wip + run: echo "::notice::WIP test passed" + + - name: "[WIP] ⚠️ Failed (Overruled)" + if: steps.terraform_apply.outcome == 'failure' && matrix.arrays.wip + run: | + echo "::warning::WIP test failed but allowed to continue" + echo "### ⚠️ WIP Test Failed (Overruled)" >> $GITHUB_STEP_SUMMARY + + - name: Fail if not WIP + if: steps.terraform_apply.outcome == 'failure' && !matrix.arrays.wip + run: exit 1 + - name: Terraform destroy if: ${{ (cancelled() || failure()) }} uses: nick-fields/retry@v2 @@ -169,7 +185,7 @@ jobs: EC2WindowsWorkloadDiscoveryIntegrationTest: needs: [ GenerateTestMatrix ] - name: ${{matrix.arrays.testName}} + name: ${{ matrix.arrays.wip && '[WIP] ' || '' }}${{matrix.arrays.testName}} runs-on: ubuntu-latest strategy: fail-fast: false @@ -203,6 +219,8 @@ jobs: run: terraform --version - name: Terraform apply + id: terraform_apply + continue-on-error: true uses: nick-fields/retry@v2 with: max_attempts: 3 @@ -230,6 +248,20 @@ jobs: terraform destroy -auto-approve && exit 1 fi + - name: "[WIP] ✅ Passed" + if: steps.terraform_apply.outcome == 'success' && matrix.arrays.wip + run: echo "::notice::WIP test passed" + + - name: "[WIP] ⚠️ Failed (Overruled)" + if: steps.terraform_apply.outcome == 'failure' && matrix.arrays.wip + run: | + echo "::warning::WIP test failed but allowed to continue" + echo "### ⚠️ WIP Test Failed (Overruled)" >> $GITHUB_STEP_SUMMARY + + - name: Fail if not WIP + if: steps.terraform_apply.outcome == 'failure' && !matrix.arrays.wip + run: exit 1 + - name: Terraform destroy if: ${{ (cancelled() || failure()) }} uses: nick-fields/retry@v2 @@ -248,7 +280,7 @@ jobs: EC2NvidiaLinuxWorkloadDiscoveryIntegrationTest: needs: [ GenerateTestMatrix ] - name: ${{matrix.arrays.testName}} + name: ${{ matrix.arrays.wip && '[WIP] ' || '' }}${{matrix.arrays.testName}} runs-on: ubuntu-latest strategy: fail-fast: false @@ -282,6 +314,8 @@ jobs: run: terraform --version - name: Terraform apply + id: terraform_apply + continue-on-error: true uses: nick-fields/retry@v2 with: max_attempts: 3 @@ -316,6 +350,20 @@ jobs: terraform destroy -auto-approve && exit 1 fi + - name: "[WIP] ✅ Passed" + if: steps.terraform_apply.outcome == 'success' && matrix.arrays.wip + run: echo "::notice::WIP test passed" + + - name: "[WIP] ⚠️ Failed (Overruled)" + if: steps.terraform_apply.outcome == 'failure' && matrix.arrays.wip + run: | + echo "::warning::WIP test failed but allowed to continue" + echo "### ⚠️ WIP Test Failed (Overruled)" >> $GITHUB_STEP_SUMMARY + + - name: Fail if not WIP + if: steps.terraform_apply.outcome == 'failure' && !matrix.arrays.wip + run: exit 1 + - name: Terraform destroy if: ${{ (cancelled() || failure()) }} uses: nick-fields/retry@v2 @@ -334,7 +382,7 @@ jobs: EC2NvidiaWindowsWorkloadDiscoveryIntegrationTest: needs: [ GenerateTestMatrix ] - name: ${{matrix.arrays.testName}} + name: ${{ matrix.arrays.wip && '[WIP] ' || '' }}${{matrix.arrays.testName}} runs-on: ubuntu-latest strategy: fail-fast: false @@ -368,6 +416,8 @@ jobs: run: terraform --version - name: Terraform apply + id: terraform_apply + continue-on-error: true uses: nick-fields/retry@v2 with: max_attempts: 3 @@ -395,6 +445,20 @@ jobs: terraform destroy -auto-approve && exit 1 fi + - name: "[WIP] ✅ Passed" + if: steps.terraform_apply.outcome == 'success' && matrix.arrays.wip + run: echo "::notice::WIP test passed" + + - name: "[WIP] ⚠️ Failed (Overruled)" + if: steps.terraform_apply.outcome == 'failure' && matrix.arrays.wip + run: | + echo "::warning::WIP test failed but allowed to continue" + echo "### ⚠️ WIP Test Failed (Overruled)" >> $GITHUB_STEP_SUMMARY + + - name: Fail if not WIP + if: steps.terraform_apply.outcome == 'failure' && !matrix.arrays.wip + run: exit 1 + - name: Terraform destroy if: ${{ (cancelled() || failure()) }} uses: nick-fields/retry@v2