diff --git a/ci/export-concourse-audit-logs-to-s3.sh b/ci/export-concourse-audit-logs-to-s3.sh new file mode 100755 index 0000000..e70e8c1 --- /dev/null +++ b/ci/export-concourse-audit-logs-to-s3.sh @@ -0,0 +1,34 @@ +#!/bin/bash +set -euo pipefail +curl -o fly "${ATC_URL}/api/v1/cli?arch=amd64&platform=linux" +chmod +x ./fly +( + set +x + ./fly --target ci login \ + --concourse-url "${ATC_URL}" \ + --username "${BASIC_AUTH_USERNAME}" \ + --password "${BASIC_AUTH_PASSWORD}" +) + +# Calculate timestamp for 1 hour ago +one_hour_ago=$(date -d '1 hour ago' '+%Y-%m-%d %H:%M:%S') +# Get list of build audit events +output=$(./fly --target ci builds --since="$one_hour_ago") + +# Get current timestamp components for S3 path +current_time=$(date -u '+%Y/%m/%d/%H/%M/%S') +epoch_time=$(date +%s) + +# Create filename with .gz extension +filename="concourse-audit${epoch_time}.txt.gz" + +# Create full S3 path +s3_path="s3://${BUCKET_NAME}/${current_time}/${filename}" + +# Compress and write output to S3 with error handling +if echo "$output" | gzip | aws s3 cp - "$s3_path" --content-encoding gzip --sse AES256; then + echo "Successfully uploaded compressed audit data to: $s3_path" +else + echo "Failed to upload audit data to S3" >&2 + exit 1 +fi \ No newline at end of file diff --git a/ci/export-concourse-audit-logs-to-s3.yml b/ci/export-concourse-audit-logs-to-s3.yml new file mode 100644 index 0000000..9755c1c --- /dev/null +++ b/ci/export-concourse-audit-logs-to-s3.yml @@ -0,0 +1,24 @@ +--- +platform: linux + +image_resource: + type: registry-image + source: + aws_access_key_id: ((ecr_aws_key)) + aws_secret_access_key: ((ecr_aws_secret)) + repository: general-task + aws_region: us-gov-west-1 + tag: latest + +inputs: +- name: concourse-config + +run: + path: concourse-config/ci/export-concourse-audit-logs-to-s3.sh + +params: + ATC_URL: + BASIC_AUTH_USERNAME: + BASIC_AUTH_PASSWORD: + BUCKET_NAME: + AWS_DEFAULT_REGION: diff --git a/ci/pipeline.yml b/ci/pipeline.yml index 0b908d4..cc022ea 100644 --- a/ci/pipeline.yml +++ b/ci/pipeline.yml @@ -90,6 +90,62 @@ jobs: <<: *deploy-params dry_run: true +- name: export-audit-logs-staging + serial: true + serial_groups: [staging] + interruptible: true + plan: + - in_parallel: + - get: hourly-timer + trigger: true + - get: concourse-config + - task: export-concourse-audit-logs-to-s3 + tags: [iaas] + file: concourse-config/ci/export-concourse-audit-logs-to-s3.yml + params: + ATC_URL: https://ci.fr-stage.cloud.gov + BASIC_AUTH_USERNAME: ((basic-auth-username-staging)) + BASIC_AUTH_PASSWORD: ((basic-auth-password-staging)) + BUCKET_NAME: ((logs_concourse_audit_events_bucket_name-staging)) + AWS_DEFAULT_REGION: us-gov-west-1 + on_failure: + put: slack + params: + text: | + :x: FAILED, could not export Concourse audit logs to s3 in staging + <$ATC_EXTERNAL_URL/teams/$BUILD_TEAM_NAME/pipelines/$BUILD_PIPELINE_NAME/jobs/$BUILD_JOB_NAME/builds/$BUILD_NAME|View build details> + channel: ((slack-channel-failure)) + username: ((slack-username)) + icon_url: ((slack-icon-url)) + +- name: export-audit-logs-production + serial: true + serial_groups: [production] + interruptible: true + plan: + - in_parallel: + - get: hourly-timer + trigger: true + - get: concourse-config + - task: export-concourse-audit-logs-to-s3 + tags: [iaas] + file: concourse-config/ci/export-concourse-audit-logs-to-s3.yml + params: + ATC_URL: https://ci.fr.cloud.gov + BASIC_AUTH_USERNAME: ((basic-auth-username-production)) + BASIC_AUTH_PASSWORD: ((basic-auth-password-production)) + BUCKET_NAME: ((logs_concourse_audit_events_bucket_name-production)) + AWS_DEFAULT_REGION: us-gov-west-1 + on_failure: + put: slack + params: + text: | + :x: FAILED, could not export Concourse audit logs to s3 in production + <$ATC_EXTERNAL_URL/teams/$BUILD_TEAM_NAME/pipelines/$BUILD_PIPELINE_NAME/jobs/$BUILD_JOB_NAME/builds/$BUILD_NAME|View build details> + channel: ((slack-channel-failure)) + username: ((slack-username)) + icon_url: ((slack-icon-url)) + - name: find-stalled-workers-staging serial: true serial_groups: [staging] @@ -484,6 +540,8 @@ groups: - find-stalled-workers-production - iptables-staging - iptables-production + - export-audit-logs-staging + - export-audit-logs-production - name: deployments jobs: - plan-concourse-staging @@ -496,3 +554,6 @@ groups: - find-stalled-workers-production - iptables-staging - iptables-production + - export-audit-logs-staging + - export-audit-logs-production +