From aee919d2d8baa7d3b86acb4e06578935b77894ea Mon Sep 17 00:00:00 2001 From: Christopher Weibel Date: Tue, 17 Feb 2026 16:39:06 -0500 Subject: [PATCH 1/7] Adding export-concourse-audit-logs-to-s3 --- ci/export-concourse-audit-logs-to-s3.sh | 34 ++++++++++++++++++++++++ ci/export-concourse-audit-logs-to-s3.yml | 22 +++++++++++++++ 2 files changed, 56 insertions(+) create mode 100755 ci/export-concourse-audit-logs-to-s3.sh create mode 100644 ci/export-concourse-audit-logs-to-s3.yml diff --git a/ci/export-concourse-audit-logs-to-s3.sh b/ci/export-concourse-audit-logs-to-s3.sh new file mode 100755 index 0000000..d75035f --- /dev/null +++ b/ci/export-concourse-audit-logs-to-s3.sh @@ -0,0 +1,34 @@ +#!/bin/bash +set -eux +curl -o fly "${ATC_URL}/api/v1/cli?arch=amd64&platform=linux" +chmod +x ./fly +( + set +x + ./fly --target ci login \ + --concourse-url "${ATC_URL}" \ + --username "${BASIC_AUTH_USERNAME}" \ + --password "${BASIC_AUTH_PASSWORD}" +) + +# Calculate timestamp for 1 hour ago +one_hour_ago=$(date -d '1 hour ago' '+%Y-%m-%d %H:%M:%S') +# Get list of build audit events +output=$(./fly --target ci builds --since="$one_hour_ago") + +# Get current timestamp components for S3 path +current_time=$(date -u '+%Y/%m/%d/%H/%M/%S') +epoch_time=$(date +%s) + +# Create filename with .gz extension +filename="concourse-audit${epoch_time}.txt.gz" + +# Create full S3 path +s3_path="s3://${BUCKET_NAME}/${current_time}/${filename}" + +# Compress and write output to S3 with error handling +if echo "$output" | gzip | aws s3 cp - "$s3_path" --content-encoding gzip; then + echo "Successfully uploaded compressed audit data to: $s3_path" +else + echo "Failed to upload audit data to S3" >&2 + exit 1 +fi \ No newline at end of file diff --git a/ci/export-concourse-audit-logs-to-s3.yml b/ci/export-concourse-audit-logs-to-s3.yml new file mode 100644 index 0000000..eb45d6c --- /dev/null +++ b/ci/export-concourse-audit-logs-to-s3.yml @@ -0,0 +1,22 @@ +--- +platform: linux + +image_resource: + type: registry-image + source: + aws_access_key_id: ((ecr_aws_key)) + aws_secret_access_key: ((ecr_aws_secret)) + repository: general-task + aws_region: us-gov-west-1 + tag: latest + +inputs: +- name: concourse-config + +run: + path: concourse-config/ci/export-concourse-audit-logs-to-s3.yml + +params: + ATC_URL: + BASIC_AUTH_USERNAME: + BASIC_AUTH_PASSWORD: From d1aed24239cdca3ba1b818bbf1b486a3aa4f1225 Mon Sep 17 00:00:00 2001 From: Christopher Weibel Date: Tue, 17 Feb 2026 16:50:31 -0500 Subject: [PATCH 2/7] Fix file path --- ci/export-concourse-audit-logs-to-s3.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci/export-concourse-audit-logs-to-s3.yml b/ci/export-concourse-audit-logs-to-s3.yml index eb45d6c..755a02f 100644 --- a/ci/export-concourse-audit-logs-to-s3.yml +++ b/ci/export-concourse-audit-logs-to-s3.yml @@ -14,7 +14,7 @@ inputs: - name: concourse-config run: - path: concourse-config/ci/export-concourse-audit-logs-to-s3.yml + path: concourse-config/ci/export-concourse-audit-logs-to-s3.sh params: ATC_URL: From 5733451a905bf4dac9bf2a3cefabc5ab09b76b0b Mon Sep 17 00:00:00 2001 From: Christopher Weibel Date: Tue, 17 Feb 2026 16:53:56 -0500 Subject: [PATCH 3/7] Add BUCKET_NAME param --- ci/export-concourse-audit-logs-to-s3.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/ci/export-concourse-audit-logs-to-s3.yml b/ci/export-concourse-audit-logs-to-s3.yml index 755a02f..b8321a4 100644 --- a/ci/export-concourse-audit-logs-to-s3.yml +++ b/ci/export-concourse-audit-logs-to-s3.yml @@ -20,3 +20,4 @@ params: ATC_URL: BASIC_AUTH_USERNAME: BASIC_AUTH_PASSWORD: + BUCKET_NAME: From 73bfb35084d9a36da27e570291c1ad706194458a Mon Sep 17 00:00:00 2001 From: Christopher Weibel Date: Tue, 17 Feb 2026 17:01:34 -0500 Subject: [PATCH 4/7] Add --sse AES256 --- ci/export-concourse-audit-logs-to-s3.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci/export-concourse-audit-logs-to-s3.sh b/ci/export-concourse-audit-logs-to-s3.sh index d75035f..bd19f06 100755 --- a/ci/export-concourse-audit-logs-to-s3.sh +++ b/ci/export-concourse-audit-logs-to-s3.sh @@ -26,7 +26,7 @@ filename="concourse-audit${epoch_time}.txt.gz" s3_path="s3://${BUCKET_NAME}/${current_time}/${filename}" # Compress and write output to S3 with error handling -if echo "$output" | gzip | aws s3 cp - "$s3_path" --content-encoding gzip; then +if echo "$output" | gzip | aws s3 cp - "$s3_path" --content-encoding gzip --sse AES256; then echo "Successfully uploaded compressed audit data to: $s3_path" else echo "Failed to upload audit data to S3" >&2 From 64ece6c1c2a15070bd09eeeca4d509740e1ec3aa Mon Sep 17 00:00:00 2001 From: Christopher Weibel Date: Wed, 18 Feb 2026 07:46:02 -0500 Subject: [PATCH 5/7] Add export AWS_DEFAULT_REGION --- ci/export-concourse-audit-logs-to-s3.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/ci/export-concourse-audit-logs-to-s3.yml b/ci/export-concourse-audit-logs-to-s3.yml index b8321a4..9755c1c 100644 --- a/ci/export-concourse-audit-logs-to-s3.yml +++ b/ci/export-concourse-audit-logs-to-s3.yml @@ -21,3 +21,4 @@ params: BASIC_AUTH_USERNAME: BASIC_AUTH_PASSWORD: BUCKET_NAME: + AWS_DEFAULT_REGION: From fd3704704e8c56e42b035c6e11bf0f495ee6e742 Mon Sep 17 00:00:00 2001 From: Christopher Weibel Date: Wed, 18 Feb 2026 08:04:37 -0500 Subject: [PATCH 6/7] Add set -euo pipefail --- ci/export-concourse-audit-logs-to-s3.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci/export-concourse-audit-logs-to-s3.sh b/ci/export-concourse-audit-logs-to-s3.sh index bd19f06..e70e8c1 100755 --- a/ci/export-concourse-audit-logs-to-s3.sh +++ b/ci/export-concourse-audit-logs-to-s3.sh @@ -1,5 +1,5 @@ #!/bin/bash -set -eux +set -euo pipefail curl -o fly "${ATC_URL}/api/v1/cli?arch=amd64&platform=linux" chmod +x ./fly ( From 312977e15ab896575bb754dcb3fb28a75345289c Mon Sep 17 00:00:00 2001 From: Christopher Weibel Date: Wed, 18 Feb 2026 08:34:46 -0500 Subject: [PATCH 7/7] Set pipeline for PR --- ci/pipeline.yml | 61 +++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 61 insertions(+) diff --git a/ci/pipeline.yml b/ci/pipeline.yml index 0b908d4..cc022ea 100644 --- a/ci/pipeline.yml +++ b/ci/pipeline.yml @@ -90,6 +90,62 @@ jobs: <<: *deploy-params dry_run: true +- name: export-audit-logs-staging + serial: true + serial_groups: [staging] + interruptible: true + plan: + - in_parallel: + - get: hourly-timer + trigger: true + - get: concourse-config + - task: export-concourse-audit-logs-to-s3 + tags: [iaas] + file: concourse-config/ci/export-concourse-audit-logs-to-s3.yml + params: + ATC_URL: https://ci.fr-stage.cloud.gov + BASIC_AUTH_USERNAME: ((basic-auth-username-staging)) + BASIC_AUTH_PASSWORD: ((basic-auth-password-staging)) + BUCKET_NAME: ((logs_concourse_audit_events_bucket_name-staging)) + AWS_DEFAULT_REGION: us-gov-west-1 + on_failure: + put: slack + params: + text: | + :x: FAILED, could not export Concourse audit logs to s3 in staging + <$ATC_EXTERNAL_URL/teams/$BUILD_TEAM_NAME/pipelines/$BUILD_PIPELINE_NAME/jobs/$BUILD_JOB_NAME/builds/$BUILD_NAME|View build details> + channel: ((slack-channel-failure)) + username: ((slack-username)) + icon_url: ((slack-icon-url)) + +- name: export-audit-logs-production + serial: true + serial_groups: [production] + interruptible: true + plan: + - in_parallel: + - get: hourly-timer + trigger: true + - get: concourse-config + - task: export-concourse-audit-logs-to-s3 + tags: [iaas] + file: concourse-config/ci/export-concourse-audit-logs-to-s3.yml + params: + ATC_URL: https://ci.fr.cloud.gov + BASIC_AUTH_USERNAME: ((basic-auth-username-production)) + BASIC_AUTH_PASSWORD: ((basic-auth-password-production)) + BUCKET_NAME: ((logs_concourse_audit_events_bucket_name-production)) + AWS_DEFAULT_REGION: us-gov-west-1 + on_failure: + put: slack + params: + text: | + :x: FAILED, could not export Concourse audit logs to s3 in production + <$ATC_EXTERNAL_URL/teams/$BUILD_TEAM_NAME/pipelines/$BUILD_PIPELINE_NAME/jobs/$BUILD_JOB_NAME/builds/$BUILD_NAME|View build details> + channel: ((slack-channel-failure)) + username: ((slack-username)) + icon_url: ((slack-icon-url)) + - name: find-stalled-workers-staging serial: true serial_groups: [staging] @@ -484,6 +540,8 @@ groups: - find-stalled-workers-production - iptables-staging - iptables-production + - export-audit-logs-staging + - export-audit-logs-production - name: deployments jobs: - plan-concourse-staging @@ -496,3 +554,6 @@ groups: - find-stalled-workers-production - iptables-staging - iptables-production + - export-audit-logs-staging + - export-audit-logs-production +