Export concourse audit logs to s3

ci/export-concourse-audit-logs-to-s3.sh

@@ -0,0 +1,34 @@
+#!/bin/bash
+set -euo pipefail
+curl -o fly "${ATC_URL}/api/v1/cli?arch=amd64&platform=linux"
+chmod +x ./fly
+(
+  set +x
+  ./fly --target ci login \
+    --concourse-url "${ATC_URL}" \
+    --username "${BASIC_AUTH_USERNAME}" \
+    --password "${BASIC_AUTH_PASSWORD}"
+)
+
+# Calculate timestamp for 1 hour ago
+one_hour_ago=$(date -d '1 hour ago' '+%Y-%m-%d %H:%M:%S')
+# Get list of build audit events
+output=$(./fly --target ci builds --since="$one_hour_ago")
+
+# Get current timestamp components for S3 path
+current_time=$(date -u '+%Y/%m/%d/%H/%M/%S')
+epoch_time=$(date +%s)
+
+# Create filename with .gz extension
+filename="concourse-audit${epoch_time}.txt.gz"
+
+# Create full S3 path
+s3_path="s3://${BUCKET_NAME}/${current_time}/${filename}"
+
+# Compress and write output to S3 with error handling
+if echo "$output" | gzip | aws s3 cp - "$s3_path" --content-encoding gzip --sse AES256; then
+    echo "Successfully uploaded compressed audit data to: $s3_path"
+else
+    echo "Failed to upload audit data to S3" >&2
+    exit 1
+fi

ci/export-concourse-audit-logs-to-s3.yml

@@ -0,0 +1,24 @@
+---
+platform: linux
+
+image_resource:
+  type: registry-image
+  source:
+    aws_access_key_id: ((ecr_aws_key))
+    aws_secret_access_key: ((ecr_aws_secret))
+    repository: general-task
+    aws_region: us-gov-west-1
+    tag: latest
+
+inputs:
+- name: concourse-config
+
+run:
+  path: concourse-config/ci/export-concourse-audit-logs-to-s3.sh
+
+params:
+  ATC_URL:
+  BASIC_AUTH_USERNAME:
+  BASIC_AUTH_PASSWORD:
+  BUCKET_NAME:
+  AWS_DEFAULT_REGION:

ci/pipeline.yml

@@ -90,6 +90,62 @@ jobs:
       <<: *deploy-params
       dry_run: true
 
+- name: export-audit-logs-staging
+  serial: true
+  serial_groups: [staging]
+  interruptible: true
+  plan:
+  - in_parallel:
+    - get: hourly-timer
+      trigger: true
+    - get: concourse-config
+  - task: export-concourse-audit-logs-to-s3
+    tags: [iaas]
+    file: concourse-config/ci/export-concourse-audit-logs-to-s3.yml
+    params:
+      ATC_URL: https://ci.fr-stage.cloud.gov
+      BASIC_AUTH_USERNAME: ((basic-auth-username-staging))
+      BASIC_AUTH_PASSWORD: ((basic-auth-password-staging))
+      BUCKET_NAME: ((logs_concourse_audit_events_bucket_name-staging))
+      AWS_DEFAULT_REGION: us-gov-west-1
+  on_failure:
+    put: slack
+    params:
+      text: |
+        :x: FAILED, could not export Concourse audit logs to s3 in staging
+        <$ATC_EXTERNAL_URL/teams/$BUILD_TEAM_NAME/pipelines/$BUILD_PIPELINE_NAME/jobs/$BUILD_JOB_NAME/builds/$BUILD_NAME|View build details>
+      channel: ((slack-channel-failure))
+      username: ((slack-username))
+      icon_url: ((slack-icon-url))
+
+- name: export-audit-logs-production
+  serial: true
+  serial_groups: [production]
+  interruptible: true
+  plan:
+  - in_parallel:
+    - get: hourly-timer
+      trigger: true
+    - get: concourse-config
+  - task: export-concourse-audit-logs-to-s3
+    tags: [iaas]
+    file: concourse-config/ci/export-concourse-audit-logs-to-s3.yml
+    params:
+      ATC_URL: https://ci.fr.cloud.gov
+      BASIC_AUTH_USERNAME: ((basic-auth-username-production))
+      BASIC_AUTH_PASSWORD: ((basic-auth-password-production))
+      BUCKET_NAME: ((logs_concourse_audit_events_bucket_name-production))
+      AWS_DEFAULT_REGION: us-gov-west-1
+  on_failure:
+    put: slack
+    params:
+      text: |
+        :x: FAILED, could not export Concourse audit logs to s3 in production
+        <$ATC_EXTERNAL_URL/teams/$BUILD_TEAM_NAME/pipelines/$BUILD_PIPELINE_NAME/jobs/$BUILD_JOB_NAME/builds/$BUILD_NAME|View build details>
+      channel: ((slack-channel-failure))
+      username: ((slack-username))
+      icon_url: ((slack-icon-url))
+
 - name: find-stalled-workers-staging
   serial: true
   serial_groups: [staging]
@@ -484,6 +540,8 @@ groups:
       - find-stalled-workers-production
       - iptables-staging
       - iptables-production
+      - export-audit-logs-staging
+      - export-audit-logs-production
   - name: deployments
     jobs:
       - plan-concourse-staging
@@ -496,3 +554,6 @@ groups:
       - find-stalled-workers-production
       - iptables-staging
       - iptables-production
+      - export-audit-logs-staging
+      - export-audit-logs-production
+