test: add git config integration tests

.github/workflows/call-test-containers-k8s.yaml

@@ -88,6 +88,8 @@ jobs:
           BATS_NO_PARALLELIZE_ACROSS_FILES: 1
           BATS_NUMBER_OF_PARALLEL_JOBS: 4
           BATS_PARALLEL_BINARY_NAME: rush
+          # See https://github.com/orgs/community/discussions/25191
+          GIT_SHA: ${{ github.event.pull_request.head.sha || github.sha }}
 
       # Debugging steps to run if something goes wrong
       - name: Debug - List all pods

.github/workflows/call-test-containers.yaml

@@ -105,6 +105,8 @@ jobs:
           BATS_NO_PARALLELIZE_ACROSS_FILES: 1
           BATS_NUMBER_OF_PARALLEL_JOBS: 4
           BATS_PARALLEL_BINARY_NAME: rush
+          # See https://github.com/orgs/community/discussions/25191
+          GIT_SHA: ${{ github.event.pull_request.head.sha || github.sha }}
 
   test-verify-signatures:
     name: Verify image signatures for ${{ inputs.image }}:${{ inputs.image-tag }}

.github/workflows/call-test-packages.yaml

@@ -158,6 +158,8 @@ jobs:
         env:
           BASE_IMAGE: ${{ matrix.config.image }}
           DISTRO: ${{ matrix.config.distro }}
+          # See https://github.com/orgs/community/discussions/25191
+          GIT_SHA: ${{ github.event.pull_request.head.sha || github.sha }}
 
   # Test packages installed on the actual target OS
   # For now we just use the ephemeral runners
@@ -249,6 +251,9 @@ jobs:
       - name: Run integration tests
         run: ./testing/bats/run-package-integration-tests.sh
         shell: bash
+        env:
+          # See https://github.com/orgs/community/discussions/25191
+          GIT_SHA: ${{ github.event.pull_request.head.sha || github.sha }}
 
       - name: Debug linux binary
         if: runner.os == 'Linux'

testing/bats/run-bats.sh

@@ -17,9 +17,13 @@ export FLUENT_BIT_BINARY=${FLUENT_BIT_BINARY:-/fluent-bit/bin/fluent-bit}
 export TELEMETRY_FORGE_AGENT_VERSION=${TELEMETRY_FORGE_AGENT_VERSION:-26.3.4}
 export TELEMETRY_FORGE_AGENT_URL="${TELEMETRY_FORGE_AGENT_URL:-https://staging.telemetryforge.io}"
 
+# Used primarily for testing the git config plugin: the actual SHA of the commit for a PR, etc.
+export GIT_SHA=${GIT_SHA:-main}
+
 echo "INFO: Testing with binary '$FLUENT_BIT_BINARY'"
 echo "INFO: Testing with version '$TELEMETRY_FORGE_AGENT_VERSION'"
 echo "INFO: Testing with URL '$TELEMETRY_FORGE_AGENT_URL'"
+echo "INFO: Testing with SHA '$GIT_SHA'"
 
 # Optional variables for container/k8s tests
 # TELEMETRY_FORGE_AGENT_IMAGE=...

testing/bats/tests/functional/plugins/git_config/resources/fluent-bit.yaml

@@ -0,0 +1,20 @@
+service:
+  flush: 1
+  grace: 1
+  log_level: info
+
+pipeline:
+  inputs:
+    - name: dummy
+      dummy: '{"message": "Switched to updated configuration", "sha": "${GIT_SHA}"}'
+      alias: input_dummy
+
+  outputs:
+    - name: stdout
+      match: '*'
+      alias: output_stdout
+    - name: exit
+      match: '*'
+      # Exit after processing 2 records to ensure the test completes in a reasonable time frame
+      record_count: 2
+      alias: output_exit

testing/bats/tests/functional/plugins/git_config/resources/initial-fluent-bit.yaml

@@ -0,0 +1,13 @@
+service:
+  flush: 1
+  grace: 1
+  log_level: debug
+
+customs:
+  - name: git_config
+    repo: https://github.com/telemetryforge/agent.git
+    # We may need to adjust this per-PR appropriately to ensure we are testing the correct branch/PR
+    ref: ${GIT_SHA}
+    path: testing/bats/tests/functional/plugins/git_config/resources/fluent-bit.yaml
+    # Do not poll too fast as we still need to apply the commit between polls
+    poll_interval: 10

testing/bats/tests/functional/plugins/git_config/verify-git-config-plugin.bats

@@ -0,0 +1,39 @@
+#!/usr/bin/env bats
+
+ensure_variables_set BATS_SUPPORT_ROOT BATS_ASSERT_ROOT BATS_FILE_ROOT FLUENT_BIT_BINARY GIT_SHA
+
+load "$BATS_SUPPORT_ROOT/load.bash"
+load "$BATS_ASSERT_ROOT/load.bash"
+load "$BATS_FILE_ROOT/load.bash"
+
+# bats file_tags=functional
+setup() {
+}
+
+teardown() {
+    if [[ -n "${SKIP_TEARDOWN:-}" ]]; then
+        echo "Skipping teardown"
+    fi
+}
+
+@test "verify git config plugin basic configuration" {
+    # Verify the configuration files are valid and the plugin can start without errors with the provided configuration
+    run "$FLUENT_BIT_BINARY" -c "$BATS_FILE_ROOT/resources/initial-fluent-bit.yaml" --dry-run
+    assert_success
+    refute_output --partial "[error]"
+
+    # This is the one we should switch to after the first poll interval
+    run "$FLUENT_BIT_BINARY" -c "$BATS_FILE_ROOT/resources/fluent-bit.yaml" --dry-run
+    assert_success
+    refute_output --partial "[error]"
+
+    run "$FLUENT_BIT_BINARY" -c "$BATS_FILE_ROOT/resources/initial-fluent-bit.yaml"
+    assert_success
+    assert_output --partial 'no existing git config found'
+    # Check we are correctly polling the repository and not encountering errors extracting the config file
+    assert_output --partial 'polling repository https://github.com/telemetryforge/agent.git'
+    refute_output --partial 'failed to extract config file'
+    assert_output --partial 'new commit detected'
+    # The output should contain the message about switching to the updated configuration after the first poll interval
+    assert_output --partial 'Switched to updated configuration'
+}

testing/bats/tests/functional/plugins/verify-custom-plugins.bats

@@ -0,0 +1,41 @@
+#!/usr/bin/env bats
+
+ensure_variables_set BATS_SUPPORT_ROOT BATS_ASSERT_ROOT BATS_FILE_ROOT FLUENT_BIT_BINARY
+
+load "$BATS_SUPPORT_ROOT/load.bash"
+load "$BATS_ASSERT_ROOT/load.bash"
+load "$BATS_FILE_ROOT/load.bash"
+
+# bats file_tags=functional
+
+setup() {
+}
+
+teardown() {
+    if [[ -n "${SKIP_TEARDOWN:-}" ]]; then
+        echo "Skipping teardown"
+    fi
+}
+
+@test "verify git_config plugin exists" {
+    run "$FLUENT_BIT_BINARY" --help
+    assert_success
+    assert_output --partial "git_config"
+    run "$FLUENT_BIT_BINARY" -C git_config --help
+    assert_success
+    refute_output --partial "[error]"
+    refute_output --partial "tried to instance a plugin name that doesn't exist"
+}
+
+@test "verify log_sampling processor exists" {
+    run "$FLUENT_BIT_BINARY" --help
+    assert_success
+    assert_output --partial "log_sampling"
+}
+
+@test "verify dedupe processor exists" {
+    run "$FLUENT_BIT_BINARY" --help
+    assert_success
+    assert_output --partial "dedupe"
+}
+