From e034fee5d38d841ada9df23424592e479c0d343f Mon Sep 17 00:00:00 2001
From: Brice Dutheil <brice.dutheil@gmail.com>
Date: Mon, 2 Feb 2026 15:48:46 +0100
Subject: [PATCH 01/15] build: Count ran tests

---
 .gitlab-ci.yml                   |  24 +++
 .gitlab/aggregate_test_counts.sh | 271 +++++++++++++++++++++++++++++++
 .gitlab/collect_results.sh       |   8 +
 .gitlab/count_tests.sh           | 120 ++++++++++++++
 4 files changed, 423 insertions(+)
 create mode 100755 .gitlab/aggregate_test_counts.sh
 create mode 100755 .gitlab/count_tests.sh

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 064b031355d..917607f7892 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -12,6 +12,7 @@ stages:
   - benchmarks
   - macrobenchmarks
   - tests
+  - aggregate
   - exploration-tests
   - ci-visibility-tests
   - generate-signing-key
@@ -450,10 +451,12 @@ test_published_artifacts:
     - gitlab_section_start "collect-reports" "Collecting reports"
     - .gitlab/collect_reports.sh --destination ./check_reports --move
     - gitlab_section_end "collect-reports"
+    - .gitlab/count_tests.sh -v "$GRADLE_TARGET" "check" "./check_reports" "./test_counts_${CI_JOB_ID}.json" || true
   artifacts:
     when: always
     paths:
       - ./check_reports
+      - './test_counts_*.json'
       - '.gradle/daemon/*/*.out.log'
   retry:
     max: 2
@@ -610,6 +613,7 @@ muzzle-dep-report:
     - .gitlab/collect_results.sh
     - .gitlab/upload_ciapp.sh $CACHE_TYPE $testJvm
     - gitlab_section_end "collect-reports"
+    - .gitlab/count_tests.sh -v "$GRADLE_TARGET" "$testJvm" "./results" "./test_counts_${CI_JOB_ID}.json"
     - URL_ENCODED_JOB_NAME=$(jq -rn --arg x "$CI_JOB_NAME" '$x|@uri')
     - echo -e "${TEXT_BOLD}${TEXT_YELLOW}See test results in Datadog:${TEXT_CLEAR} https://app.datadoghq.com/ci/test/runs?query=test_level%3Atest%20%40test.service%3Add-trace-java%20%40ci.pipeline.id%3A${CI_PIPELINE_ID}%20%40ci.job.name%3A%22${URL_ENCODED_JOB_NAME}%22"
   artifacts:
@@ -618,6 +622,7 @@ muzzle-dep-report:
       - ./reports.tar
       - ./profiles.tar
       - ./results
+      - './test_counts_*.json'
       - '.gradle/daemon/*/*.out.log'
     reports:
       junit: results/*.xml
@@ -790,6 +795,25 @@ test_smoke_semeru8_debugger:
     NON_DEFAULT_JVMS: "true"
     testJvm: "semeru8"
 
+aggregate_test_counts:
+  image: ghcr.io/datadog/dd-trace-java-docker-build:${BUILDER_IMAGE_VERSION_PREFIX}base
+  stage: aggregate
+  # Note: No explicit 'needs' or 'dependencies' required
+  # By default, GitLab CI automatically downloads artifacts from ALL jobs in previous stages
+  # This job collects test_counts_*.json files from all test/check jobs via stage ordering
+  rules:
+    - if: '$POPULATE_CACHE'
+      when: never
+    - when: always
+  script:
+    - *set_datadog_api_keys
+    - .gitlab/aggregate_test_counts.sh -v
+  artifacts:
+    when: always
+    paths:
+      - test_counts_summary.json
+      - test_counts_report.md
+
 deploy_to_profiling_backend:
   stage: publish
   needs: [ build ]
diff --git a/.gitlab/aggregate_test_counts.sh b/.gitlab/aggregate_test_counts.sh
new file mode 100755
index 00000000000..eecf2afcef8
--- /dev/null
+++ b/.gitlab/aggregate_test_counts.sh
@@ -0,0 +1,271 @@
+#!/usr/bin/env bash
+
+set -e
+
+# Aggregate test counts from all test jobs using jq for all processing
+# Usage: aggregate_test_counts.sh [-v] [aggregate_dir]
+
+# https://docs.gitlab.com/ci/variables/predefined_variables/
+
+VERBOSE=0
+if [ "$1" = "-v" ]; then
+    VERBOSE=1
+    shift
+fi
+
+AGGREGATE_DIR="${1:-./test_counts_aggregate}"
+OUTPUT_FILE="test_counts_summary.json"
+REPORT_FILE="test_counts_report.md"
+
+log_verbose() {
+    if [ $VERBOSE -eq 1 ]; then
+        echo "[aggregate] $*" >&2
+    fi
+}
+
+mkdir -p "$AGGREGATE_DIR"
+
+echo "Aggregating test counts..."
+log_verbose "Pipeline ID: ${CI_PIPELINE_ID:-unknown}"
+log_verbose "Commit: ${CI_COMMIT_SHA:-unknown}"
+log_verbose "Branch: ${CI_COMMIT_BRANCH:-unknown}"
+log_verbose "Aggregate directory: $AGGREGATE_DIR"
+log_verbose "Output file: $OUTPUT_FILE"
+log_verbose "Report file: $REPORT_FILE"
+
+# Find all test count files (exclude summary to avoid reprocessing previous runs)
+log_verbose "Searching for test count files (test_counts_<jobid>.json)"
+mapfile -t COUNT_FILES < <(find . -name "test_counts_*.json" -not -name "$OUTPUT_FILE" -type f 2>/dev/null | sort)
+JOB_COUNT=${#COUNT_FILES[@]}
+log_verbose "Found $JOB_COUNT test count files"
+
+if [ $JOB_COUNT -eq 0 ]; then
+    echo "No test count files found"
+    exit 0
+fi
+
+# Validate and filter out invalid JSON files
+log_verbose "Validating JSON files"
+VALID_FILES=()
+INVALID_COUNT=0
+GITLAB_BASE_URL="${CI_PROJECT_URL}"
+
+for file in "${COUNT_FILES[@]}"; do
+    # More thorough validation: try to parse the structure we expect
+    if jq -e 'type == "object" and has("ci_job_id") and has("total_tests")' "$file" >/dev/null 2>&1; then
+        VALID_FILES+=("$file")
+    else
+        # Extract job ID from filename (e.g., test_counts_1234.json -> 1234)
+        filename=$(basename "$file")
+        job_id="${filename#test_counts_}"
+        job_id="${job_id%.json}"
+        artifact_url="${GITLAB_BASE_URL}/-/jobs/${job_id}/artifacts/file/${filename}"
+
+        echo "⚠️  WARNING: Skipping invalid/empty JSON file: $file" >&2
+        echo "   Artifact URL: $artifact_url" >&2
+        log_verbose "Invalid JSON file: $file ($artifact_url)"
+        INVALID_COUNT=$((INVALID_COUNT + 1))
+    fi
+done
+
+VALID_COUNT=${#VALID_FILES[@]}
+log_verbose "Valid files: $VALID_COUNT, Invalid files: $INVALID_COUNT"
+
+if [ $VALID_COUNT -eq 0 ]; then
+    echo "ERROR: No valid test count files found (all $JOB_COUNT files are invalid)"
+    exit 1
+fi
+
+# Use only valid files for processing
+COUNT_FILES=("${VALID_FILES[@]}")
+JOB_COUNT=$VALID_COUNT
+
+# Process ALL files with a SINGLE jq invocation
+log_verbose "Processing all files with jq"
+
+# Capture jq output and errors
+JQ_ERROR_FILE=$(mktemp)
+trap 'rm -f "$JQ_ERROR_FILE"' EXIT
+
+if ! AGGREGATED_DATA=$(jq -s '
+# Sort by base job name (before colon), then jvm_version (numeric), then test_category
+. | sort_by([
+  (.ci_job_name | split(":")[0]),
+  (if .jvm_version == "stable" then 100 elif ((.jvm_version | gsub("[^0-9]"; "")) as $nums | $nums == "") then 0 else (.jvm_version | gsub("[^0-9]"; "") | tonumber) end),
+  .test_category
+]) |
+{
+  pipeline_id: $pipeline_id,
+  commit_sha: $commit_sha,
+  branch: $branch,
+  timestamp: $timestamp,
+  test_jobs: .,
+  summary: {
+    total_tests: (map(.total_tests) | add),
+    total_passed: (map(.passed_tests) | add),
+    total_failed: (map(.failed_tests) | add),
+    total_skipped: (map(.skipped_tests) | add)
+  },
+  table_rows: map(
+    [.test_category, .jvm_version, .ci_job_name, .total_tests, .passed_tests, .failed_tests, .skipped_tests] |
+    "| \(.[0]) | \(.[1]) | \(.[2]) | \(.[3]) | \(.[4]) | \(.[5]) | \(.[6]) |"
+  ),
+  zero_test_jobs: map(
+    select(.total_tests == 0) |
+    {
+      category: .test_category,
+      jvm: .jvm_version,
+      job: .ci_job_name,
+      alert: "⚠️ **WARNING**: Zero tests in \(.test_category) on JVM \(.jvm_version) (job: \(.ci_job_name))"
+    }
+  )
+}
+' \
+  --arg pipeline_id "${CI_PIPELINE_ID:-unknown}" \
+  --arg commit_sha "${CI_COMMIT_SHA:-unknown}" \
+  --arg branch "${CI_COMMIT_BRANCH:-unknown}" \
+  --arg timestamp "$(date -u +%Y-%m-%dT%H:%M:%SZ)" \
+  "${COUNT_FILES[@]}" 2>"$JQ_ERROR_FILE"); then
+
+    # Extract problematic file from jq error message
+    ERROR_MSG=$(cat "$JQ_ERROR_FILE")
+    echo "ERROR: jq processing failed: $ERROR_MSG" >&2
+
+    # Try to extract filename and job ID from error
+    if [[ "$ERROR_MSG" =~ test_counts_([0-9]+)\.json ]]; then
+        problem_job_id="${BASH_REMATCH[1]}"
+        problem_file="./test_counts_${problem_job_id}.json"
+
+        echo "" >&2
+        echo "⚠️  Problematic file: $problem_file" >&2
+
+        if [ -n "$GITLAB_BASE_URL" ]; then
+            artifact_url="${GITLAB_BASE_URL}/-/jobs/${problem_job_id}/artifacts/file/test_counts_${problem_job_id}.json"
+            echo "   Artifact URL: $artifact_url" >&2
+        fi
+
+        # Show a snippet of the problematic file
+        if [ -f "$problem_file" ]; then
+            echo "" >&2
+            echo "File contents around the error:" >&2
+            head -20 "$problem_file" >&2
+        fi
+    fi
+
+    exit 1
+fi
+
+# Extract summary values
+TOTAL_TESTS_ALL=$(echo "$AGGREGATED_DATA" | jq -r '.summary.total_tests')
+TOTAL_PASSED_ALL=$(echo "$AGGREGATED_DATA" | jq -r '.summary.total_passed')
+TOTAL_FAILED_ALL=$(echo "$AGGREGATED_DATA" | jq -r '.summary.total_failed')
+TOTAL_SKIPPED_ALL=$(echo "$AGGREGATED_DATA" | jq -r '.summary.total_skipped')
+ZERO_TEST_COUNT=$(echo "$AGGREGATED_DATA" | jq -r '.zero_test_jobs | length')
+
+log_verbose "Overall totals: $TOTAL_TESTS_ALL tests ($TOTAL_PASSED_ALL passed, $TOTAL_FAILED_ALL failed, $TOTAL_SKIPPED_ALL skipped)"
+log_verbose "Jobs with zero tests: $ZERO_TEST_COUNT"
+
+# ANSI color codes
+BOLD='\033[1m'
+GREEN='\033[32m'
+RED='\033[31m'
+YELLOW='\033[33m'
+CYAN='\033[36m'
+GRAY='\033[90m'
+RESET='\033[0m'
+
+# Color based on values
+FAILED_COLOR=$GRAY
+SKIPPED_COLOR=$GRAY
+[ "$TOTAL_FAILED_ALL" -gt 0 ] && FAILED_COLOR=$RED
+[ "$TOTAL_SKIPPED_ALL" -gt 0 ] && SKIPPED_COLOR=$YELLOW
+
+echo ""
+echo -e "${CYAN}${BOLD}Pipeline Test Summary:${RESET}"
+echo -e "  ${BOLD}Total:${RESET}   $TOTAL_TESTS_ALL"
+echo -e "  ${GREEN}Passed:${RESET}  $TOTAL_PASSED_ALL"
+echo -e "  ${FAILED_COLOR}Failed:${RESET}  $TOTAL_FAILED_ALL"
+echo -e "  ${SKIPPED_COLOR}Skipped:${RESET} $TOTAL_SKIPPED_ALL"
+echo ""
+
+# Verbose logging for each job with artifact links
+if [ $VERBOSE -eq 1 ]; then
+    echo "$AGGREGATED_DATA" | jq -r --arg base_url "$GITLAB_BASE_URL" '.test_jobs[] |
+        "  → Job: \(.ci_job_name) | Tests: \(.total_tests) (passed: \(.passed_tests), failed: \(.failed_tests), skipped: \(.skipped_tests))\n    Artifact: \($base_url)/-/jobs/\(.ci_job_id)/artifacts/file/test_counts_\(.ci_job_id).json"' >&2
+fi
+
+# Write JSON summary (just extract the parts we need)
+log_verbose "Creating summary JSON file"
+echo "$AGGREGATED_DATA" | jq '{
+  pipeline_id,
+  commit_sha,
+  branch,
+  timestamp,
+  test_jobs,
+  summary
+}' > "$OUTPUT_FILE"
+
+echo "Summary written to $OUTPUT_FILE"
+
+# Create markdown report
+log_verbose "Generating markdown report"
+cat > "$REPORT_FILE" <<EOF
+# Test Count Report
+
+**Pipeline ID:** ${CI_PIPELINE_ID:-unknown}
+**Commit:** ${CI_COMMIT_SHA:-unknown}
+**Branch:** ${CI_COMMIT_BRANCH:-unknown}
+**Date:** $(date -u +"%Y-%m-%d %H:%M:%S UTC")
+
+## Overall Summary
+
+| Metric | Count |
+|--------|-------|
+| Total Tests | $TOTAL_TESTS_ALL |
+| Passed | $TOTAL_PASSED_ALL |
+| Failed | $TOTAL_FAILED_ALL |
+| Skipped | $TOTAL_SKIPPED_ALL |
+
+## Breakdown by Test Category and JVM Version
+
+| Test Category | JVM Version | Job Name | Total | Passed | Failed | Skipped |
+|---------------|-------------|----------|-------|--------|--------|---------|
+EOF
+
+# Extract and write table rows from jq output
+echo "$AGGREGATED_DATA" | jq -r '.table_rows[]' >> "$REPORT_FILE"
+
+cat >> "$REPORT_FILE" <<EOF
+
+## Alerts
+
+EOF
+
+# Check for zero test counts
+log_verbose "Checking for alerts"
+ALERT_COUNT=0
+
+if [ "$TOTAL_TESTS_ALL" -eq 0 ]; then
+    echo "⚠️ **CRITICAL**: No tests were executed in this pipeline!" >> "$REPORT_FILE"
+    log_verbose "ALERT: Zero tests in entire pipeline!"
+    ALERT_COUNT=$((ALERT_COUNT + 1))
+fi
+
+# Extract and write zero-test alerts from jq output
+if [ "$ZERO_TEST_COUNT" -gt 0 ]; then
+    echo "$AGGREGATED_DATA" | jq -r '.zero_test_jobs[].alert' >> "$REPORT_FILE"
+    ALERT_COUNT=$((ALERT_COUNT + ZERO_TEST_COUNT))
+
+    if [ $VERBOSE -eq 1 ]; then
+        echo "$AGGREGATED_DATA" | jq -r '.zero_test_jobs[] | "ALERT: Zero tests in job '"'"'\(.job)'"'"' (\(.category), JVM \(.jvm))"' >&2
+    fi
+fi
+
+log_verbose "Found $ALERT_COUNT alerts ($ZERO_TEST_COUNT jobs with zero tests)"
+
+echo "" >> "$REPORT_FILE"
+echo "---" >> "$REPORT_FILE"
+echo "*This report is automatically generated. See [test-coverage.md](../docs/test-coverage.md) for details.*" >> "$REPORT_FILE"
+
+echo "Report written to $REPORT_FILE"
+cat "$REPORT_FILE"
diff --git a/.gitlab/collect_results.sh b/.gitlab/collect_results.sh
index 44e3f6a81b1..ba404df9a00 100755
--- a/.gitlab/collect_results.sh
+++ b/.gitlab/collect_results.sh
@@ -55,6 +55,14 @@ echo "Saving test results:"
 while IFS= read -r -d '' RESULT_XML_FILE
 do
   echo -n "- $RESULT_XML_FILE"
+  # Assuming the path looks like that: dd-java-agent/instrumentation/tomcat/tomcat-5.5/build/test-results/forkedTest/TEST-TomcatServletV1ForkedTest.xml
+  # it will extracts 3 components from the path (counting from the end), to form the new name AGGREGATED_FILE_NAME:
+  #
+  #  1. Field 1 (from end): The XML filename itself
+  #  2. Field 2 (from end): The test suite type (test, forkedTest, etc.)
+  #  3. Field 5 (from end): The module/subproject name
+  #
+  # E.g. for the example path: tomcat-5.5_forkedTest_TEST-TomcatServletV1ForkedTest.xml
   AGGREGATED_FILE_NAME=$(echo "$RESULT_XML_FILE" | rev | cut -d "/" -f 1,2,5 | rev | tr "/" "_")
   echo -n " as $AGGREGATED_FILE_NAME"
   cp "$RESULT_XML_FILE" "$TEST_RESULTS_DIR/$AGGREGATED_FILE_NAME"
diff --git a/.gitlab/count_tests.sh b/.gitlab/count_tests.sh
new file mode 100755
index 00000000000..c06b71982e9
--- /dev/null
+++ b/.gitlab/count_tests.sh
@@ -0,0 +1,120 @@
+#!/usr/bin/env bash
+
+set -e
+
+# Count tests from JUnit XML reports and emit metrics
+# Usage: count_tests.sh [-v] <test_category> <jvm_version> <results_dir> <output_file>
+
+# https://docs.gitlab.com/ci/variables/predefined_variables/
+
+VERBOSE=0
+if [ "$1" = "-v" ]; then
+    VERBOSE=1
+    shift
+fi
+
+TEST_CATEGORY="${1:-unknown}"
+JVM_VERSION="${2:-unknown}"
+RESULTS_DIR="${3:-./results}"
+OUTPUT_FILE="${4:-./test_counts.json}"
+
+log_verbose() {
+    if [ $VERBOSE -eq 1 ]; then
+        echo "[count_tests] $*" >&2
+    fi
+}
+
+log_verbose "Job: ${CI_JOB_NAME:-unknown} (ID: ${CI_JOB_ID:-unknown})"
+log_verbose "Test category: $TEST_CATEGORY"
+log_verbose "JVM version: $JVM_VERSION"
+log_verbose "Results directory: $RESULTS_DIR"
+log_verbose "Output file: $OUTPUT_FILE"
+
+if [ ! -d "$RESULTS_DIR" ]; then
+    echo "Results directory not found: $RESULTS_DIR"
+    log_verbose "Directory does not exist, exiting"
+    exit 0
+fi
+
+# Count tests from JUnit XML files
+TOTAL_TESTS=0
+TOTAL_FAILURES=0
+TOTAL_ERRORS=0
+TOTAL_SKIPPED=0
+XML_FILE_COUNT=0
+
+echo "Counting tests in $RESULTS_DIR for $TEST_CATEGORY on JVM $JVM_VERSION"
+
+# Find all XML files and count tests
+log_verbose "Searching for XML files in $RESULTS_DIR"
+for xml_file in $(find "$RESULTS_DIR" -name "*.xml" -type f 2>/dev/null); do
+    XML_FILE_COUNT=$((XML_FILE_COUNT + 1))
+    log_verbose "Processing file $XML_FILE_COUNT: $xml_file"
+    if [ -f "$xml_file" ]; then
+        # Extract test counts from testsuite tags
+        # Handle both <testsuite> and <testsuites> formats
+        tests=$(grep -oP '(?<=tests=")[0-9]+' "$xml_file" 2>/dev/null | head -1 || echo "0")
+        failures=$(grep -oP '(?<=failures=")[0-9]+' "$xml_file" 2>/dev/null | head -1 || echo "0")
+        errors=$(grep -oP '(?<=errors=")[0-9]+' "$xml_file" 2>/dev/null | head -1 || echo "0")
+        skipped=$(grep -oP '(?<=skipped=")[0-9]+' "$xml_file" 2>/dev/null | head -1 || echo "0")
+
+        log_verbose "  → tests=$tests, failures=$failures, errors=$errors, skipped=$skipped"
+
+        TOTAL_TESTS=$((TOTAL_TESTS + tests))
+        TOTAL_FAILURES=$((TOTAL_FAILURES + failures))
+        TOTAL_ERRORS=$((TOTAL_ERRORS + errors))
+        TOTAL_SKIPPED=$((TOTAL_SKIPPED + skipped))
+    fi
+done
+
+log_verbose "Processed $XML_FILE_COUNT XML files"
+
+TOTAL_PASSED=$((TOTAL_TESTS - TOTAL_FAILURES - TOTAL_ERRORS - TOTAL_SKIPPED))
+
+# ANSI color codes
+BOLD='\033[1m'
+GREEN='\033[32m'
+RED='\033[31m'
+YELLOW='\033[33m'
+CYAN='\033[36m'
+GRAY='\033[90m'
+RESET='\033[0m'
+
+# Color based on values
+FAILED_COLOR=$GRAY
+ERRORS_COLOR=$GRAY
+SKIPPED_COLOR=$GRAY
+[ $TOTAL_FAILURES -gt 0 ] && FAILED_COLOR=$RED
+[ $TOTAL_ERRORS -gt 0 ] && ERRORS_COLOR=$RED
+[ $TOTAL_SKIPPED -gt 0 ] && SKIPPED_COLOR=$YELLOW
+
+echo -e "${CYAN}${BOLD}Test counts for $TEST_CATEGORY on JVM $JVM_VERSION:${RESET}"
+echo -e "  ${BOLD}Total:${RESET}   $TOTAL_TESTS"
+echo -e "  ${GREEN}Passed:${RESET}  $TOTAL_PASSED"
+echo -e "  ${FAILED_COLOR}Failed:${RESET}  $TOTAL_FAILURES"
+echo -e "  ${ERRORS_COLOR}Errors:${RESET}  $TOTAL_ERRORS"
+echo -e "  ${SKIPPED_COLOR}Skipped:${RESET} $TOTAL_SKIPPED"
+
+# Create JSON output
+log_verbose "Writing JSON output to $OUTPUT_FILE"
+cat > "$OUTPUT_FILE" <<EOF
+{
+  "test_category": "$TEST_CATEGORY",
+  "jvm_version": "$JVM_VERSION",
+  "ci_job_name": "${CI_JOB_NAME:-unknown}",
+  "ci_job_id": "${CI_JOB_ID:-unknown}",
+  "ci_pipeline_id": "${CI_PIPELINE_ID:-unknown}",
+  "ci_commit_sha": "${CI_COMMIT_SHA:-unknown}",
+  "ci_commit_branch": "${CI_COMMIT_BRANCH:-unknown}",
+  "ci_node_index": "${CI_NODE_INDEX:-1}",
+  "ci_node_total": "${CI_NODE_TOTAL:-1}",
+  "total_tests": $TOTAL_TESTS,
+  "passed_tests": $TOTAL_PASSED,
+  "failed_tests": $TOTAL_FAILURES,
+  "error_tests": $TOTAL_ERRORS,
+  "skipped_tests": $TOTAL_SKIPPED,
+  "timestamp": "$(date -u +%Y-%m-%dT%H:%M:%SZ)"
+}
+EOF
+
+echo "Test count output written to $OUTPUT_FILE"

From 5cdabfd9cf6e9935e3c3b96dbd20a1f96184c662 Mon Sep 17 00:00:00 2001
From: Brice Dutheil <brice.dutheil@gmail.com>
Date: Tue, 3 Feb 2026 15:17:56 +0100
Subject: [PATCH 02/15] build: Don't include check jobs

---
 .gitlab-ci.yml                   |  2 --
 .gitlab/aggregate_test_counts.sh | 38 ++++++++++++++++++++++++++++----
 2 files changed, 34 insertions(+), 6 deletions(-)

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 917607f7892..9d6ec1e20ce 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -451,12 +451,10 @@ test_published_artifacts:
     - gitlab_section_start "collect-reports" "Collecting reports"
     - .gitlab/collect_reports.sh --destination ./check_reports --move
     - gitlab_section_end "collect-reports"
-    - .gitlab/count_tests.sh -v "$GRADLE_TARGET" "check" "./check_reports" "./test_counts_${CI_JOB_ID}.json" || true
   artifacts:
     when: always
     paths:
       - ./check_reports
-      - './test_counts_*.json'
       - '.gradle/daemon/*/*.out.log'
   retry:
     max: 2
diff --git a/.gitlab/aggregate_test_counts.sh b/.gitlab/aggregate_test_counts.sh
index eecf2afcef8..46c79c1483e 100755
--- a/.gitlab/aggregate_test_counts.sh
+++ b/.gitlab/aggregate_test_counts.sh
@@ -7,6 +7,12 @@ set -e
 
 # https://docs.gitlab.com/ci/variables/predefined_variables/
 
+# Source GitLab utilities for section formatting
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+if [ -f "$SCRIPT_DIR/gitlab-utils.sh" ]; then
+    source "$SCRIPT_DIR/gitlab-utils.sh"
+fi
+
 VERBOSE=0
 if [ "$1" = "-v" ]; then
     VERBOSE=1
@@ -188,12 +194,36 @@ echo -e "  ${FAILED_COLOR}Failed:${RESET}  $TOTAL_FAILED_ALL"
 echo -e "  ${SKIPPED_COLOR}Skipped:${RESET} $TOTAL_SKIPPED_ALL"
 echo ""
 
-# Verbose logging for each job with artifact links
-if [ $VERBOSE -eq 1 ]; then
-    echo "$AGGREGATED_DATA" | jq -r --arg base_url "$GITLAB_BASE_URL" '.test_jobs[] |
-        "  → Job: \(.ci_job_name) | Tests: \(.total_tests) (passed: \(.passed_tests), failed: \(.failed_tests), skipped: \(.skipped_tests))\n    Artifact: \($base_url)/-/jobs/\(.ci_job_id)/artifacts/file/test_counts_\(.ci_job_id).json"' >&2
+# Links to Datadog CI Visibility and Test Optimization
+if [ -n "${CI_PIPELINE_ID}" ]; then
+    echo -e "${BOLD}${YELLOW}See test results in Datadog:${RESET}"
+    echo -e "  ${CYAN}CI Visibility:${RESET} https://app.datadoghq.com/ci/test/runs?query=test_level%3Atest%20%40test.service%3Add-trace-java%20%40ci.pipeline.id%3A${CI_PIPELINE_ID}"
+    echo -e "  ${CYAN}Test Optimization:${RESET} https://app.datadoghq.com/ci/settings/test-optimization?search=dd-trace-java"
+    echo ""
+fi
+
+# Display alerts in log output
+if [ "$TOTAL_TESTS_ALL" -eq 0 ] || [ "$ZERO_TEST_COUNT" -gt 0 ]; then
+    echo -e "${RED}${BOLD}Alerts:${RESET}"
+
+    if [ "$TOTAL_TESTS_ALL" -eq 0 ]; then
+        echo -e "  ${RED}🚨 CRITICAL: No tests were executed in this pipeline!${RESET}"
+    fi
+
+    if [ "$ZERO_TEST_COUNT" -gt 0 ]; then
+        echo -e "  ${YELLOW}⚠️  WARNING: $ZERO_TEST_COUNT job(s) with zero tests:${RESET}"
+        echo "$AGGREGATED_DATA" | jq -r '.zero_test_jobs[] | "    • \(.job) (\(.category), JVM \(.jvm))"'
+    fi
+
+    echo ""
 fi
 
+# Verbose logging for each job with artifact links
+gitlab_section_start "test-job-details" "Detailed Test Results by Job"
+echo "$AGGREGATED_DATA" | jq -r --arg base_url "$GITLAB_BASE_URL" '.test_jobs[] |
+    "  → Job: \(.ci_job_name) | Tests: \(.total_tests) (passed: \(.passed_tests), failed: \(.failed_tests), skipped: \(.skipped_tests))\n    Artifact: \($base_url)/-/jobs/\(.ci_job_id)/artifacts/file/test_counts_\(.ci_job_id).json"' >&2
+gitlab_section_end "test-job-details"
+
 # Write JSON summary (just extract the parts we need)
 log_verbose "Creating summary JSON file"
 echo "$AGGREGATED_DATA" | jq '{

From de2f6b5b17b9be554134e780d22b0e4be1338b05 Mon Sep 17 00:00:00 2001
From: Brice Dutheil <brice.dutheil@gmail.com>
Date: Tue, 3 Feb 2026 15:44:04 +0100
Subject: [PATCH 03/15] build: Refactor with functions

---
 .gitlab/aggregate_test_counts.sh | 554 ++++++++++++++++++-------------
 1 file changed, 327 insertions(+), 227 deletions(-)

diff --git a/.gitlab/aggregate_test_counts.sh b/.gitlab/aggregate_test_counts.sh
index 46c79c1483e..7d05be342bd 100755
--- a/.gitlab/aggregate_test_counts.sh
+++ b/.gitlab/aggregate_test_counts.sh
@@ -23,279 +23,379 @@ AGGREGATE_DIR="${1:-./test_counts_aggregate}"
 OUTPUT_FILE="test_counts_summary.json"
 REPORT_FILE="test_counts_report.md"
 
+# IMPORTANT: Heredocs in this function use <<- (with hyphen) to allow indentation in source code
+# while producing unindented output. This requires TABS (not spaces) for leading whitespace.
+# Using tabs is not a matter of preference or style; it's how the language is defined.
+
 log_verbose() {
     if [ $VERBOSE -eq 1 ]; then
         echo "[aggregate] $*" >&2
     fi
 }
 
-mkdir -p "$AGGREGATE_DIR"
+find_and_validate_test_files() {
+    local aggregate_dir="$1"
+    local output_file="$2"
 
-echo "Aggregating test counts..."
-log_verbose "Pipeline ID: ${CI_PIPELINE_ID:-unknown}"
-log_verbose "Commit: ${CI_COMMIT_SHA:-unknown}"
-log_verbose "Branch: ${CI_COMMIT_BRANCH:-unknown}"
-log_verbose "Aggregate directory: $AGGREGATE_DIR"
-log_verbose "Output file: $OUTPUT_FILE"
-log_verbose "Report file: $REPORT_FILE"
-
-# Find all test count files (exclude summary to avoid reprocessing previous runs)
-log_verbose "Searching for test count files (test_counts_<jobid>.json)"
-mapfile -t COUNT_FILES < <(find . -name "test_counts_*.json" -not -name "$OUTPUT_FILE" -type f 2>/dev/null | sort)
-JOB_COUNT=${#COUNT_FILES[@]}
-log_verbose "Found $JOB_COUNT test count files"
-
-if [ $JOB_COUNT -eq 0 ]; then
-    echo "No test count files found"
-    exit 0
-fi
+    log_verbose "Searching for test count files (test_counts_<jobid>.json)"
 
-# Validate and filter out invalid JSON files
-log_verbose "Validating JSON files"
-VALID_FILES=()
-INVALID_COUNT=0
-GITLAB_BASE_URL="${CI_PROJECT_URL}"
-
-for file in "${COUNT_FILES[@]}"; do
-    # More thorough validation: try to parse the structure we expect
-    if jq -e 'type == "object" and has("ci_job_id") and has("total_tests")' "$file" >/dev/null 2>&1; then
-        VALID_FILES+=("$file")
-    else
-        # Extract job ID from filename (e.g., test_counts_1234.json -> 1234)
-        filename=$(basename "$file")
-        job_id="${filename#test_counts_}"
-        job_id="${job_id%.json}"
-        artifact_url="${GITLAB_BASE_URL}/-/jobs/${job_id}/artifacts/file/${filename}"
-
-        echo "⚠️  WARNING: Skipping invalid/empty JSON file: $file" >&2
-        echo "   Artifact URL: $artifact_url" >&2
-        log_verbose "Invalid JSON file: $file ($artifact_url)"
-        INVALID_COUNT=$((INVALID_COUNT + 1))
-    fi
-done
+    # Find all test count files (exclude summary to avoid reprocessing previous runs)
+    local -a found_files
+    mapfile -t found_files < <(find "$aggregate_dir" -name "test_counts_*.json" -not -name "$output_file" -type f 2>/dev/null | sort)
 
-VALID_COUNT=${#VALID_FILES[@]}
-log_verbose "Valid files: $VALID_COUNT, Invalid files: $INVALID_COUNT"
+    local job_count=${#found_files[@]}
+    log_verbose "Found $job_count test count files"
 
-if [ $VALID_COUNT -eq 0 ]; then
-    echo "ERROR: No valid test count files found (all $JOB_COUNT files are invalid)"
-    exit 1
-fi
+    if [ "$job_count" -eq 0 ]; then
+        echo "No test count files found" >&2
+        return 1
+    fi
 
-# Use only valid files for processing
-COUNT_FILES=("${VALID_FILES[@]}")
-JOB_COUNT=$VALID_COUNT
-
-# Process ALL files with a SINGLE jq invocation
-log_verbose "Processing all files with jq"
-
-# Capture jq output and errors
-JQ_ERROR_FILE=$(mktemp)
-trap 'rm -f "$JQ_ERROR_FILE"' EXIT
-
-if ! AGGREGATED_DATA=$(jq -s '
-# Sort by base job name (before colon), then jvm_version (numeric), then test_category
-. | sort_by([
-  (.ci_job_name | split(":")[0]),
-  (if .jvm_version == "stable" then 100 elif ((.jvm_version | gsub("[^0-9]"; "")) as $nums | $nums == "") then 0 else (.jvm_version | gsub("[^0-9]"; "") | tonumber) end),
-  .test_category
-]) |
-{
-  pipeline_id: $pipeline_id,
-  commit_sha: $commit_sha,
-  branch: $branch,
-  timestamp: $timestamp,
-  test_jobs: .,
-  summary: {
-    total_tests: (map(.total_tests) | add),
-    total_passed: (map(.passed_tests) | add),
-    total_failed: (map(.failed_tests) | add),
-    total_skipped: (map(.skipped_tests) | add)
-  },
-  table_rows: map(
-    [.test_category, .jvm_version, .ci_job_name, .total_tests, .passed_tests, .failed_tests, .skipped_tests] |
-    "| \(.[0]) | \(.[1]) | \(.[2]) | \(.[3]) | \(.[4]) | \(.[5]) | \(.[6]) |"
-  ),
-  zero_test_jobs: map(
-    select(.total_tests == 0) |
-    {
-      category: .test_category,
-      jvm: .jvm_version,
-      job: .ci_job_name,
-      alert: "⚠️ **WARNING**: Zero tests in \(.test_category) on JVM \(.jvm_version) (job: \(.ci_job_name))"
-    }
-  )
-}
-' \
-  --arg pipeline_id "${CI_PIPELINE_ID:-unknown}" \
-  --arg commit_sha "${CI_COMMIT_SHA:-unknown}" \
-  --arg branch "${CI_COMMIT_BRANCH:-unknown}" \
-  --arg timestamp "$(date -u +%Y-%m-%dT%H:%M:%SZ)" \
-  "${COUNT_FILES[@]}" 2>"$JQ_ERROR_FILE"); then
-
-    # Extract problematic file from jq error message
-    ERROR_MSG=$(cat "$JQ_ERROR_FILE")
-    echo "ERROR: jq processing failed: $ERROR_MSG" >&2
-
-    # Try to extract filename and job ID from error
-    if [[ "$ERROR_MSG" =~ test_counts_([0-9]+)\.json ]]; then
-        problem_job_id="${BASH_REMATCH[1]}"
-        problem_file="./test_counts_${problem_job_id}.json"
-
-        echo "" >&2
-        echo "⚠️  Problematic file: $problem_file" >&2
-
-        if [ -n "$GITLAB_BASE_URL" ]; then
-            artifact_url="${GITLAB_BASE_URL}/-/jobs/${problem_job_id}/artifacts/file/test_counts_${problem_job_id}.json"
+    # Validate and filter out invalid JSON files
+    log_verbose "Validating JSON files"
+    local -a valid_files=()
+    local invalid_count=0
+    local gitlab_base_url="${CI_PROJECT_URL}"
+
+    for file in "${found_files[@]}"; do
+        # More thorough validation: try to parse the structure we expect
+        if jq -e 'type == "object" and has("ci_job_id") and has("total_tests")' "$file" >/dev/null 2>&1; then
+            valid_files+=("$file")
+        else
+            # Extract job ID from filename (e.g., test_counts_1234.json -> 1234)
+            local filename
+            filename=$(basename "$file")
+            local job_id="${filename#test_counts_}"
+            job_id="${job_id%.json}"
+            local artifact_url="${gitlab_base_url}/-/jobs/${job_id}/artifacts/file/${filename}"
+
+            echo "⚠️  WARNING: Skipping invalid/empty JSON file: $file" >&2
             echo "   Artifact URL: $artifact_url" >&2
+            log_verbose "Invalid JSON file: $file ($artifact_url)"
+            ((invalid_count++))
         fi
+    done
+
+    local valid_count=${#valid_files[@]}
+    log_verbose "Valid files: $valid_count, Invalid files: $invalid_count"
+
+    if [ "$valid_count" -eq 0 ]; then
+        echo "ERROR: No valid test count files found (all $job_count files are invalid)" >&2
+        return 1
+    fi
+
+    # Return valid files by printing them (caller will capture)
+    printf '%s\n' "${valid_files[@]}"
+    return 0
+}
+
+aggregate_test_data() {
+    local -a count_files=("$@")
+
+    log_verbose "Processing all files with jq"
+
+    # Capture jq output and errors
+    local jq_error_file
+    jq_error_file=$(mktemp)
+    trap 'rm -f "$jq_error_file"' RETURN
+
+    # Aggregate data using jq (heredoc with <<- strips leading tabs)
+    # Using tabs is not a matter of preference or style; it's how the language is defined.
+    local jq_program
+    jq_program=$(cat <<-'JQ'
+	# Sort by base job name (before colon), then jvm_version (numeric), then test_category
+	. | sort_by([
+	  (.ci_job_name | split(":")[0]),
+	  (if .jvm_version == "stable" then 100 elif ((.jvm_version | gsub("[^0-9]"; "")) as $nums | $nums == "") then 0 else (.jvm_version | gsub("[^0-9]"; "") | tonumber) end),
+	  .test_category
+	]) |
+	{
+	  pipeline_id: $pipeline_id,
+	  commit_sha: $commit_sha,
+	  branch: $branch,
+	  timestamp: $timestamp,
+	  test_jobs: .,
+	  summary: {
+	    total_tests: (map(.total_tests) | add),
+	    total_passed: (map(.passed_tests) | add),
+	    total_failed: (map(.failed_tests) | add),
+	    total_skipped: (map(.skipped_tests) | add)
+	  },
+	  table_rows: map(
+	    [.test_category, .jvm_version, .ci_job_name, .total_tests, .passed_tests, .failed_tests, .skipped_tests] |
+	    "| \(.[0]) | \(.[1]) | \(.[2]) | \(.[3]) | \(.[4]) | \(.[5]) | \(.[6]) |"
+	  ),
+	  zero_test_jobs: map(
+	    select(.total_tests == 0) |
+	    {
+	      category: .test_category,
+	      jvm: .jvm_version,
+	      job: .ci_job_name,
+	      alert: "⚠️ **WARNING**: Zero tests in \(.test_category) on JVM \(.jvm_version) (job: \(.ci_job_name))"
+	    }
+	  )
+	}
+	JQ
+    )
+
+    local aggregated_data
+    if ! aggregated_data=$(jq -s "$jq_program" \
+      --arg pipeline_id "${CI_PIPELINE_ID:-unknown}" \
+      --arg commit_sha "${CI_COMMIT_SHA:-unknown}" \
+      --arg branch "${CI_COMMIT_BRANCH:-unknown}" \
+      --arg timestamp "$(date -u +%Y-%m-%dT%H:%M:%SZ)" \
+      "${count_files[@]}" 2>"$jq_error_file"); then
+
+        # Extract problematic file from jq error message
+        local error_msg
+        error_msg=$(cat "$jq_error_file")
+        echo "ERROR: jq processing failed: $error_msg" >&2
+
+        # Try to extract filename and job ID from error
+        if [[ "$error_msg" =~ test_counts_([0-9]+)\.json ]]; then
+            local problem_job_id="${BASH_REMATCH[1]}"
+            local problem_file="./test_counts_${problem_job_id}.json"
+            local gitlab_base_url="${CI_PROJECT_URL}"
 
-        # Show a snippet of the problematic file
-        if [ -f "$problem_file" ]; then
             echo "" >&2
-            echo "File contents around the error:" >&2
-            head -20 "$problem_file" >&2
+            echo "⚠️  Problematic file: $problem_file" >&2
+
+            if [ -n "$gitlab_base_url" ]; then
+                local artifact_url="${gitlab_base_url}/-/jobs/${problem_job_id}/artifacts/file/test_counts_${problem_job_id}.json"
+                echo "   Artifact URL: $artifact_url" >&2
+            fi
+
+            # Show a snippet of the problematic file
+            if [ -f "$problem_file" ]; then
+                echo "" >&2
+                echo "File contents around the error:" >&2
+                head -20 "$problem_file" >&2
+            fi
         fi
+
+        return 1
     fi
 
-    exit 1
-fi
+    # Return aggregated data
+    echo "$aggregated_data"
+    return 0
+}
 
-# Extract summary values
-TOTAL_TESTS_ALL=$(echo "$AGGREGATED_DATA" | jq -r '.summary.total_tests')
-TOTAL_PASSED_ALL=$(echo "$AGGREGATED_DATA" | jq -r '.summary.total_passed')
-TOTAL_FAILED_ALL=$(echo "$AGGREGATED_DATA" | jq -r '.summary.total_failed')
-TOTAL_SKIPPED_ALL=$(echo "$AGGREGATED_DATA" | jq -r '.summary.total_skipped')
-ZERO_TEST_COUNT=$(echo "$AGGREGATED_DATA" | jq -r '.zero_test_jobs | length')
-
-log_verbose "Overall totals: $TOTAL_TESTS_ALL tests ($TOTAL_PASSED_ALL passed, $TOTAL_FAILED_ALL failed, $TOTAL_SKIPPED_ALL skipped)"
-log_verbose "Jobs with zero tests: $ZERO_TEST_COUNT"
-
-# ANSI color codes
-BOLD='\033[1m'
-GREEN='\033[32m'
-RED='\033[31m'
-YELLOW='\033[33m'
-CYAN='\033[36m'
-GRAY='\033[90m'
-RESET='\033[0m'
-
-# Color based on values
-FAILED_COLOR=$GRAY
-SKIPPED_COLOR=$GRAY
-[ "$TOTAL_FAILED_ALL" -gt 0 ] && FAILED_COLOR=$RED
-[ "$TOTAL_SKIPPED_ALL" -gt 0 ] && SKIPPED_COLOR=$YELLOW
-
-echo ""
-echo -e "${CYAN}${BOLD}Pipeline Test Summary:${RESET}"
-echo -e "  ${BOLD}Total:${RESET}   $TOTAL_TESTS_ALL"
-echo -e "  ${GREEN}Passed:${RESET}  $TOTAL_PASSED_ALL"
-echo -e "  ${FAILED_COLOR}Failed:${RESET}  $TOTAL_FAILED_ALL"
-echo -e "  ${SKIPPED_COLOR}Skipped:${RESET} $TOTAL_SKIPPED_ALL"
-echo ""
-
-# Links to Datadog CI Visibility and Test Optimization
-if [ -n "${CI_PIPELINE_ID}" ]; then
-    echo -e "${BOLD}${YELLOW}See test results in Datadog:${RESET}"
-    echo -e "  ${CYAN}CI Visibility:${RESET} https://app.datadoghq.com/ci/test/runs?query=test_level%3Atest%20%40test.service%3Add-trace-java%20%40ci.pipeline.id%3A${CI_PIPELINE_ID}"
-    echo -e "  ${CYAN}Test Optimization:${RESET} https://app.datadoghq.com/ci/settings/test-optimization?search=dd-trace-java"
-    echo ""
-fi
+display_summary() {
+    local aggregated_data="$1"
+
+    # Extract summary values
+    local total_tests
+    local total_passed
+    local total_failed
+    local total_skipped
+    local zero_test_count
+
+    total_tests=$(echo "$aggregated_data" | jq -r '.summary.total_tests')
+    total_passed=$(echo "$aggregated_data" | jq -r '.summary.total_passed')
+    total_failed=$(echo "$aggregated_data" | jq -r '.summary.total_failed')
+    total_skipped=$(echo "$aggregated_data" | jq -r '.summary.total_skipped')
+    zero_test_count=$(echo "$aggregated_data" | jq -r '.zero_test_jobs | length')
+
+    log_verbose "Overall totals: $total_tests tests ($total_passed passed, $total_failed failed, $total_skipped skipped)"
+    log_verbose "Jobs with zero tests: $zero_test_count"
+
+    # ANSI color codes
+    local bold='\033[1m'
+    local green='\033[32m'
+    local red='\033[31m'
+    local yellow='\033[33m'
+    local cyan='\033[36m'
+    local gray='\033[90m'
+    local reset='\033[0m'
+
+    # Color based on values
+    local failed_color=$gray
+    local skipped_color=$gray
+    [ "$total_failed" -gt 0 ] && failed_color=$red
+    [ "$total_skipped" -gt 0 ] && skipped_color=$yellow
 
-# Display alerts in log output
-if [ "$TOTAL_TESTS_ALL" -eq 0 ] || [ "$ZERO_TEST_COUNT" -gt 0 ]; then
-    echo -e "${RED}${BOLD}Alerts:${RESET}"
+    echo ""
+    echo -e "${cyan}${bold}Pipeline Test Summary:${reset}"
+    echo -e "  ${bold}Total:${reset}   $total_tests"
+    echo -e "  ${green}Passed:${reset}  $total_passed"
+    echo -e "  ${failed_color}Failed:${reset}  $total_failed"
+    echo -e "  ${skipped_color}Skipped:${reset} $total_skipped"
+    echo ""
 
-    if [ "$TOTAL_TESTS_ALL" -eq 0 ]; then
-        echo -e "  ${RED}🚨 CRITICAL: No tests were executed in this pipeline!${RESET}"
+    # Links to Datadog CI Visibility and Test Optimization
+    if [ -n "${CI_PIPELINE_ID}" ]; then
+        echo -e "${bold}${yellow}See test results in Datadog:${reset}"
+        echo -e "  ${cyan}CI Visibility:${reset} https://app.datadoghq.com/ci/test/runs?query=test_level%3Atest%20%40test.service%3Add-trace-java%20%40ci.pipeline.id%3A${CI_PIPELINE_ID}"
+        echo -e "  ${cyan}Test Optimization:${reset} https://app.datadoghq.com/ci/settings/test-optimization?search=dd-trace-java"
+        echo ""
     fi
 
-    if [ "$ZERO_TEST_COUNT" -gt 0 ]; then
-        echo -e "  ${YELLOW}⚠️  WARNING: $ZERO_TEST_COUNT job(s) with zero tests:${RESET}"
-        echo "$AGGREGATED_DATA" | jq -r '.zero_test_jobs[] | "    • \(.job) (\(.category), JVM \(.jvm))"'
+    # Display alerts in log output
+    if [ "$total_tests" -eq 0 ] || [ "$zero_test_count" -gt 0 ]; then
+        echo -e "${red}${bold}Alerts:${reset}"
+
+        if [ "$total_tests" -eq 0 ]; then
+            echo -e "  ${red}🚨 CRITICAL: No tests were executed in this pipeline!${reset}"
+        fi
+
+        if [ "$zero_test_count" -gt 0 ]; then
+            echo -e "  ${yellow}⚠️  WARNING: $zero_test_count job(s) with zero tests:${reset}"
+            echo "$aggregated_data" | jq -r '.zero_test_jobs[] | "    • \(.job) (\(.category), JVM \(.jvm))"'
+        fi
+
+        echo ""
     fi
+}
 
-    echo ""
-fi
+display_detailed_results() {
+    local aggregated_data="$1"
+    local gitlab_base_url="${CI_PROJECT_URL}"
 
-# Verbose logging for each job with artifact links
-gitlab_section_start "test-job-details" "Detailed Test Results by Job"
-echo "$AGGREGATED_DATA" | jq -r --arg base_url "$GITLAB_BASE_URL" '.test_jobs[] |
+    gitlab_section_start "test-job-details" "Detailed Test Results by Job"
+    echo "$aggregated_data" | jq -r --arg base_url "$gitlab_base_url" '.test_jobs[] |
     "  → Job: \(.ci_job_name) | Tests: \(.total_tests) (passed: \(.passed_tests), failed: \(.failed_tests), skipped: \(.skipped_tests))\n    Artifact: \($base_url)/-/jobs/\(.ci_job_id)/artifacts/file/test_counts_\(.ci_job_id).json"' >&2
-gitlab_section_end "test-job-details"
+    gitlab_section_end "test-job-details"
+}
 
-# Write JSON summary (just extract the parts we need)
-log_verbose "Creating summary JSON file"
-echo "$AGGREGATED_DATA" | jq '{
-  pipeline_id,
-  commit_sha,
-  branch,
-  timestamp,
-  test_jobs,
-  summary
-}' > "$OUTPUT_FILE"
+write_json_summary() {
+    local aggregated_data="$1"
+    local output_file="$2"
+
+    log_verbose "Creating summary JSON file"
+    echo "$aggregated_data" | jq '{
+      pipeline_id,
+      commit_sha,
+      branch,
+      timestamp,
+      test_jobs,
+      summary
+    }' > "$output_file"
+    echo "Summary written to $output_file"
+}
 
-echo "Summary written to $OUTPUT_FILE"
+write_markdown_report() {
+    local aggregated_data="$1"
+    local report_file="$2"
 
-# Create markdown report
-log_verbose "Generating markdown report"
-cat > "$REPORT_FILE" <<EOF
-# Test Count Report
+    log_verbose "Generating markdown report"
 
-**Pipeline ID:** ${CI_PIPELINE_ID:-unknown}
-**Commit:** ${CI_COMMIT_SHA:-unknown}
-**Branch:** ${CI_COMMIT_BRANCH:-unknown}
-**Date:** $(date -u +"%Y-%m-%d %H:%M:%S UTC")
+    # Extract summary values
+    local total_tests
+    local total_passed
+    local total_failed
+    local total_skipped
+    local zero_test_count
 
-## Overall Summary
+    total_tests=$(echo "$aggregated_data" | jq -r '.summary.total_tests')
+    total_passed=$(echo "$aggregated_data" | jq -r '.summary.total_passed')
+    total_failed=$(echo "$aggregated_data" | jq -r '.summary.total_failed')
+    total_skipped=$(echo "$aggregated_data" | jq -r '.summary.total_skipped')
+    zero_test_count=$(echo "$aggregated_data" | jq -r '.zero_test_jobs | length')
 
-| Metric | Count |
-|--------|-------|
-| Total Tests | $TOTAL_TESTS_ALL |
-| Passed | $TOTAL_PASSED_ALL |
-| Failed | $TOTAL_FAILED_ALL |
-| Skipped | $TOTAL_SKIPPED_ALL |
+    # IMPORTANT: Heredocs in this function use <<- (with hyphen) to allow indentation in source code
+    # while producing unindented output. This requires TABS (not spaces) for leading whitespace.
+    # Using tabs is not a matter of preference or style; it's how the language is defined.
 
-## Breakdown by Test Category and JVM Version
+    # Write report header and summary table
+    cat > "$report_file" <<-EOF  # <<- strips leading tabs from heredoc content
+	# Test Count Report
 
-| Test Category | JVM Version | Job Name | Total | Passed | Failed | Skipped |
-|---------------|-------------|----------|-------|--------|--------|---------|
-EOF
+	**Pipeline ID:** ${CI_PIPELINE_ID:-unknown}
+	**Commit:** ${CI_COMMIT_SHA:-unknown}
+	**Branch:** ${CI_COMMIT_BRANCH:-unknown}
+	**Date:** $(date -u +"%Y-%m-%d %H:%M:%S UTC")
 
-# Extract and write table rows from jq output
-echo "$AGGREGATED_DATA" | jq -r '.table_rows[]' >> "$REPORT_FILE"
+	## Overall Summary
 
-cat >> "$REPORT_FILE" <<EOF
+	| Metric | Count |
+	|--------|-------|
+	| Total Tests | $total_tests |
+	| Passed | $total_passed |
+	| Failed | $total_failed |
+	| Skipped | $total_skipped |
 
-## Alerts
+	## Breakdown by Test Category and JVM Version
 
-EOF
+	| Test Category | JVM Version | Job Name | Total | Passed | Failed | Skipped |
+	|---------------|-------------|----------|-------|--------|--------|---------|
+	EOF
 
-# Check for zero test counts
-log_verbose "Checking for alerts"
-ALERT_COUNT=0
+    # Write table rows
+    echo "$aggregated_data" | jq -r '.table_rows[]' >> "$report_file"
 
-if [ "$TOTAL_TESTS_ALL" -eq 0 ]; then
-    echo "⚠️ **CRITICAL**: No tests were executed in this pipeline!" >> "$REPORT_FILE"
-    log_verbose "ALERT: Zero tests in entire pipeline!"
-    ALERT_COUNT=$((ALERT_COUNT + 1))
-fi
+    # Write alerts section
+    cat >> "$report_file" <<-EOF  # <<- strips leading tabs
 
-# Extract and write zero-test alerts from jq output
-if [ "$ZERO_TEST_COUNT" -gt 0 ]; then
-    echo "$AGGREGATED_DATA" | jq -r '.zero_test_jobs[].alert' >> "$REPORT_FILE"
-    ALERT_COUNT=$((ALERT_COUNT + ZERO_TEST_COUNT))
+	## Alerts
 
-    if [ $VERBOSE -eq 1 ]; then
-        echo "$AGGREGATED_DATA" | jq -r '.zero_test_jobs[] | "ALERT: Zero tests in job '"'"'\(.job)'"'"' (\(.category), JVM \(.jvm))"' >&2
+	EOF
+
+    local alert_count=0
+
+    # Check for critical alert (no tests in entire pipeline)
+    if [ "$total_tests" -eq 0 ]; then
+        echo "⚠️ **CRITICAL**: No tests were executed in this pipeline!" >> "$report_file"
+        log_verbose "ALERT: Zero tests in entire pipeline!"
+        alert_count=$((alert_count + 1))
+    fi
+
+    # Check for zero-test job alerts
+    if [ "$zero_test_count" -gt 0 ]; then
+        echo "$aggregated_data" | jq -r '.zero_test_jobs[].alert' >> "$report_file"
+        alert_count=$((alert_count + zero_test_count))
+
+        if [ $VERBOSE -eq 1 ]; then
+            echo "$aggregated_data" | jq -r '.zero_test_jobs[] | "ALERT: Zero tests in job '"'"'\(.job)'"'"' (\(.category), JVM \(.jvm))"' >&2
+        fi
     fi
+
+    log_verbose "Found $alert_count alerts ($zero_test_count jobs with zero tests)"
+
+    # Write report footer
+    cat >> "$report_file" <<-EOF  # <<- strips leading tabs
+
+	---
+	*This report is automatically generated. See [test-coverage.md](../docs/test-coverage.md) for details.*
+	EOF
+
+    echo "Report written to $report_file"
+}
+
+mkdir -p "$AGGREGATE_DIR"
+
+echo "Aggregating test counts..."
+
+# Log configuration details
+while IFS= read -r line; do
+    log_verbose "$line"
+done <<-EOF  # <<- strips leading tabs
+	Pipeline ID: ${CI_PIPELINE_ID:-unknown}
+	Commit: ${CI_COMMIT_SHA:-unknown}
+	Branch: ${CI_COMMIT_BRANCH:-unknown}
+	Aggregate directory: $AGGREGATE_DIR
+	Output file: $OUTPUT_FILE
+	Report file: $REPORT_FILE
+	EOF
+
+# Find and validate test count files
+mapfile -t VALID_FILES < <(find_and_validate_test_files "." "$OUTPUT_FILE")
+if [ ${#VALID_FILES[@]} -eq 0 ]; then
+    exit 0
+fi
+
+# Aggregate test data
+if ! AGGREGATED_DATA=$(aggregate_test_data "${VALID_FILES[@]}"); then
+    exit 1
 fi
 
-log_verbose "Found $ALERT_COUNT alerts ($ZERO_TEST_COUNT jobs with zero tests)"
+# Display summary and alerts in log
+display_summary "$AGGREGATED_DATA"
 
-echo "" >> "$REPORT_FILE"
-echo "---" >> "$REPORT_FILE"
-echo "*This report is automatically generated. See [test-coverage.md](../docs/test-coverage.md) for details.*" >> "$REPORT_FILE"
+# Display detailed results in collapsible section
+display_detailed_results "$AGGREGATED_DATA"
 
-echo "Report written to $REPORT_FILE"
+# Write reports
+write_json_summary "$AGGREGATED_DATA" "$OUTPUT_FILE"
+write_markdown_report "$AGGREGATED_DATA" "$REPORT_FILE"
 cat "$REPORT_FILE"

From 7267f0907ef9ab903d5116f663ed937b8a74fbb6 Mon Sep 17 00:00:00 2001
From: Brice Dutheil <brice.dutheil@gmail.com>
Date: Wed, 4 Feb 2026 11:18:58 +0100
Subject: [PATCH 04/15] chore: Remove verbose output

---
 .gitlab-ci.yml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 9d6ec1e20ce..6ef5856028d 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -611,7 +611,7 @@ muzzle-dep-report:
     - .gitlab/collect_results.sh
     - .gitlab/upload_ciapp.sh $CACHE_TYPE $testJvm
     - gitlab_section_end "collect-reports"
-    - .gitlab/count_tests.sh -v "$GRADLE_TARGET" "$testJvm" "./results" "./test_counts_${CI_JOB_ID}.json"
+    - .gitlab/count_tests.sh "$GRADLE_TARGET" "$testJvm" "./results" "./test_counts_${CI_JOB_ID}.json"
     - URL_ENCODED_JOB_NAME=$(jq -rn --arg x "$CI_JOB_NAME" '$x|@uri')
     - echo -e "${TEXT_BOLD}${TEXT_YELLOW}See test results in Datadog:${TEXT_CLEAR} https://app.datadoghq.com/ci/test/runs?query=test_level%3Atest%20%40test.service%3Add-trace-java%20%40ci.pipeline.id%3A${CI_PIPELINE_ID}%20%40ci.job.name%3A%22${URL_ENCODED_JOB_NAME}%22"
   artifacts:
@@ -805,7 +805,7 @@ aggregate_test_counts:
     - when: always
   script:
     - *set_datadog_api_keys
-    - .gitlab/aggregate_test_counts.sh -v
+    - .gitlab/aggregate_test_counts.sh
   artifacts:
     when: always
     paths:

From 56ad8f0e6113aa294fb2fb87be8f899f96bdb86a Mon Sep 17 00:00:00 2001
From: Brice Dutheil <brice.dutheil@gmail.com>
Date: Wed, 4 Feb 2026 11:19:19 +0100
Subject: [PATCH 05/15] chore: Add a breakdown by JVM and test job kind

---
 .gitlab/aggregate_test_counts.sh | 103 ++++++++++++++++++++++++++++++-
 1 file changed, 101 insertions(+), 2 deletions(-)

diff --git a/.gitlab/aggregate_test_counts.sh b/.gitlab/aggregate_test_counts.sh
index 7d05be342bd..263f75f8244 100755
--- a/.gitlab/aggregate_test_counts.sh
+++ b/.gitlab/aggregate_test_counts.sh
@@ -121,6 +121,39 @@ aggregate_test_data() {
 	    total_failed: (map(.failed_tests) | add),
 	    total_skipped: (map(.skipped_tests) | add)
 	  },
+	  by_jvm: (group_by(.jvm_version) | map({
+	    jvm_version: .[0].jvm_version,
+	    total_tests: (map(.total_tests) | add),
+	    total_passed: (map(.passed_tests) | add),
+	    total_failed: (map(.failed_tests) | add),
+	    total_skipped: (map(.skipped_tests) | add),
+	    job_count: length
+	  }) | sort_by(
+	    if .jvm_version == "stable" then 100
+	    elif ((.jvm_version | gsub("[^0-9]"; "")) as $nums | $nums == "") then 0
+	    else (.jvm_version | gsub("[^0-9]"; "") | tonumber)
+	    end
+	  )),
+	  by_job_kind_and_jvm: (
+	    # Extract base job name (before the matrix suffix like ": [8, 2/6]")
+	    map(. + {job_kind: (.ci_job_name | split(":")[0])}) |
+	    group_by([.job_kind, .jvm_version]) |
+	    map({
+	      job_kind: .[0].job_kind,
+	      jvm_version: .[0].jvm_version,
+	      total_tests: (map(.total_tests) | add),
+	      total_passed: (map(.passed_tests) | add),
+	      total_failed: (map(.failed_tests) | add),
+	      total_skipped: (map(.skipped_tests) | add),
+	      split_count: length
+	    }) | sort_by([
+	      .job_kind,
+	      (if .jvm_version == "stable" then 100
+	       elif ((.jvm_version | gsub("[^0-9]"; "")) as $nums | $nums == "") then 0
+	       else (.jvm_version | gsub("[^0-9]"; "") | tonumber)
+	       end)
+	    ])
+	  ),
 	  table_rows: map(
 	    [.test_category, .jvm_version, .ci_job_name, .total_tests, .passed_tests, .failed_tests, .skipped_tests] |
 	    "| \(.[0]) | \(.[1]) | \(.[2]) | \(.[3]) | \(.[4]) | \(.[5]) | \(.[6]) |"
@@ -248,6 +281,42 @@ display_summary() {
     fi
 }
 
+display_jvm_breakdown() {
+    local aggregated_data="$1"
+
+    gitlab_section_start "test-jvm-breakdown" "Test Breakdown by JVM Version"
+
+    # Header
+    printf "%-15s %12s %12s %12s %12s %10s\n" "JVM Version" "Total Tests" "Passed" "Failed" "Skipped" "Jobs" >&2
+    printf "%-15s %12s %12s %12s %12s %10s\n" "---------------" "------------" "------------" "------------" "------------" "----------" >&2
+
+    # Data rows
+    echo "$aggregated_data" | jq -r '.by_jvm[] |
+    "\(.jvm_version)|\(.total_tests)|\(.total_passed)|\(.total_failed)|\(.total_skipped)|\(.job_count)"' | while IFS='|' read -r jvm total passed failed skipped jobs; do
+        printf "%-15s %12s %12s %12s %12s %10s\n" "$jvm" "$total" "$passed" "$failed" "$skipped" "$jobs" >&2
+    done
+
+    gitlab_section_end "test-jvm-breakdown"
+}
+
+display_job_kind_breakdown() {
+    local aggregated_data="$1"
+
+    gitlab_section_start "test-job-kind-breakdown" "Test Breakdown by Job Kind and JVM (ignoring splits)"
+
+    # Header
+    printf "%-40s %-15s %12s %12s %12s %12s %8s\n" "Job Kind" "JVM" "Total Tests" "Passed" "Failed" "Skipped" "Splits" >&2
+    printf "%-40s %-15s %12s %12s %12s %12s %8s\n" "----------------------------------------" "---------------" "------------" "------------" "------------" "------------" "--------" >&2
+
+    # Data rows
+    echo "$aggregated_data" | jq -r '.by_job_kind_and_jvm[] |
+    "\(.job_kind)|\(.jvm_version)|\(.total_tests)|\(.total_passed)|\(.total_failed)|\(.total_skipped)|\(.split_count)"' | while IFS='|' read -r job_kind jvm total passed failed skipped splits; do
+        printf "%-40s %-15s %12s %12s %12s %12s %8s\n" "$job_kind" "$jvm" "$total" "$passed" "$failed" "$skipped" "$splits" >&2
+    done
+
+    gitlab_section_end "test-job-kind-breakdown"
+}
+
 display_detailed_results() {
     local aggregated_data="$1"
     local gitlab_base_url="${CI_PROJECT_URL}"
@@ -269,7 +338,9 @@ write_json_summary() {
       branch,
       timestamp,
       test_jobs,
-      summary
+      summary,
+      by_jvm,
+      by_job_kind_and_jvm
     }' > "$output_file"
     echo "Summary written to $output_file"
 }
@@ -315,7 +386,31 @@ write_markdown_report() {
 	| Failed | $total_failed |
 	| Skipped | $total_skipped |
 
-	## Breakdown by Test Category and JVM Version
+	## Breakdown by JVM Version
+
+	| JVM Version | Total Tests | Passed | Failed | Skipped | Job Count |
+	|-------------|-------------|--------|--------|---------|-----------|
+	EOF
+
+    # Write JVM breakdown rows
+    echo "$aggregated_data" | jq -r '.by_jvm[] | "| \(.jvm_version) | \(.total_tests) | \(.total_passed) | \(.total_failed) | \(.total_skipped) | \(.job_count) |"' >> "$report_file"
+
+    # Write job kind breakdown section
+    cat >> "$report_file" <<-EOF
+
+	## Breakdown by Job Kind and JVM Version
+
+	| Job Kind | JVM Version | Total Tests | Passed | Failed | Skipped | Splits |
+	|----------|-------------|-------------|--------|--------|---------|--------|
+	EOF
+
+    # Write job kind breakdown rows
+    echo "$aggregated_data" | jq -r '.by_job_kind_and_jvm[] | "| \(.job_kind) | \(.jvm_version) | \(.total_tests) | \(.total_passed) | \(.total_failed) | \(.total_skipped) | \(.split_count) |"' >> "$report_file"
+
+    # Write detailed breakdown section
+    cat >> "$report_file" <<-EOF
+
+	## Detailed Breakdown by Test Category and JVM Version
 
 	| Test Category | JVM Version | Job Name | Total | Passed | Failed | Skipped |
 	|---------------|-------------|----------|-------|--------|--------|---------|
@@ -392,6 +487,10 @@ fi
 # Display summary and alerts in log
 display_summary "$AGGREGATED_DATA"
 
+# Display breakdowns in collapsible sections
+display_jvm_breakdown "$AGGREGATED_DATA"
+display_job_kind_breakdown "$AGGREGATED_DATA"
+
 # Display detailed results in collapsible section
 display_detailed_results "$AGGREGATED_DATA"
 

From c2946c55bdc055020e00c0b014b25d9116667e99 Mon Sep 17 00:00:00 2001
From: Brice Dutheil <brice.dutheil@gmail.com>
Date: Wed, 4 Feb 2026 12:54:47 +0100
Subject: [PATCH 06/15] chore: Contribute DD tags for test jobs

---
 .gitlab-ci.yml | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 6ef5856028d..232317c35a7 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -590,6 +590,8 @@ muzzle-dep-report:
       when: on_success
   script:
     - *gitlab_base_ref_params
+    - export JOB_BASE_NAME="${CI_JOB_NAME%%:*}"
+    - export DD_TAGS="test.configuration.jvm:${testJvm},test.configuration.split:${CI_NODE_INDEX}/${CI_NODE_TOTAL},test.configuration.job_name:${JOB_BASE_NAME}"
     - >
       if [ "$PROFILE_TESTS" == "true" ] && [ "$testJvm" != "ibm8" ] && [ "$testJvm" != "oracle8" ];
       then

From e6e349ecc8190b127dd26150a4aa36e6ba7a6285 Mon Sep 17 00:00:00 2001
From: Brice Dutheil <brice.dutheil@gmail.com>
Date: Wed, 4 Feb 2026 14:48:42 +0100
Subject: [PATCH 07/15] chore: Contribute DD tags for test jobs (2)

---
 .gitlab/upload_ciapp.sh | 12 ++++++++++++
 1 file changed, 12 insertions(+)

diff --git a/.gitlab/upload_ciapp.sh b/.gitlab/upload_ciapp.sh
index ecf0388813f..dbba1138938 100755
--- a/.gitlab/upload_ciapp.sh
+++ b/.gitlab/upload_ciapp.sh
@@ -21,6 +21,17 @@ java_prop() {
 junit_upload() {
     # based on tracer implementation: https://github.com/DataDog/dd-trace-java/blob/master/dd-java-agent/agent-bootstrap/src/main/java/datadog/trace/bootstrap/instrumentation/decorator/TestDecorator.java#L55-L77
     # Overwriting the tag with the GitHub repo URL instead of the GitLab one. Otherwise, some Test Optimization features won't work.
+
+    # Parse DD_TAGS environment variable and convert to --tags arguments
+    local dd_tags_args=""
+    if [ -n "$DD_TAGS" ]; then
+        # Split DD_TAGS by comma and create --tags argument for each
+        IFS=',' read -ra TAG_ARRAY <<< "$DD_TAGS"
+        for tag in "${TAG_ARRAY[@]}"; do
+            dd_tags_args="$dd_tags_args --tags \"$tag\""
+        done
+    fi
+
     DD_API_KEY=$1 \
         datadog-ci junit upload --service $SERVICE_NAME \
         --logs \
@@ -32,6 +43,7 @@ junit_upload() {
         --tags "os.platform:$(java_prop os.name)" \
         --tags "os.version:$(java_prop os.version)" \
         --tags "git.repository_url:https://github.com/DataDog/dd-trace-java" \
+        $dd_tags_args \
         ./results
 }
 

From 31ce8e9619f6fe8953e96a2caa1528eff794b4a0 Mon Sep 17 00:00:00 2001
From: Brice Dutheil <brice.dutheil@gmail.com>
Date: Wed, 4 Feb 2026 14:49:54 +0100
Subject: [PATCH 08/15] chore: Do not cat markdown report unless `-v` is passed

---
 .gitlab/aggregate_test_counts.sh | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/.gitlab/aggregate_test_counts.sh b/.gitlab/aggregate_test_counts.sh
index 263f75f8244..5d72d3d0576 100755
--- a/.gitlab/aggregate_test_counts.sh
+++ b/.gitlab/aggregate_test_counts.sh
@@ -497,4 +497,8 @@ display_detailed_results "$AGGREGATED_DATA"
 # Write reports
 write_json_summary "$AGGREGATED_DATA" "$OUTPUT_FILE"
 write_markdown_report "$AGGREGATED_DATA" "$REPORT_FILE"
-cat "$REPORT_FILE"
+
+# Only output markdown report in verbose mode
+if [ $VERBOSE -eq 1 ]; then
+    cat "$REPORT_FILE"
+fi

From cab481012c308713351fb4c9e940d775e7a3a48e Mon Sep 17 00:00:00 2001
From: Brice Dutheil <brice.dutheil@gmail.com>
Date: Wed, 4 Feb 2026 16:17:59 +0100
Subject: [PATCH 09/15] fix: Array-based tags for tag propagation

---
 .gitlab/upload_ciapp.sh | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

diff --git a/.gitlab/upload_ciapp.sh b/.gitlab/upload_ciapp.sh
index dbba1138938..c54f838ee3d 100755
--- a/.gitlab/upload_ciapp.sh
+++ b/.gitlab/upload_ciapp.sh
@@ -23,12 +23,13 @@ junit_upload() {
     # Overwriting the tag with the GitHub repo URL instead of the GitLab one. Otherwise, some Test Optimization features won't work.
 
     # Parse DD_TAGS environment variable and convert to --tags arguments
-    local dd_tags_args=""
+    # Using bash array for proper argument quoting
+    local dd_tags_args=()
     if [ -n "$DD_TAGS" ]; then
         # Split DD_TAGS by comma and create --tags argument for each
         IFS=',' read -ra TAG_ARRAY <<< "$DD_TAGS"
         for tag in "${TAG_ARRAY[@]}"; do
-            dd_tags_args="$dd_tags_args --tags \"$tag\""
+            dd_tags_args+=(--tags "$tag")
         done
     fi
 
@@ -43,7 +44,7 @@ junit_upload() {
         --tags "os.platform:$(java_prop os.name)" \
         --tags "os.version:$(java_prop os.version)" \
         --tags "git.repository_url:https://github.com/DataDog/dd-trace-java" \
-        $dd_tags_args \
+        "${dd_tags_args[@]}" \
         ./results
 }
 

From 5f2aa0beb8d87b4ddc8bb676ff87335eaa72dde4 Mon Sep 17 00:00:00 2001
From: Brice Dutheil <brice.dutheil@gmail.com>
Date: Wed, 4 Feb 2026 18:27:56 +0100
Subject: [PATCH 10/15] chore: Debug log

---
 .gitlab/upload_ciapp.sh | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/.gitlab/upload_ciapp.sh b/.gitlab/upload_ciapp.sh
index c54f838ee3d..91a8243c4a1 100755
--- a/.gitlab/upload_ciapp.sh
+++ b/.gitlab/upload_ciapp.sh
@@ -33,6 +33,10 @@ junit_upload() {
         done
     fi
 
+    # Debug logging
+    echo "DEBUG: DD_TAGS=$DD_TAGS"
+    echo "DEBUG: Executing datadog-ci with trace enabled:"
+    set -x
     DD_API_KEY=$1 \
         datadog-ci junit upload --service $SERVICE_NAME \
         --logs \
@@ -46,6 +50,7 @@ junit_upload() {
         --tags "git.repository_url:https://github.com/DataDog/dd-trace-java" \
         "${dd_tags_args[@]}" \
         ./results
+    set +x
 }
 
 # Upload code coverage results to Datadog

From f977ad5352742c9d36a2efda63f31283795df93a Mon Sep 17 00:00:00 2001
From: Brice Dutheil <brice.dutheil@gmail.com>
Date: Wed, 4 Feb 2026 18:41:32 +0100
Subject: [PATCH 11/15] chore: Sort breakdown

---
 .gitlab/aggregate_test_counts.sh | 29 +++++++++++++++++++++--------
 1 file changed, 21 insertions(+), 8 deletions(-)

diff --git a/.gitlab/aggregate_test_counts.sh b/.gitlab/aggregate_test_counts.sh
index 5d72d3d0576..870e09cac86 100755
--- a/.gitlab/aggregate_test_counts.sh
+++ b/.gitlab/aggregate_test_counts.sh
@@ -1,4 +1,5 @@
 #!/usr/bin/env bash
+# spotless:off - This file uses heredocs with <<- which require tabs, not spaces
 
 set -e
 
@@ -128,12 +129,17 @@ aggregate_test_data() {
 	    total_failed: (map(.failed_tests) | add),
 	    total_skipped: (map(.skipped_tests) | add),
 	    job_count: length
-	  }) | sort_by(
-	    if .jvm_version == "stable" then 100
-	    elif ((.jvm_version | gsub("[^0-9]"; "")) as $nums | $nums == "") then 0
-	    else (.jvm_version | gsub("[^0-9]"; "") | tonumber)
-	    end
-	  )),
+	  }) | sort_by([
+	    # First: numeric value (8, 11, 17, 21, 25)
+	    (if .jvm_version == "stable" then 1000
+	     elif ((.jvm_version | gsub("[^0-9]"; "")) as $nums | $nums == "") then 0
+	     else (.jvm_version | gsub("[^0-9]"; "") | tonumber)
+	     end),
+	    # Second: plain numbers before prefixed versions (8 before semeru8, 17 before graalvm17)
+	    (if (.jvm_version | test("^[0-9]+$")) then 0 else 1 end),
+	    # Third: alphabetically by full name
+	    .jvm_version
+	  ])),
 	  by_job_kind_and_jvm: (
 	    # Extract base job name (before the matrix suffix like ": [8, 2/6]")
 	    map(. + {job_kind: (.ci_job_name | split(":")[0])}) |
@@ -148,10 +154,15 @@ aggregate_test_data() {
 	      split_count: length
 	    }) | sort_by([
 	      .job_kind,
-	      (if .jvm_version == "stable" then 100
+	      # First: numeric value
+	      (if .jvm_version == "stable" then 1000
 	       elif ((.jvm_version | gsub("[^0-9]"; "")) as $nums | $nums == "") then 0
 	       else (.jvm_version | gsub("[^0-9]"; "") | tonumber)
-	       end)
+	       end),
+	      # Second: plain numbers before prefixed versions
+	      (if (.jvm_version | test("^[0-9]+$")) then 0 else 1 end),
+	      # Third: alphabetically by full name
+	      .jvm_version
 	    ])
 	  ),
 	  table_rows: map(
@@ -502,3 +513,5 @@ write_markdown_report "$AGGREGATED_DATA" "$REPORT_FILE"
 if [ $VERBOSE -eq 1 ]; then
     cat "$REPORT_FILE"
 fi
+
+# spotless:on

From f8aa4acff888f376272e4c4a24f0f3428a7b3d0a Mon Sep 17 00:00:00 2001
From: Brice Dutheil <brice.dutheil@gmail.com>
Date: Wed, 4 Feb 2026 18:49:00 +0100
Subject: [PATCH 12/15] chore: after script is executed in a different shell
 and don't see env var in before script

---
 .gitlab-ci.yml          | 2 ++
 .gitlab/upload_ciapp.sh | 2 --
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 232317c35a7..4fea46a89d6 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -607,6 +607,8 @@ muzzle-dep-report:
     - *container_info
     - *cgroup_info
     - source .gitlab/gitlab-utils.sh
+    - export JOB_BASE_NAME="${CI_JOB_NAME%%:*}"
+    - export DD_TAGS="test.configuration.jvm:${testJvm},test.configuration.split:${CI_NODE_INDEX}/${CI_NODE_TOTAL},test.configuration.job_name:${JOB_BASE_NAME}"
     - gitlab_section_start "collect-reports" "Collecting reports"
     - .gitlab/collect_reports.sh
     - if [ "$PROFILE_TESTS" == "true" ]; then .gitlab/collect_profiles.sh; fi
diff --git a/.gitlab/upload_ciapp.sh b/.gitlab/upload_ciapp.sh
index 91a8243c4a1..3ae6ae6719c 100755
--- a/.gitlab/upload_ciapp.sh
+++ b/.gitlab/upload_ciapp.sh
@@ -36,7 +36,6 @@ junit_upload() {
     # Debug logging
     echo "DEBUG: DD_TAGS=$DD_TAGS"
     echo "DEBUG: Executing datadog-ci with trace enabled:"
-    set -x
     DD_API_KEY=$1 \
         datadog-ci junit upload --service $SERVICE_NAME \
         --logs \
@@ -50,7 +49,6 @@ junit_upload() {
         --tags "git.repository_url:https://github.com/DataDog/dd-trace-java" \
         "${dd_tags_args[@]}" \
         ./results
-    set +x
 }
 
 # Upload code coverage results to Datadog

From 506d18b6a22d856f3afd1de7b8643d0b9a1ac25d Mon Sep 17 00:00:00 2001
From: Brice Dutheil <brice.dutheil@gmail.com>
Date: Wed, 4 Feb 2026 18:57:45 +0100
Subject: [PATCH 13/15] chore: collect data in upload_ciapp.sh directly

---
 .gitlab-ci.yml          |  4 ----
 .gitlab/upload_ciapp.sh | 29 ++++++++++++++++-------------
 2 files changed, 16 insertions(+), 17 deletions(-)

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 4fea46a89d6..6ef5856028d 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -590,8 +590,6 @@ muzzle-dep-report:
       when: on_success
   script:
     - *gitlab_base_ref_params
-    - export JOB_BASE_NAME="${CI_JOB_NAME%%:*}"
-    - export DD_TAGS="test.configuration.jvm:${testJvm},test.configuration.split:${CI_NODE_INDEX}/${CI_NODE_TOTAL},test.configuration.job_name:${JOB_BASE_NAME}"
     - >
       if [ "$PROFILE_TESTS" == "true" ] && [ "$testJvm" != "ibm8" ] && [ "$testJvm" != "oracle8" ];
       then
@@ -607,8 +605,6 @@ muzzle-dep-report:
     - *container_info
     - *cgroup_info
     - source .gitlab/gitlab-utils.sh
-    - export JOB_BASE_NAME="${CI_JOB_NAME%%:*}"
-    - export DD_TAGS="test.configuration.jvm:${testJvm},test.configuration.split:${CI_NODE_INDEX}/${CI_NODE_TOTAL},test.configuration.job_name:${JOB_BASE_NAME}"
     - gitlab_section_start "collect-reports" "Collecting reports"
     - .gitlab/collect_reports.sh
     - if [ "$PROFILE_TESTS" == "true" ]; then .gitlab/collect_profiles.sh; fi
diff --git a/.gitlab/upload_ciapp.sh b/.gitlab/upload_ciapp.sh
index 3ae6ae6719c..940e22770db 100755
--- a/.gitlab/upload_ciapp.sh
+++ b/.gitlab/upload_ciapp.sh
@@ -3,6 +3,8 @@ SERVICE_NAME="dd-trace-java"
 CACHE_TYPE=$1
 TEST_JVM=$2
 
+# CI_JOB_NAME, CI_NODE_INDEX, and CI_NODE_TOTAL are read from GitLab CI environment
+
 # JAVA_???_HOME are set in the base image for each used JDK https://github.com/DataDog/dd-trace-java-docker-build/blob/master/Dockerfile#L86
 JAVA_HOME="JAVA_${TEST_JVM}_HOME"
 JAVA_BIN="${!JAVA_HOME}/bin/java"
@@ -22,20 +24,21 @@ junit_upload() {
     # based on tracer implementation: https://github.com/DataDog/dd-trace-java/blob/master/dd-java-agent/agent-bootstrap/src/main/java/datadog/trace/bootstrap/instrumentation/decorator/TestDecorator.java#L55-L77
     # Overwriting the tag with the GitHub repo URL instead of the GitLab one. Otherwise, some Test Optimization features won't work.
 
-    # Parse DD_TAGS environment variable and convert to --tags arguments
-    # Using bash array for proper argument quoting
-    local dd_tags_args=()
-    if [ -n "$DD_TAGS" ]; then
-        # Split DD_TAGS by comma and create --tags argument for each
-        IFS=',' read -ra TAG_ARRAY <<< "$DD_TAGS"
-        for tag in "${TAG_ARRAY[@]}"; do
-            dd_tags_args+=(--tags "$tag")
-        done
+    # Build custom tags array directly from arguments
+    local custom_tags_args=()
+
+    # Extract job base name from CI_JOB_NAME (strip matrix suffix)
+    local job_base_name="${CI_JOB_NAME%%:*}"
+
+    # Add custom test configuration tags
+    custom_tags_args+=(--tags "test.configuration.jvm:${TEST_JVM}")
+    if [ -n "$CI_NODE_INDEX" ] && [ -n "$CI_NODE_TOTAL" ]; then
+        custom_tags_args+=(--tags "test.configuration.split:${CI_NODE_INDEX}/${CI_NODE_TOTAL}")
+    fi
+    if [ -n "$job_base_name" ]; then
+        custom_tags_args+=(--tags "test.configuration.job_name:${job_base_name}")
     fi
 
-    # Debug logging
-    echo "DEBUG: DD_TAGS=$DD_TAGS"
-    echo "DEBUG: Executing datadog-ci with trace enabled:"
     DD_API_KEY=$1 \
         datadog-ci junit upload --service $SERVICE_NAME \
         --logs \
@@ -47,7 +50,7 @@ junit_upload() {
         --tags "os.platform:$(java_prop os.name)" \
         --tags "os.version:$(java_prop os.version)" \
         --tags "git.repository_url:https://github.com/DataDog/dd-trace-java" \
-        "${dd_tags_args[@]}" \
+        "${custom_tags_args[@]}" \
         ./results
 }
 

From 3c6f52ca854e39d99f44d35fcaa6edca0d2eae95 Mon Sep 17 00:00:00 2001
From: Brice Dutheil <brice.dutheil@gmail.com>
Date: Thu, 5 Feb 2026 09:54:55 +0100
Subject: [PATCH 14/15] fix: Test counting now counts testcase element

---
 .gitlab/count_tests.sh | 15 +++++++++------
 1 file changed, 9 insertions(+), 6 deletions(-)

diff --git a/.gitlab/count_tests.sh b/.gitlab/count_tests.sh
index c06b71982e9..a75ff787b57 100755
--- a/.gitlab/count_tests.sh
+++ b/.gitlab/count_tests.sh
@@ -51,12 +51,15 @@ for xml_file in $(find "$RESULTS_DIR" -name "*.xml" -type f 2>/dev/null); do
     XML_FILE_COUNT=$((XML_FILE_COUNT + 1))
     log_verbose "Processing file $XML_FILE_COUNT: $xml_file"
     if [ -f "$xml_file" ]; then
-        # Extract test counts from testsuite tags
-        # Handle both <testsuite> and <testsuites> formats
-        tests=$(grep -oP '(?<=tests=")[0-9]+' "$xml_file" 2>/dev/null | head -1 || echo "0")
-        failures=$(grep -oP '(?<=failures=")[0-9]+' "$xml_file" 2>/dev/null | head -1 || echo "0")
-        errors=$(grep -oP '(?<=errors=")[0-9]+' "$xml_file" 2>/dev/null | head -1 || echo "0")
-        skipped=$(grep -oP '(?<=skipped=")[0-9]+' "$xml_file" 2>/dev/null | head -1 || echo "0")
+        # Count actual <testcase> elements (more accurate than testsuite attributes)
+        # This matches how datadog-ci counts tests
+        tests=$(grep -c '<testcase' "$xml_file" 2>/dev/null || echo "0")
+
+        # Count <failure>, <error>, and <skipped> tags
+        # These are more reliable than trying to match testcase+failure in one grep
+        failures=$(grep -c '<failure' "$xml_file" 2>/dev/null || echo "0")
+        errors=$(grep -c '<error' "$xml_file" 2>/dev/null || echo "0")
+        skipped=$(grep -c '<skipped' "$xml_file" 2>/dev/null || echo "0")
 
         log_verbose "  → tests=$tests, failures=$failures, errors=$errors, skipped=$skipped"
 

From 262e38355d8687435b676b29418cf93225b063a5 Mon Sep 17 00:00:00 2001
From: Brice Dutheil <brice.dutheil@gmail.com>
Date: Thu, 5 Feb 2026 14:57:56 +0100
Subject: [PATCH 15/15] chore: Rename stage to test-summary

---
 .gitlab-ci.yml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 6ef5856028d..da0b0162b79 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -12,7 +12,7 @@ stages:
   - benchmarks
   - macrobenchmarks
   - tests
-  - aggregate
+  - test-summary
   - exploration-tests
   - ci-visibility-tests
   - generate-signing-key
@@ -795,7 +795,7 @@ test_smoke_semeru8_debugger:
 
 aggregate_test_counts:
   image: ghcr.io/datadog/dd-trace-java-docker-build:${BUILDER_IMAGE_VERSION_PREFIX}base
-  stage: aggregate
+  stage: test-summary
   # Note: No explicit 'needs' or 'dependencies' required
   # By default, GitLab CI automatically downloads artifacts from ALL jobs in previous stages
   # This job collects test_counts_*.json files from all test/check jobs via stage ordering