diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 6a570f099a..47d110b1f2 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -143,6 +143,8 @@ merge-gate: package-trigger: stage: build needs: [ "generate-templates" ] + inherit: + variables: false trigger: include: - artifact: .gitlab/package-gen.yml @@ -153,8 +155,6 @@ package-trigger: pipeline_variables: true variables: PARENT_PIPELINE_ID: $CI_PIPELINE_ID - GIT_SUBMODULE_STRATEGY: recursive - GIT_SUBMODULE_PATHS: libdatadog tests/FeatureFlags/ffe-system-test-data appsec/third_party/cpp-base64 appsec/third_party/libddwaf appsec/third_party/libddwaf-rust appsec/third_party/msgpack-c NIGHTLY_BUILD: $NIGHTLY_BUILD # Runs after the full CI completes. Triggered in two situations: diff --git a/.gitlab/generate-package.php b/.gitlab/generate-package.php index a952d859af..2c30826e53 100644 --- a/.gitlab/generate-package.php +++ b/.gitlab/generate-package.php @@ -119,6 +119,10 @@ function appsec_image_from_tag_mapping(string $tag): string variables: FF_ENABLE_BASH_EXIT_CODE_CHECK: "true" FF_USE_NEW_BASH_EVAL_STRATEGY: "true" + GIT_SUBMODULE_STRATEGY: recursive + GIT_SUBMODULE_PATHS: libdatadog tests/FeatureFlags/ffe-system-test-data appsec/third_party/cpp-base64 appsec/third_party/libddwaf appsec/third_party/libddwaf-rust appsec/third_party/msgpack-c + RELIABILITY_ENV_BRANCH: "master" + SYSTEM_TESTS_LIBRARY: php CARGO_HOME: "${CI_PROJECT_DIR}/.cache/cargo" # One pipeline injection package size ratchet @@ -176,6 +180,13 @@ function appsec_image_from_tag_mapping(string $tag): string tags: [ "arch:amd64" ] script: - ./.gitlab/append-build-id.sh + - | + SYSTEM_TESTS_SHA=$(git ls-remote https://github.com/DataDog/system-tests.git refs/heads/main | awk 'NR == 1 { print $1 }') + if ! printf '%s\n' "$SYSTEM_TESTS_SHA" | grep -Eq '^[0-9a-f]{40}$'; then + echo "Failed to resolve a valid system-tests commit: $SYSTEM_TESTS_SHA" + exit 1 + fi + printf 'SYSTEM_TESTS_SHA=%s\n' "$SYSTEM_TESTS_SHA" > system-tests.env # Upgrading composer - composer self-update --no-interaction # Installing dependencies with composer @@ -190,6 +201,8 @@ function appsec_image_from_tag_mapping(string $tag): string paths: - VERSION - ./src/bridge/_generated*.php + reports: + dotenv: system-tests.env /etc/apt/sources.list.d/docker.list - apt-get update -o dir::state::lists="$APT_CACHE/lists" - apt-get install -y --no-install-recommends -o dir::state::lists="$APT_CACHE/lists" -o dir::cache::archives="$APT_CACHE/archives" docker-ce docker-ce-cli containerd.io docker-buildx-plugin docker-compose-plugin - - # Install Python dependencies - pip install -U pip virtualenv + set -e + if ! printf '%s\n' "${SYSTEM_TESTS_SHA:-}" | grep -Eq '^[0-9a-f]{40}$'; then + echo "Missing or invalid SYSTEM_TESTS_SHA: ${SYSTEM_TESTS_SHA:-}" + exit 1 + fi + git init -q system-tests + git -C system-tests fetch --quiet --depth 1 https://github.com/DataDog/system-tests.git "$SYSTEM_TESTS_SHA" + git -C system-tests checkout --quiet --detach "$SYSTEM_TESTS_SHA" + test "$(git -C system-tests rev-parse HEAD)" = "$SYSTEM_TESTS_SHA" + test "$(git -C system-tests rev-parse --is-shallow-repository)" = "true" + - CI_IMAGE="$CI_JOB_IMAGE" python3 system-tests/utils/ci/gitlab/build_ci_image.py --check-only - /tmp/vault kv get --format=json "kv/k8s/gitlab-runner/dd-trace-php/datadoghq-api-key" 2>/dev/null | python3 -c "import sys,json;print(json.load(sys.stdin)['data']['data']['key'])" > /tmp/.dd-api-key 2>/dev/null || true - - git clone https://github.com/DataDog/system-tests.git - mv packages/{datadog-setup.php,dd-library-php-*x86_64-linux-gnu.tar.gz} system-tests/binaries - cd system-tests - - ./build.sh $BUILD_SH_ARGS - cache: - - key: v0-$CI_JOB_NAME_SLUG-cache - when: always - paths: - - .cache/ + - ln -sf /system-tests/venv venv + - source venv/bin/activate + - export PYTHONPATH="$CI_PROJECT_DIR/system-tests" + - if [ -n "$BUILD_SH_ARGS" ]; then ./build.sh $BUILD_SH_ARGS; fi after_script: - DATADOG_API_KEY=$(cat /tmp/.dd-api-key 2>/dev/null) || true - mkdir -p artifacts && for f in system-tests/logs*/reportJunit.xml; do dir=$(basename $(dirname "$f")); cp "$f" "artifacts/reportJunit_${dir}.xml" 2>/dev/null || true; done @@ -1346,14 +1349,14 @@ function appsec_image_from_tag_mapping(string $tag): string "System Tests: [php-fpm-8.5, default]": extends: .system_tests variables: - BUILD_SH_ARGS: -w php-fpm-8.5 php + BUILD_SH_ARGS: "-i weblog -w php-fpm-8.5 php" script: - ./run.sh "System Tests: [php-fpm-8.5]": extends: .system_tests variables: - BUILD_SH_ARGS: -w php-fpm-8.5 php + BUILD_SH_ARGS: "-i weblog -w php-fpm-8.5 php" parallel: matrix: - TESTSUITE: @@ -1386,7 +1389,7 @@ function appsec_image_from_tag_mapping(string $tag): string extends: .system_tests timeout: 4h variables: - BUILD_SH_ARGS: -w php + BUILD_SH_ARGS: "-i weblog -w php" # Expand the DinD loopback volume to avoid running out of disk space. # See https://datadoghq.atlassian.net/wiki/spaces/K8S/pages/2874901299/How+to+use+Micro+VMs#DinD-in-CI DOCKER_LOOPBACK_SIZE: 50G @@ -1407,7 +1410,7 @@ function appsec_image_from_tag_mapping(string $tag): string "System Tests: [parametric]": extends: .system_tests variables: - BUILD_SH_ARGS: "-i runner" + BUILD_SH_ARGS: "" script: - ./run.sh PARAMETRIC