diff --git a/Dockerfile.redhat b/Dockerfile.redhat index 688069e2e6..eee75111e0 100644 --- a/Dockerfile.redhat +++ b/Dockerfile.redhat @@ -301,8 +301,25 @@ COPY package.json /ovms/ RUN cp -v /etc/ssl/certs/ca-bundle.crt /etc/ssl/certs/ca-certificates.crt ARG KONFLUX=1 # This is a workaround to avoid bazel fetching dependencies during build, which causes issues in Konflux. Should be disabled when building outside of Konflux to speed up the build by using bazel cache and avoid fetching dependencies which are already cached. -RUN if [ "$KONFLUX" == "0" ] ; then true ; else exit 0 ; fi ; \ - bazel build --jobs=$JOBS ${debug_bazel_flags} //:ovms_dependencies @com_google_googletest//:gtest +# GITHUB_TOKEN (optional) avoids GitHub anonymous-fetch throttling for WORKSPACE git_repository() rules. +# Mounted as a BuildKit secret and read only by git's own credential-helper subprocess, so `set -x` below never echoes it. +RUN --mount=type=secret,id=github_token \ + if [ "$KONFLUX" == "0" ] ; then true ; else exit 0 ; fi ; \ + if [ -s /run/secrets/github_token ]; then \ + set +x ; \ + github_token=$(cat /run/secrets/github_token) ; \ + git config --system url."https://${github_token}@github.com/".insteadOf "https://github.com/" ; \ + unset github_token ; \ + set -x ; \ + fi ; \ + bazel build --jobs=$JOBS ${debug_bazel_flags} //:ovms_dependencies @com_google_googletest//:gtest ; \ + build_status=$? ; \ + set +x ; \ + github_token=$(cat /run/secrets/github_token 2>/dev/null) ; \ + git config --system --unset url."https://${github_token}@github.com/".insteadOf 2>/dev/null || true ; \ + unset github_token ; \ + set -x ; \ + exit $build_status COPY src/ /ovms/src/ @@ -332,13 +349,43 @@ RUN rm -f /usr/lib64/cmake/OpenSSL/OpenSSLConfig.cmake # Builds unit tests together with ovms server in one step # It speeds up CI when tests are executed outside of the image building # hadolint ignore=SC2046 -RUN bazel build --jobs=$JOBS ${debug_bazel_flags} ${minitrace_flags} //src:ovms $(if [ "$OPTIMIZE_BUILDING_TESTS" == "1" ] ; then echo -n //src:ovms_test; fi) +RUN --mount=type=secret,id=github_token \ + if [ -s /run/secrets/github_token ]; then \ + set +x ; \ + github_token=$(cat /run/secrets/github_token) ; \ + git config --system url."https://${github_token}@github.com/".insteadOf "https://github.com/" ; \ + unset github_token ; \ + set -x ; \ + fi ; \ + bazel build --jobs=$JOBS ${debug_bazel_flags} ${minitrace_flags} //src:ovms $(if [ "$OPTIMIZE_BUILDING_TESTS" == "1" ] ; then echo -n //src:ovms_test; fi) ; \ + build_status=$? ; \ + set +x ; \ + github_token=$(cat /run/secrets/github_token 2>/dev/null) ; \ + git config --system --unset url."https://${github_token}@github.com/".insteadOf 2>/dev/null || true ; \ + unset github_token ; \ + set -x ; \ + exit $build_status # espeak-ng is built as a separate step, independent of the OVMS binary. # Set ESPEAK=0 to skip the espeak build. ARG ESPEAK=1 # hadolint ignore=DL3059 -RUN if [ "$ESPEAK" == "1" ]; then bazel build --jobs=$JOBS ${debug_bazel_flags} //third_party:espeak_ng //third_party:espeak_ng_data; fi +RUN --mount=type=secret,id=github_token \ + if [ -s /run/secrets/github_token ]; then \ + set +x ; \ + github_token=$(cat /run/secrets/github_token) ; \ + git config --system url."https://${github_token}@github.com/".insteadOf "https://github.com/" ; \ + unset github_token ; \ + set -x ; \ + fi ; \ + if [ "$ESPEAK" == "1" ]; then bazel build --jobs=$JOBS ${debug_bazel_flags} //third_party:espeak_ng //third_party:espeak_ng_data; fi ; \ + build_status=$? ; \ + set +x ; \ + github_token=$(cat /run/secrets/github_token 2>/dev/null) ; \ + git config --system --unset url."https://${github_token}@github.com/".insteadOf 2>/dev/null || true ; \ + unset github_token ; \ + set -x ; \ + exit $build_status # Tests execution COPY ci/check_coverage.bat /ovms/ @@ -366,14 +413,59 @@ ARG CAPI_FLAGS="--strip=always --config=mp_off_py_off --//:distro=redhat" ARG JOBS=40 ARG LTO_ENABLE=OFF WORKDIR /ovms -RUN bazel build --jobs $JOBS ${CAPI_FLAGS} //src:ovms_shared +RUN --mount=type=secret,id=github_token \ + if [ -s /run/secrets/github_token ]; then \ + set +x ; \ + github_token=$(cat /run/secrets/github_token) ; \ + git config --system url."https://${github_token}@github.com/".insteadOf "https://github.com/" ; \ + unset github_token ; \ + set -x ; \ + fi ; \ + bazel build --jobs $JOBS ${CAPI_FLAGS} //src:ovms_shared ; \ + build_status=$? ; \ + set +x ; \ + github_token=$(cat /run/secrets/github_token 2>/dev/null) ; \ + git config --system --unset url."https://${github_token}@github.com/".insteadOf 2>/dev/null || true ; \ + unset github_token ; \ + set -x ; \ + exit $build_status # C api app with bazel # hadolint ignore=DL3059 -RUN bazel build --jobs $JOBS ${CAPI_FLAGS} //src:capi_cpp_example +RUN --mount=type=secret,id=github_token \ + if [ -s /run/secrets/github_token ]; then \ + set +x ; \ + github_token=$(cat /run/secrets/github_token) ; \ + git config --system url."https://${github_token}@github.com/".insteadOf "https://github.com/" ; \ + unset github_token ; \ + set -x ; \ + fi ; \ + bazel build --jobs $JOBS ${CAPI_FLAGS} //src:capi_cpp_example ; \ + build_status=$? ; \ + set +x ; \ + github_token=$(cat /run/secrets/github_token 2>/dev/null) ; \ + git config --system --unset url."https://${github_token}@github.com/".insteadOf 2>/dev/null || true ; \ + unset github_token ; \ + set -x ; \ + exit $build_status # C-API benchmark app -RUN bazel build --jobs=$JOBS ${CAPI_FLAGS} //src:capi_benchmark && ./bazel-bin/src/capi_benchmark --niter 2 --nstreams 1 --servable_name "dummy" +RUN --mount=type=secret,id=github_token \ + if [ -s /run/secrets/github_token ]; then \ + set +x ; \ + github_token=$(cat /run/secrets/github_token) ; \ + git config --system url."https://${github_token}@github.com/".insteadOf "https://github.com/" ; \ + unset github_token ; \ + set -x ; \ + fi ; \ + bazel build --jobs=$JOBS ${CAPI_FLAGS} //src:capi_benchmark && ./bazel-bin/src/capi_benchmark --niter 2 --nstreams 1 --servable_name "dummy" ; \ + build_status=$? ; \ + set +x ; \ + github_token=$(cat /run/secrets/github_token 2>/dev/null) ; \ + git config --system --unset url."https://${github_token}@github.com/".insteadOf 2>/dev/null || true ; \ + unset github_token ; \ + set -x ; \ + exit $build_status # C-api C/C++ app with gcc COPY MakefileCapi /ovms/ RUN make -f MakefileCapi cpp CAPI_FLAGS="${CAPI_FLAGS}" && \ diff --git a/Dockerfile.ubuntu b/Dockerfile.ubuntu index c95124f997..ffa42d8d96 100644 --- a/Dockerfile.ubuntu +++ b/Dockerfile.ubuntu @@ -291,13 +291,45 @@ COPY package.json /ovms/ RUN if [ "$BASE_OS" == "ubuntu24" ] ; then update-alternatives --install /usr/bin/python3 python3 /usr/bin/python3.10 1; fi # prebuild dependencies before copying sources & test dependencies for caching +# GITHUB_TOKEN (optional) avoids GitHub anonymous-fetch throttling for WORKSPACE git_repository() rules. +# Mounted as a BuildKit secret and read only by git's own credential-helper subprocess, so `set -x` below never echoes it. # hadolint ignore=DL3059 -RUN bazel build --jobs=$JOBS ${debug_bazel_flags} //:ovms_dependencies @com_google_googletest//:gtest +RUN --mount=type=secret,id=github_token \ + if [ -s /run/secrets/github_token ]; then \ + set +x ; \ + github_token=$(cat /run/secrets/github_token) ; \ + git config --system url."https://${github_token}@github.com/".insteadOf "https://github.com/" ; \ + unset github_token ; \ + set -x ; \ + fi ; \ + bazel build --jobs=$JOBS ${debug_bazel_flags} //:ovms_dependencies @com_google_googletest//:gtest ; \ + build_status=$? ; \ + set +x ; \ + github_token=$(cat /run/secrets/github_token 2>/dev/null) ; \ + git config --system --unset url."https://${github_token}@github.com/".insteadOf 2>/dev/null || true ; \ + unset github_token ; \ + set -x ; \ + exit $build_status COPY src/ /ovms/src/ # hadolint ignore=DL3059 -RUN bazel build --jobs=$JOBS ${debug_bazel_flags} @org_tensorflow//tensorflow/core:framework +RUN --mount=type=secret,id=github_token \ + if [ -s /run/secrets/github_token ]; then \ + set +x ; \ + github_token=$(cat /run/secrets/github_token) ; \ + git config --system url."https://${github_token}@github.com/".insteadOf "https://github.com/" ; \ + unset github_token ; \ + set -x ; \ + fi ; \ + bazel build --jobs=$JOBS ${debug_bazel_flags} @org_tensorflow//tensorflow/core:framework ; \ + build_status=$? ; \ + set +x ; \ + github_token=$(cat /run/secrets/github_token 2>/dev/null) ; \ + git config --system --unset url."https://${github_token}@github.com/".insteadOf 2>/dev/null || true ; \ + unset github_token ; \ + set -x ; \ + exit $build_status # hadolint ignore=DL3059 RUN if [ "$BASE_OS" == "ubuntu24" ] ; then update-alternatives --install /usr/bin/python3 python3 /usr/bin/python3.12 2; fi @@ -339,13 +371,43 @@ ARG OPTIMIZE_BUILDING_TESTS=0 # Builds unit tests together with ovms server in one step # It speeds up CI when tests are executed outside of the image building # hadolint ignore=SC2046 -RUN if [ "$FUZZER_BUILD" == "0" ]; then bazel build --jobs=$JOBS ${debug_bazel_flags} ${minitrace_flags} //src:ovms $(if [ "${OPTIMIZE_BUILDING_TESTS}" == "1" ] ; then echo -n //src:ovms_test; fi); fi; +RUN --mount=type=secret,id=github_token \ + if [ -s /run/secrets/github_token ]; then \ + set +x ; \ + github_token=$(cat /run/secrets/github_token) ; \ + git config --system url."https://${github_token}@github.com/".insteadOf "https://github.com/" ; \ + unset github_token ; \ + set -x ; \ + fi ; \ + if [ "$FUZZER_BUILD" == "0" ]; then bazel build --jobs=$JOBS ${debug_bazel_flags} ${minitrace_flags} //src:ovms $(if [ "${OPTIMIZE_BUILDING_TESTS}" == "1" ] ; then echo -n //src:ovms_test; fi); fi ; \ + build_status=$? ; \ + set +x ; \ + github_token=$(cat /run/secrets/github_token 2>/dev/null) ; \ + git config --system --unset url."https://${github_token}@github.com/".insteadOf 2>/dev/null || true ; \ + unset github_token ; \ + set -x ; \ + exit $build_status # espeak-ng is built as a separate step, independent of the OVMS binary. # Set ESPEAK=0 to skip the espeak build. ARG ESPEAK=1 # hadolint ignore=DL3059 -RUN if [ "$ESPEAK" == "1" ]; then bazel build --jobs=$JOBS ${debug_bazel_flags} //third_party:espeak_ng //third_party:espeak_ng_data; fi +RUN --mount=type=secret,id=github_token \ + if [ -s /run/secrets/github_token ]; then \ + set +x ; \ + github_token=$(cat /run/secrets/github_token) ; \ + git config --system url."https://${github_token}@github.com/".insteadOf "https://github.com/" ; \ + unset github_token ; \ + set -x ; \ + fi ; \ + if [ "$ESPEAK" == "1" ]; then bazel build --jobs=$JOBS ${debug_bazel_flags} //third_party:espeak_ng //third_party:espeak_ng_data; fi ; \ + build_status=$? ; \ + set +x ; \ + github_token=$(cat /run/secrets/github_token 2>/dev/null) ; \ + git config --system --unset url."https://${github_token}@github.com/".insteadOf 2>/dev/null || true ; \ + unset github_token ; \ + set -x ; \ + exit $build_status ARG RUN_TESTS=0 RUN if [ "$RUN_TESTS" == "1" ] ; then mkdir -p demos/common/export_models/ && mv export_model.py demos/common/export_models/ && ./prepare_llm_models.sh /ovms/src/test/llm_testing docker && ./run_unit_tests.sh ; fi @@ -368,15 +430,60 @@ ARG CAPI_FLAGS="--strip=always --config mp_off_py_off --//:distro=ubuntu" ARG JOBS=40 ARG FUZZER_BUILD=0 WORKDIR /ovms -RUN bazel build --jobs=$JOBS ${CAPI_FLAGS} //src:ovms_shared +RUN --mount=type=secret,id=github_token \ + if [ -s /run/secrets/github_token ]; then \ + set +x ; \ + github_token=$(cat /run/secrets/github_token) ; \ + git config --system url."https://${github_token}@github.com/".insteadOf "https://github.com/" ; \ + unset github_token ; \ + set -x ; \ + fi ; \ + bazel build --jobs=$JOBS ${CAPI_FLAGS} //src:ovms_shared ; \ + build_status=$? ; \ + set +x ; \ + github_token=$(cat /run/secrets/github_token 2>/dev/null) ; \ + git config --system --unset url."https://${github_token}@github.com/".insteadOf 2>/dev/null || true ; \ + unset github_token ; \ + set -x ; \ + exit $build_status # C-API benchmark app -RUN if [ "$FUZZER_BUILD" == "0" ]; then bazel build --jobs=$JOBS ${CAPI_FLAGS} //src:capi_benchmark && ./bazel-bin/src/capi_benchmark --niter 2 --nstreams 1 --servable_name "dummy"; fi; +RUN --mount=type=secret,id=github_token \ + if [ -s /run/secrets/github_token ]; then \ + set +x ; \ + github_token=$(cat /run/secrets/github_token) ; \ + git config --system url."https://${github_token}@github.com/".insteadOf "https://github.com/" ; \ + unset github_token ; \ + set -x ; \ + fi ; \ + if [ "$FUZZER_BUILD" == "0" ]; then bazel build --jobs=$JOBS ${CAPI_FLAGS} //src:capi_benchmark && ./bazel-bin/src/capi_benchmark --niter 2 --nstreams 1 --servable_name "dummy"; fi ; \ + build_status=$? ; \ + set +x ; \ + github_token=$(cat /run/secrets/github_token 2>/dev/null) ; \ + git config --system --unset url."https://${github_token}@github.com/".insteadOf 2>/dev/null || true ; \ + unset github_token ; \ + set -x ; \ + exit $build_status ARG FUZZER_BUILD=0 # C api app with bazel # hadolint ignore=DL3059 -RUN if [ "$FUZZER_BUILD" == "0" ]; then bazel build ${CAPI_FLAGS} --jobs $JOBS //src:capi_cpp_example; fi; +RUN --mount=type=secret,id=github_token \ + if [ -s /run/secrets/github_token ]; then \ + set +x ; \ + github_token=$(cat /run/secrets/github_token) ; \ + git config --system url."https://${github_token}@github.com/".insteadOf "https://github.com/" ; \ + unset github_token ; \ + set -x ; \ + fi ; \ + if [ "$FUZZER_BUILD" == "0" ]; then bazel build ${CAPI_FLAGS} --jobs $JOBS //src:capi_cpp_example; fi ; \ + build_status=$? ; \ + set +x ; \ + github_token=$(cat /run/secrets/github_token 2>/dev/null) ; \ + git config --system --unset url."https://${github_token}@github.com/".insteadOf 2>/dev/null || true ; \ + unset github_token ; \ + set -x ; \ + exit $build_status # C-api C/C++ app with gcc COPY MakefileCapi /ovms/ diff --git a/Makefile b/Makefile index aebb6bbdf8..23cdd22929 100644 --- a/Makefile +++ b/Makefile @@ -31,6 +31,13 @@ STYLE_CHECK_DIRS := src HTTP_PROXY := "$(http_proxy)" HTTPS_PROXY := "$(https_proxy)" NO_PROXY := "$(no_proxy)" +# Optional: avoids GitHub anonymous-fetch throttling during WORKSPACE git_repository() fetches. +# Passed as a BuildKit --secret (never --build-arg) so the token is never written to an image layer or history. +GITHUB_TOKEN ?= +SECRET_ARGS := +ifneq ($(GITHUB_TOKEN),) +SECRET_ARGS := --secret id=github_token,env=GITHUB_TOKEN +endif ifeq ($(shell uname),Darwin) # MacOS CORES_TOTAL := $(shell sysctl -n hw.physicalcpu) @@ -368,14 +375,16 @@ else @touch .workspace/metadata.json endif @cat .workspace/metadata.json - docker $(BUILDX) build $(NO_CACHE_OPTION) -f Dockerfile.$(DIST_OS) . \ + docker buildx build $(NO_CACHE_OPTION) -f Dockerfile.$(DIST_OS) . \ $(BUILD_ARGS) \ + $(SECRET_ARGS) \ -t $(OVMS_CPP_DOCKER_IMAGE)-build:$(OVMS_CPP_IMAGE_TAG)$(IMAGE_TAG_SUFFIX) \ --target=build targz_package: - docker $(BUILDX) build -f Dockerfile.$(DIST_OS) . \ + docker buildx build -f Dockerfile.$(DIST_OS) . \ $(BUILD_ARGS) \ + $(SECRET_ARGS) \ --build-arg BUILD_IMAGE=$(BUILD_IMAGE) \ -t $(OVMS_CPP_DOCKER_IMAGE)-pkg:$(OVMS_CPP_IMAGE_TAG) \ --target=pkg && \ @@ -383,8 +392,9 @@ targz_package: ID=$$(docker create $(OVMS_CPP_DOCKER_IMAGE)-pkg:$(OVMS_CPP_IMAGE_TAG)) && \ docker cp $$ID:/ovms_pkg/$(OS)/ovms.tar dist/$(OS)/ && \ docker rm $$ID - docker $(BUILDX) build -f Dockerfile.$(DIST_OS) . \ + docker buildx build -f Dockerfile.$(DIST_OS) . \ $(BUILD_ARGS) \ + $(SECRET_ARGS) \ --build-arg BUILD_IMAGE=$(BUILD_IMAGE) \ -t $(OVMS_CPP_DOCKER_IMAGE)-capi:$(OVMS_CPP_IMAGE_TAG) \ --target=capi-build && \ @@ -403,12 +413,14 @@ ifeq ($(BASE_OS),redhat) else $(eval NPU:=1) endif - docker $(BUILDX) build $(NO_CACHE_OPTION) -f Dockerfile.$(DIST_OS) . \ + docker buildx build $(NO_CACHE_OPTION) -f Dockerfile.$(DIST_OS) . \ $(BUILD_ARGS) \ + $(SECRET_ARGS) \ -t $(OVMS_CPP_DOCKER_IMAGE):$(OVMS_CPP_IMAGE_TAG)$(IMAGE_TAG_SUFFIX) \ --target=release && \ - docker $(BUILDX) build $(NO_CACHE_OPTION) -f Dockerfile.$(DIST_OS) . \ + docker buildx build $(NO_CACHE_OPTION) -f Dockerfile.$(DIST_OS) . \ $(BUILD_ARGS) \ + $(SECRET_ARGS) \ --build-arg GPU=1 \ --build-arg NPU=$(NPU) \ -t $(OVMS_CPP_DOCKER_IMAGE)-gpu:$(OVMS_CPP_IMAGE_TAG)$(IMAGE_TAG_SUFFIX) \ @@ -445,8 +457,9 @@ ifeq ($(BASE_OS),redhat) endif release_image: - docker $(BUILDX) build $(NO_CACHE_OPTION) -f Dockerfile.$(DIST_OS) . \ + docker buildx build $(NO_CACHE_OPTION) -f Dockerfile.$(DIST_OS) . \ $(BUILD_ARGS) \ + $(SECRET_ARGS) \ --build-arg BUILD_IMAGE=$(BUILD_IMAGE) \ --build-arg GPU=$(GPU) \ --build-arg NPU=$(NPU) \