Skip to content

Github unanounymous support. #4509

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We鈥檒l occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
rasapala wants to merge 1 commit into
base: main
Choose a base branch
from
Open

Github unanounymous support. #4509

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
106 changes: 99 additions & 7 deletions Dockerfile.redhat
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -301,8 +301,25 @@ COPY package.json /ovms/
RUN cp -v /etc/ssl/certs/ca-bundle.crt /etc/ssl/certs/ca-certificates.crt

ARG KONFLUX=1 # This is a workaround to avoid bazel fetching dependencies during build, which causes issues in Konflux. Should be disabled when building outside of Konflux to speed up the build by using bazel cache and avoid fetching dependencies which are already cached.
RUN if [ "$KONFLUX" == "0" ] ; then true ; else exit 0 ; fi ; \
bazel build --jobs=$JOBS ${debug_bazel_flags} //:ovms_dependencies @com_google_googletest//:gtest
# GITHUB_TOKEN (optional) avoids GitHub anonymous-fetch throttling for WORKSPACE git_repository() rules.
# Mounted as a BuildKit secret and read only by git's own credential-helper subprocess, so `set -x` below never echoes it.
RUN --mount=type=secret,id=github_token \
if [ "$KONFLUX" == "0" ] ; then true ; else exit 0 ; fi ; \
if [ -s /run/secrets/github_token ]; then \
set +x ; \
github_token=$(cat /run/secrets/github_token) ; \
git config --system url."https://${github_token}@github.com/".insteadOf "https://github.com/" ; \
unset github_token ; \
set -x ; \
fi ; \
bazel build --jobs=$JOBS ${debug_bazel_flags} //:ovms_dependencies @com_google_googletest//:gtest ; \
build_status=$? ; \
set +x ; \
github_token=$(cat /run/secrets/github_token 2>/dev/null) ; \
git config --system --unset url."https://${github_token}@github.com/".insteadOf 2>/dev/null || true ; \
unset github_token ; \
set -x ; \
exit $build_status

COPY src/ /ovms/src/

Expand Down Expand Up @@ -332,13 +349,43 @@ RUN rm -f /usr/lib64/cmake/OpenSSL/OpenSSLConfig.cmake
# Builds unit tests together with ovms server in one step
# It speeds up CI when tests are executed outside of the image building
# hadolint ignore=SC2046
RUN bazel build --jobs=$JOBS ${debug_bazel_flags} ${minitrace_flags} //src:ovms $(if [ "$OPTIMIZE_BUILDING_TESTS" == "1" ] ; then echo -n //src:ovms_test; fi)
RUN --mount=type=secret,id=github_token \
if [ -s /run/secrets/github_token ]; then \
set +x ; \
github_token=$(cat /run/secrets/github_token) ; \
git config --system url."https://${github_token}@github.com/".insteadOf "https://github.com/" ; \
unset github_token ; \
set -x ; \
fi ; \
bazel build --jobs=$JOBS ${debug_bazel_flags} ${minitrace_flags} //src:ovms $(if [ "$OPTIMIZE_BUILDING_TESTS" == "1" ] ; then echo -n //src:ovms_test; fi) ; \
build_status=$? ; \
set +x ; \
github_token=$(cat /run/secrets/github_token 2>/dev/null) ; \
git config --system --unset url."https://${github_token}@github.com/".insteadOf 2>/dev/null || true ; \
unset github_token ; \
set -x ; \
exit $build_status

# espeak-ng is built as a separate step, independent of the OVMS binary.
# Set ESPEAK=0 to skip the espeak build.
ARG ESPEAK=1
# hadolint ignore=DL3059
RUN if [ "$ESPEAK" == "1" ]; then bazel build --jobs=$JOBS ${debug_bazel_flags} //third_party:espeak_ng //third_party:espeak_ng_data; fi
RUN --mount=type=secret,id=github_token \
if [ -s /run/secrets/github_token ]; then \
set +x ; \
github_token=$(cat /run/secrets/github_token) ; \
git config --system url."https://${github_token}@github.com/".insteadOf "https://github.com/" ; \
unset github_token ; \
set -x ; \
fi ; \
if [ "$ESPEAK" == "1" ]; then bazel build --jobs=$JOBS ${debug_bazel_flags} //third_party:espeak_ng //third_party:espeak_ng_data; fi ; \
build_status=$? ; \
set +x ; \
github_token=$(cat /run/secrets/github_token 2>/dev/null) ; \
git config --system --unset url."https://${github_token}@github.com/".insteadOf 2>/dev/null || true ; \
unset github_token ; \
set -x ; \
exit $build_status

# Tests execution
COPY ci/check_coverage.bat /ovms/
Expand Down Expand Up @@ -366,14 +413,59 @@ ARG CAPI_FLAGS="--strip=always --config=mp_off_py_off --//:distro=redhat"
ARG JOBS=40
ARG LTO_ENABLE=OFF
WORKDIR /ovms
RUN bazel build --jobs $JOBS ${CAPI_FLAGS} //src:ovms_shared
RUN --mount=type=secret,id=github_token \
if [ -s /run/secrets/github_token ]; then \
set +x ; \
github_token=$(cat /run/secrets/github_token) ; \
git config --system url."https://${github_token}@github.com/".insteadOf "https://github.com/" ; \
unset github_token ; \
set -x ; \
fi ; \
bazel build --jobs $JOBS ${CAPI_FLAGS} //src:ovms_shared ; \
build_status=$? ; \
set +x ; \
github_token=$(cat /run/secrets/github_token 2>/dev/null) ; \
git config --system --unset url."https://${github_token}@github.com/".insteadOf 2>/dev/null || true ; \
unset github_token ; \
set -x ; \
exit $build_status

# C api app with bazel
# hadolint ignore=DL3059
RUN bazel build --jobs $JOBS ${CAPI_FLAGS} //src:capi_cpp_example
RUN --mount=type=secret,id=github_token \
if [ -s /run/secrets/github_token ]; then \
set +x ; \
github_token=$(cat /run/secrets/github_token) ; \
git config --system url."https://${github_token}@github.com/".insteadOf "https://github.com/" ; \
unset github_token ; \
set -x ; \
fi ; \
bazel build --jobs $JOBS ${CAPI_FLAGS} //src:capi_cpp_example ; \
build_status=$? ; \
set +x ; \
github_token=$(cat /run/secrets/github_token 2>/dev/null) ; \
git config --system --unset url."https://${github_token}@github.com/".insteadOf 2>/dev/null || true ; \
unset github_token ; \
set -x ; \
exit $build_status

# C-API benchmark app
RUN bazel build --jobs=$JOBS ${CAPI_FLAGS} //src:capi_benchmark && ./bazel-bin/src/capi_benchmark --niter 2 --nstreams 1 --servable_name "dummy"
RUN --mount=type=secret,id=github_token \
if [ -s /run/secrets/github_token ]; then \
set +x ; \
github_token=$(cat /run/secrets/github_token) ; \
git config --system url."https://${github_token}@github.com/".insteadOf "https://github.com/" ; \
unset github_token ; \
set -x ; \
fi ; \
bazel build --jobs=$JOBS ${CAPI_FLAGS} //src:capi_benchmark && ./bazel-bin/src/capi_benchmark --niter 2 --nstreams 1 --servable_name "dummy" ; \
build_status=$? ; \
set +x ; \
github_token=$(cat /run/secrets/github_token 2>/dev/null) ; \
git config --system --unset url."https://${github_token}@github.com/".insteadOf 2>/dev/null || true ; \
unset github_token ; \
set -x ; \
exit $build_status
# C-api C/C++ app with gcc
COPY MakefileCapi /ovms/
RUN make -f MakefileCapi cpp CAPI_FLAGS="${CAPI_FLAGS}" && \
Expand Down
121 changes: 114 additions & 7 deletions Dockerfile.ubuntu
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -291,13 +291,45 @@ COPY package.json /ovms/
RUN if [ "$BASE_OS" == "ubuntu24" ] ; then update-alternatives --install /usr/bin/python3 python3 /usr/bin/python3.10 1; fi

# prebuild dependencies before copying sources & test dependencies for caching
# GITHUB_TOKEN (optional) avoids GitHub anonymous-fetch throttling for WORKSPACE git_repository() rules.
# Mounted as a BuildKit secret and read only by git's own credential-helper subprocess, so `set -x` below never echoes it.
# hadolint ignore=DL3059
RUN bazel build --jobs=$JOBS ${debug_bazel_flags} //:ovms_dependencies @com_google_googletest//:gtest
RUN --mount=type=secret,id=github_token \
if [ -s /run/secrets/github_token ]; then \
set +x ; \
github_token=$(cat /run/secrets/github_token) ; \
git config --system url."https://${github_token}@github.com/".insteadOf "https://github.com/" ; \
unset github_token ; \
set -x ; \
fi ; \
bazel build --jobs=$JOBS ${debug_bazel_flags} //:ovms_dependencies @com_google_googletest//:gtest ; \
build_status=$? ; \
set +x ; \
github_token=$(cat /run/secrets/github_token 2>/dev/null) ; \
git config --system --unset url."https://${github_token}@github.com/".insteadOf 2>/dev/null || true ; \
unset github_token ; \
set -x ; \
exit $build_status

COPY src/ /ovms/src/

# hadolint ignore=DL3059
RUN bazel build --jobs=$JOBS ${debug_bazel_flags} @org_tensorflow//tensorflow/core:framework
RUN --mount=type=secret,id=github_token \
if [ -s /run/secrets/github_token ]; then \
set +x ; \
github_token=$(cat /run/secrets/github_token) ; \
git config --system url."https://${github_token}@github.com/".insteadOf "https://github.com/" ; \
unset github_token ; \
set -x ; \
fi ; \
bazel build --jobs=$JOBS ${debug_bazel_flags} @org_tensorflow//tensorflow/core:framework ; \
build_status=$? ; \
set +x ; \
github_token=$(cat /run/secrets/github_token 2>/dev/null) ; \
git config --system --unset url."https://${github_token}@github.com/".insteadOf 2>/dev/null || true ; \
unset github_token ; \
set -x ; \
exit $build_status

# hadolint ignore=DL3059
RUN if [ "$BASE_OS" == "ubuntu24" ] ; then update-alternatives --install /usr/bin/python3 python3 /usr/bin/python3.12 2; fi
Expand Down Expand Up @@ -339,13 +371,43 @@ ARG OPTIMIZE_BUILDING_TESTS=0
# Builds unit tests together with ovms server in one step
# It speeds up CI when tests are executed outside of the image building
# hadolint ignore=SC2046
RUN if [ "$FUZZER_BUILD" == "0" ]; then bazel build --jobs=$JOBS ${debug_bazel_flags} ${minitrace_flags} //src:ovms $(if [ "${OPTIMIZE_BUILDING_TESTS}" == "1" ] ; then echo -n //src:ovms_test; fi); fi;
RUN --mount=type=secret,id=github_token \
if [ -s /run/secrets/github_token ]; then \
set +x ; \
github_token=$(cat /run/secrets/github_token) ; \
git config --system url."https://${github_token}@github.com/".insteadOf "https://github.com/" ; \
unset github_token ; \
set -x ; \
fi ; \
if [ "$FUZZER_BUILD" == "0" ]; then bazel build --jobs=$JOBS ${debug_bazel_flags} ${minitrace_flags} //src:ovms $(if [ "${OPTIMIZE_BUILDING_TESTS}" == "1" ] ; then echo -n //src:ovms_test; fi); fi ; \
build_status=$? ; \
set +x ; \
github_token=$(cat /run/secrets/github_token 2>/dev/null) ; \
git config --system --unset url."https://${github_token}@github.com/".insteadOf 2>/dev/null || true ; \
unset github_token ; \
set -x ; \
exit $build_status

# espeak-ng is built as a separate step, independent of the OVMS binary.
# Set ESPEAK=0 to skip the espeak build.
ARG ESPEAK=1
# hadolint ignore=DL3059
RUN if [ "$ESPEAK" == "1" ]; then bazel build --jobs=$JOBS ${debug_bazel_flags} //third_party:espeak_ng //third_party:espeak_ng_data; fi
RUN --mount=type=secret,id=github_token \
if [ -s /run/secrets/github_token ]; then \
set +x ; \
github_token=$(cat /run/secrets/github_token) ; \
git config --system url."https://${github_token}@github.com/".insteadOf "https://github.com/" ; \
unset github_token ; \
set -x ; \
fi ; \
if [ "$ESPEAK" == "1" ]; then bazel build --jobs=$JOBS ${debug_bazel_flags} //third_party:espeak_ng //third_party:espeak_ng_data; fi ; \
build_status=$? ; \
set +x ; \
github_token=$(cat /run/secrets/github_token 2>/dev/null) ; \
git config --system --unset url."https://${github_token}@github.com/".insteadOf 2>/dev/null || true ; \
unset github_token ; \
set -x ; \
exit $build_status

ARG RUN_TESTS=0
RUN if [ "$RUN_TESTS" == "1" ] ; then mkdir -p demos/common/export_models/ && mv export_model.py demos/common/export_models/ && ./prepare_llm_models.sh /ovms/src/test/llm_testing docker && ./run_unit_tests.sh ; fi
Expand All @@ -368,15 +430,60 @@ ARG CAPI_FLAGS="--strip=always --config mp_off_py_off --//:distro=ubuntu"
ARG JOBS=40
ARG FUZZER_BUILD=0
WORKDIR /ovms
RUN bazel build --jobs=$JOBS ${CAPI_FLAGS} //src:ovms_shared
RUN --mount=type=secret,id=github_token \
if [ -s /run/secrets/github_token ]; then \
set +x ; \
github_token=$(cat /run/secrets/github_token) ; \
git config --system url."https://${github_token}@github.com/".insteadOf "https://github.com/" ; \
unset github_token ; \
set -x ; \
fi ; \
bazel build --jobs=$JOBS ${CAPI_FLAGS} //src:ovms_shared ; \
build_status=$? ; \
set +x ; \
github_token=$(cat /run/secrets/github_token 2>/dev/null) ; \
git config --system --unset url."https://${github_token}@github.com/".insteadOf 2>/dev/null || true ; \
unset github_token ; \
set -x ; \
exit $build_status

# C-API benchmark app
RUN if [ "$FUZZER_BUILD" == "0" ]; then bazel build --jobs=$JOBS ${CAPI_FLAGS} //src:capi_benchmark && ./bazel-bin/src/capi_benchmark --niter 2 --nstreams 1 --servable_name "dummy"; fi;
RUN --mount=type=secret,id=github_token \
if [ -s /run/secrets/github_token ]; then \
set +x ; \
github_token=$(cat /run/secrets/github_token) ; \
git config --system url."https://${github_token}@github.com/".insteadOf "https://github.com/" ; \
unset github_token ; \
set -x ; \
fi ; \
if [ "$FUZZER_BUILD" == "0" ]; then bazel build --jobs=$JOBS ${CAPI_FLAGS} //src:capi_benchmark && ./bazel-bin/src/capi_benchmark --niter 2 --nstreams 1 --servable_name "dummy"; fi ; \
build_status=$? ; \
set +x ; \
github_token=$(cat /run/secrets/github_token 2>/dev/null) ; \
git config --system --unset url."https://${github_token}@github.com/".insteadOf 2>/dev/null || true ; \
unset github_token ; \
set -x ; \
exit $build_status

ARG FUZZER_BUILD=0
# C api app with bazel
# hadolint ignore=DL3059
RUN if [ "$FUZZER_BUILD" == "0" ]; then bazel build ${CAPI_FLAGS} --jobs $JOBS //src:capi_cpp_example; fi;
RUN --mount=type=secret,id=github_token \
if [ -s /run/secrets/github_token ]; then \
set +x ; \
github_token=$(cat /run/secrets/github_token) ; \
git config --system url."https://${github_token}@github.com/".insteadOf "https://github.com/" ; \
unset github_token ; \
set -x ; \
fi ; \
if [ "$FUZZER_BUILD" == "0" ]; then bazel build ${CAPI_FLAGS} --jobs $JOBS //src:capi_cpp_example; fi ; \
build_status=$? ; \
set +x ; \
github_token=$(cat /run/secrets/github_token 2>/dev/null) ; \
git config --system --unset url."https://${github_token}@github.com/".insteadOf 2>/dev/null || true ; \
unset github_token ; \
set -x ; \
exit $build_status

# C-api C/C++ app with gcc
COPY MakefileCapi /ovms/
Expand Down
25 changes: 19 additions & 6 deletions Makefile
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -31,6 +31,13 @@ STYLE_CHECK_DIRS := src
HTTP_PROXY := "$(http_proxy)"
HTTPS_PROXY := "$(https_proxy)"
NO_PROXY := "$(no_proxy)"
# Optional: avoids GitHub anonymous-fetch throttling during WORKSPACE git_repository() fetches.
# Passed as a BuildKit --secret (never --build-arg) so the token is never written to an image layer or history.
GITHUB_TOKEN ?=
SECRET_ARGS :=
ifneq ($(GITHUB_TOKEN),)
SECRET_ARGS := --secret id=github_token,env=GITHUB_TOKEN
endif
ifeq ($(shell uname),Darwin)
# MacOS
CORES_TOTAL := $(shell sysctl -n hw.physicalcpu)
Expand Down Expand Up @@ -368,23 +375,26 @@ else
@touch .workspace/metadata.json
endif
@cat .workspace/metadata.json
docker $(BUILDX) build $(NO_CACHE_OPTION) -f Dockerfile.$(DIST_OS) . \
docker buildx build $(NO_CACHE_OPTION) -f Dockerfile.$(DIST_OS) . \
$(BUILD_ARGS) \
$(SECRET_ARGS) \
-t $(OVMS_CPP_DOCKER_IMAGE)-build:$(OVMS_CPP_IMAGE_TAG)$(IMAGE_TAG_SUFFIX) \
--target=build

targz_package:
docker $(BUILDX) build -f Dockerfile.$(DIST_OS) . \
docker buildx build -f Dockerfile.$(DIST_OS) . \
$(BUILD_ARGS) \
$(SECRET_ARGS) \
--build-arg BUILD_IMAGE=$(BUILD_IMAGE) \
-t $(OVMS_CPP_DOCKER_IMAGE)-pkg:$(OVMS_CPP_IMAGE_TAG) \
--target=pkg && \
rm -vrf dist/$(OS) && mkdir -p dist/$(OS) && \
ID=$$(docker create $(OVMS_CPP_DOCKER_IMAGE)-pkg:$(OVMS_CPP_IMAGE_TAG)) && \
docker cp $$ID:/ovms_pkg/$(OS)/ovms.tar dist/$(OS)/ && \
docker rm $$ID
docker $(BUILDX) build -f Dockerfile.$(DIST_OS) . \
docker buildx build -f Dockerfile.$(DIST_OS) . \
$(BUILD_ARGS) \
$(SECRET_ARGS) \
--build-arg BUILD_IMAGE=$(BUILD_IMAGE) \
-t $(OVMS_CPP_DOCKER_IMAGE)-capi:$(OVMS_CPP_IMAGE_TAG) \
--target=capi-build && \
Expand All @@ -403,12 +413,14 @@ ifeq ($(BASE_OS),redhat)
else
$(eval NPU:=1)
endif
docker $(BUILDX) build $(NO_CACHE_OPTION) -f Dockerfile.$(DIST_OS) . \
docker buildx build $(NO_CACHE_OPTION) -f Dockerfile.$(DIST_OS) . \
$(BUILD_ARGS) \
$(SECRET_ARGS) \
-t $(OVMS_CPP_DOCKER_IMAGE):$(OVMS_CPP_IMAGE_TAG)$(IMAGE_TAG_SUFFIX) \
--target=release && \
docker $(BUILDX) build $(NO_CACHE_OPTION) -f Dockerfile.$(DIST_OS) . \
docker buildx build $(NO_CACHE_OPTION) -f Dockerfile.$(DIST_OS) . \
$(BUILD_ARGS) \
$(SECRET_ARGS) \
--build-arg GPU=1 \
--build-arg NPU=$(NPU) \
-t $(OVMS_CPP_DOCKER_IMAGE)-gpu:$(OVMS_CPP_IMAGE_TAG)$(IMAGE_TAG_SUFFIX) \
Expand Down Expand Up @@ -445,8 +457,9 @@ ifeq ($(BASE_OS),redhat)
endif

release_image:
docker $(BUILDX) build $(NO_CACHE_OPTION) -f Dockerfile.$(DIST_OS) . \
docker buildx build $(NO_CACHE_OPTION) -f Dockerfile.$(DIST_OS) . \
$(BUILD_ARGS) \
$(SECRET_ARGS) \
--build-arg BUILD_IMAGE=$(BUILD_IMAGE) \
--build-arg GPU=$(GPU) \
--build-arg NPU=$(NPU) \
Expand Down