diff --git a/import-automation/executor/Dockerfile b/import-automation/executor/Dockerfile index 67550a56ff..c8d702773b 100644 --- a/import-automation/executor/Dockerfile +++ b/import-automation/executor/Dockerfile @@ -14,11 +14,13 @@ # See the License for the specific language governing permissions and # limitations under the License. +# docker buildx build --load --build-context data=/data/repo/path --build-arg build_type=local -f Dockerfile . -t dc-import-executor + # build_type: local or cloud ARG build_type=cloud # build base image with all the dependencies installed -FROM python:3.12.8 as base +FROM python:3.12.8 AS base RUN apt-get update && apt-get install -y jq \ gnupg \ apt-transport-https \ @@ -55,18 +57,19 @@ ENV GRPC_VERBOSITY=ERROR ADD https://storage.googleapis.com/datacommons_public/import_tools/import-tool.jar /import-tool.jar COPY requirements.txt . -RUN pip install -r requirements.txt +COPY --from=ghcr.io/astral-sh/uv:latest /uv /uvx /bin/ +RUN uv pip install --system -r requirements.txt # build_type: local (copy local data repo) -FROM base as local +FROM base AS local COPY --from=data . /data # build_type: cloud (clone github data repo) -FROM base as cloud +FROM base AS cloud ARG CACHE_BUSTER=1 RUN echo "Cache buster: $CACHE_BUSTER" && git clone https://github.com/datacommonsorg/data.git # build the final image -FROM ${build_type} as final +FROM ${build_type} AS final WORKDIR /data/import-automation/executor ENTRYPOINT ["python", "main.py"] diff --git a/import-automation/executor/cloudbuild.yaml b/import-automation/executor/cloudbuild.yaml index 70ca1434de..5a2fe7e906 100644 --- a/import-automation/executor/cloudbuild.yaml +++ b/import-automation/executor/cloudbuild.yaml @@ -20,30 +20,22 @@ steps: - name: 'gcr.io/cloud-builders/docker' args: ['push', '${_DOCKER_IMAGE}', '--all-tags'] - # Install dependencies - - id: python_install - name: python:3.12.8 - entrypoint: pip - args: ['install', '-r', 'requirements.txt', '--user'] - dir: 'import-automation/executor' - # Run import executor integration test - - name: python:3.12.8 - entrypoint: python - args: ['verify_import.py'] + - name: '${_DOCKER_IMAGE}:${COMMIT_SHA}' + entrypoint: bash + args: + - '-c' + - | + python verify_import.py dir: 'import-automation/executor' - waitFor: - - python_install # Run unit tests - - name: 'python:3.12.8' + - name: '${_DOCKER_IMAGE}:${COMMIT_SHA}' args: - '-c' - | ./run_tests.sh -a entrypoint: bash - waitFor: - - python_install # Tag image as stable and push - name: 'gcr.io/cloud-builders/docker'