OBSDATA-13340 Add monitor to find length of queue at Routers

.dockerignore

@@ -17,7 +17,6 @@
 # under the License.
 
 .git
-**/*.jar
 **/*.class
 dist
 target

.github/CODEOWNERS

@@ -0,0 +1 @@
+*	@confluentinc/obs-data

.gitignore

@@ -8,7 +8,7 @@ target
 *.swo
 *.pyc
 .classpath
-.idea
+.idea/
 .project
 .PVS-Studio
 .settings/

.semaphore/semaphore.yml

@@ -0,0 +1,157 @@
+version: v1.0
+name: Apache Druid
+agent:
+  machine:
+    type: s1-prod-ubuntu24-04-amd64-3
+execution_time_limit:
+  hours: 3
+blocks:
+  - name: "Install"
+    task:
+      env_vars: &env_vars
+        - name: MVN
+          value: "mvn -B"
+        - name: MAVEN_OPTS
+          value: "-Dmaven.repo.local=.m2"
+        - name: MAVEN_SKIP
+          value: >
+            -Danimal.sniffer.skip=true
+            -Dcheckstyle.skip=true
+            -Ddruid.console.skip=true
+            -Denforcer.skip=true
+            -Dforbiddenapis.skip=true
+            -Dmaven.javadoc.skip=true
+            -Dpmd.skip=true
+            -Dspotbugs.skip=true
+
+        - name: MAVEN_SKIP_TESTS
+          value: "-DskipTests -Djacoco.skip=true"
+      prologue:
+        commands:
+          - echo $SEMAPHORE_WORKFLOW_ID
+          - sem-version java 17
+          - checkout
+      jobs:
+        - name: "Install"
+          commands:
+            # This is a change meant to validate semaphore public builds
+            # so thus removing configurations for Confluent's internal CodeArtifact
+            - rm ~/.m2/settings.xml
+            - >
+              MAVEN_OPTS="${MAVEN_OPTS} -Xmx3000m" ${MVN} clean install
+              -q -ff ${MAVEN_SKIP} ${MAVEN_SKIP_TESTS} -T1C
+            # downstream tests depend on artifacts installed by mvn install into .m2
+            # also cache target to avoid the cost of recompiling tests
+            - tar zcf cache-post-install.tgz .m2 target
+            - artifact push workflow cache-post-install.tgz
+
+  - name: "Tests"
+    task:
+      env_vars: *env_vars
+      prologue:
+        commands:
+          - echo $SEMAPHORE_WORKFLOW_ID
+          - sem-version java 17
+          - checkout
+          - artifact pull workflow cache-post-install.tgz
+          - tar zxf cache-post-install.tgz
+          # This is a change meant to validate semaphore public builds
+          # so thus removing configurations for Confluent's internal CodeArtifact
+          - rm ~/.m2/settings.xml
+      jobs:
+        - name: "animal sniffer checks"
+          commands:
+            - ${MVN} test-compile ${MAVEN_SKIP} ${MAVEN_SKIP_TESTS}
+            - ${MVN} animal-sniffer:check --fail-at-end
+
+        - name: "checkstyle"
+          commands:
+            - ${MVN} checkstyle:checkstyle --fail-at-end
+
+        - name: "enforcer checks"
+          commands:
+            - ${MVN} enforcer:enforce --fail-at-end
+
+        - name: "forbidden api checks"
+          commands:
+            - ${MVN} test-compile ${MAVEN_SKIP} ${MAVEN_SKIP_TESTS}
+            - ${MVN} forbiddenapis:check forbiddenapis:testCheck --fail-at-end
+
+        - name: "pmd checks"
+          commands:
+            - ${MVN} pmd:check --fail-at-end  # TODO: consider adding pmd:cpd-check
+
+        - name: "spotbugs checks"
+          commands:
+            - ${MVN} spotbugs:check --fail-at-end -pl '!benchmarks'
+
+        - name: "analyze dependencies"
+          commands:
+            - >
+              ${MVN} ${MAVEN_SKIP} dependency:analyze -DoutputXML=true -DignoreNonCompile=true -DfailOnWarning=true --fail-at-end || { echo "
+
+              The dependency analysis has found a dependency that is either:
+              1) Used and undeclared: These are available as a transitive dependency but should be explicitly
+              added to the POM to ensure the dependency version. The XML to add the dependencies to the POM is
+              shown above.
+              2) Unused and declared: These are not needed and removing them from the POM will speed up the build
+              and reduce the artifact size. The dependencies to remove are shown above.
+              If there are false positive dependency analysis warnings, they can be suppressed:
+              https://maven.apache.org/plugins/maven-dependency-plugin/analyze-mojo.html#usedDependencies
+              https://maven.apache.org/plugins/maven-dependency-plugin/examples/exclude-dependencies-from-dependency-analysis.html
+              For more information, refer to:
+              https://maven.apache.org/plugins/maven-dependency-plugin/analyze-mojo.html
+              " && false; }
+
+        - name: "Confluent Extensions"
+          env_vars:
+            - name: MAVEN_PROJECTS
+              value: extensions-contrib/confluent-extensions
+          commands: &run_tests
+            - >
+              MAVEN_OPTS="${MAVEN_OPTS} -Xmx1g" ${MVN} test -pl ${MAVEN_PROJECTS}
+              ${MAVEN_SKIP} -Dremoteresources.skip=true
+
+        - name: "Server"
+          env_vars:
+            - name: MAVEN_PROJECTS
+              value: server
+          commands: *run_tests
+
+        - name: "Processing"
+          env_vars:
+            - name: MAVEN_PROJECTS
+              value: processing
+          commands: *run_tests
+
+        - name: "Indexing Service"
+          env_vars:
+            - name: MAVEN_PROJECTS
+              value: indexing-service
+          commands: *run_tests
+
+        - name: "Kafka Indexing Service"
+          env_vars:
+            - name: MAVEN_PROJECTS
+              value: extensions-core/kafka-indexing-service
+          commands: *run_tests
+
+        - name: "Web Console"
+          env_vars:
+            - name: MAVEN_PROJECTS
+              value: web-console
+          commands: *run_tests
+
+        - name: "Services"
+          env_vars:
+            - name: MAVEN_PROJECTS
+              value: services
+            - name: MAVEN_OPTS
+              value: "-Dmaven.repo.local=.m2 -DforkCount=1 -DreuseForks=false"
+          commands: *run_tests
+
+        - name: "Other Tests"
+          env_vars:
+            - name: MAVEN_PROJECTS
+              value: '!server,!processing,!indexing-service,!extensions-core/kafka-indexing-service,!extensions-contrib/confluent-extensions,!integration-tests-ex/cases,!web-console,!services'
+          commands: *run_tests

codestyle/checkstyle-suppressions.xml

@@ -53,4 +53,8 @@
   <suppress checks="ImportOrder" message="^'java\..*'.*" />
 
   <suppress checks="AvoidStaticImport" files="[\\/]src[\\/](test)[\\/]"/>
+
+  <!-- Confluent Extensions -->
+  <suppress checks="Header" files="[\\/]io[\\/]confluent[\\/]" />
+  <suppress checks="PackageName" files="[\\/]io[\\/]confluent[\\/]" />
 </suppressions>

distribution/docker/Dockerfile

@@ -18,6 +18,7 @@
 #
 
 ARG JDK_VERSION=17
+ARG BASE_IMAGE=gcr.io/distroless/java$JDK_VERSION-debian12
 
 # The platform is explicitly specified as x64 to build the Druid distribution.
 # This is because it's not able to build the distribution on arm64 due to dependency problem of web-console. See: https://github.com/apache/druid/issues/13012
@@ -36,7 +37,7 @@ RUN export DEBIAN_FRONTEND=noninteractive \
 COPY . /src
 WORKDIR /src
 RUN --mount=type=cache,target=/root/.m2 if [ "$BUILD_FROM_SOURCE" = "true" ]; then \
-      mvn -B -ff -q \
+      mvn -B -ff -q dependency:go-offline \
       install \
       -Pdist,bundle-contrib-exts \
       -Pskip-static-checks,skip-tests \
@@ -49,56 +50,116 @@ RUN --mount=type=cache,target=/root/.m2 VERSION=$(mvn -B -q org.apache.maven.plu
  && tar -zxf ./distribution/target/apache-druid-${VERSION}-bin.tar.gz -C /opt \
  && mv /opt/apache-druid-${VERSION} /opt/druid
 
-FROM alpine:3 as bash-static
+FROM busybox:1.34.1-glibc as busybox
+
+FROM $BASE_IMAGE
+
+LABEL maintainer="Apache Druid Developers <dev@druid.apache.org>"
+
+USER root
+
+COPY --from=busybox /bin/busybox /busybox/busybox
+RUN ["/busybox/busybox", "sh", "-c", "if [ ! -x \"$(command -v bash)\" ]; then \
+                                        /busybox/busybox --install /bin; \
+                                      else \
+                                        rm /busybox/busybox; \
+                                      fi;"]
+# Predefined builtin arg, see: https://docs.docker.com/engine/reference/builder/#automatic-platform-args-in-the-global-scope
 ARG TARGETARCH
+
 #
 # Download bash-static binary to execute scripts that require bash.
 # Although bash-static supports multiple platforms, but there's no need for us to support all those platform, amd64 and arm64 are enough.
 #
 ARG BASH_URL_BASE="https://github.com/robxu9/bash-static/releases/download/5.1.016-1.2.3"
-RUN if [ "$TARGETARCH" = "arm64" ]; then \
-      BASH_URL="${BASH_URL_BASE}/bash-linux-aarch64" ; \
-    elif [ "$TARGETARCH" = "amd64" ]; then \
-      BASH_URL="${BASH_URL_BASE}/bash-linux-x86_64" ; \
+RUN if [ ! -x "$(command -v bash)" ]; then \
+      if [ "$TARGETARCH" = "arm64" ]; then \
+        BASH_URL="${BASH_URL_BASE}/bash-linux-aarch64" ; \
+      elif [ "$TARGETARCH" = "amd64" ]; then \
+        BASH_URL="${BASH_URL_BASE}/bash-linux-x86_64" ; \
+      else \
+        echo "Unsupported architecture ($TARGETARCH)" && exit 1; \
+      fi; \
+      echo "Downloading bash-static from ${BASH_URL}" \
+      && wget ${BASH_URL} -O /bin/bash \
+      && chmod 755 /bin/bash; \
+    fi;
+
+RUN if [ ! -x "$(command -v useradd)" ]; then \
+      addgroup -S -g 1000 druid \
+      && adduser -S -u 1000 -D -H -h /opt/druid -s /bin/sh -g '' -G druid druid; \
     else \
-      echo "Unsupported architecture ($TARGETARCH)" && exit 1; \
-    fi; \
-    echo "Downloading bash-static from ${BASH_URL}" \
-    && wget ${BASH_URL} -O /bin/bash
-
-FROM busybox:1.35.0-glibc as busybox
-
-FROM gcr.io/distroless/java$JDK_VERSION-debian12
-LABEL maintainer="Apache Druid Developers <dev@druid.apache.org>"
-
-COPY --from=busybox /bin/busybox /busybox/busybox
-RUN ["/busybox/busybox", "--install", "/bin"]
-
-
-RUN addgroup -S -g 1000 druid \
- && adduser -S -u 1000 -D -H -h /opt/druid -s /bin/sh -g '' -G druid druid
-
-
-COPY --from=bash-static /bin/bash /bin/bash
-RUN chmod 755 /bin/bash
+      groupadd --system --gid 1000 druid \
+      && useradd --system --uid 1000 -M --home /opt/druid --shell /bin/sh -c '' --gid 1000 druid; \
+    fi;
 
+COPY --chown=druid:druid --from=builder /opt /opt
 COPY distribution/docker/druid.sh /druid.sh
 COPY distribution/docker/peon.sh /peon.sh
-COPY distribution/docker/deduplicate_jars.sh /deduplicate_jars.sh
+COPY --chown=druid:druid distribution/docker/extra_jars/ /opt/druid/lib/
 
 # create necessary directories which could be mounted as volume
-# copy and de-duplicate jars from builder in same layer to reduce image size
 #   /opt/druid/var is used to keep individual files(e.g. log) of each Druid service
 #   /opt/shared is used to keep segments and task logs shared among Druid services
-RUN --mount=type=bind,from=builder,source=/opt,target=/builder/opt \
- mkdir -p /opt/druid/var /opt/shared \
- && cp -r /builder/opt/druid /opt/ \
- && /deduplicate_jars.sh /opt/druid \
- && chown -R druid:druid /opt/druid /opt/shared \
+RUN mkdir /opt/druid/var /opt/shared \
+ && chown druid:druid /opt/druid/var /opt/shared \
  && chmod 775 /opt/druid/var /opt/shared
 
+# Install iproute2 to get the ip command needed to set config of druid.host IP address
+# Command needed in druid.sh Line 140;
+RUN if [ ! -x "$(command -v ip)" ]; then \
+      if [ -x "$(command -v apt)" ]; then \
+        apt update \
+        && apt install -y iproute2; \
+      else \
+        apk add iproute2; \
+      fi; \
+    fi;
+
+# OBSDATA-13070: Bake AWS CLI v2 and async-profiler into the image.
+# Shoreline notebooks previously downloaded these at runtime from the public internet
+# without integrity verification. Installing at build time eliminates supply chain risk.
+ARG AWS_CLI_VERSION=2.34.11
+ARG ASYNC_PROFILER_VERSION=4.3
+RUN set -e \
+    && if [ "$TARGETARCH" = "arm64" ]; then ARCH=aarch64; AP_ARCH=arm64; else ARCH=x86_64; AP_ARCH=x64; fi \
+    && if [ -x "$(command -v apt)" ]; then \
+         apt-get update && apt-get install -y --no-install-recommends curl htop strace bind-tools netcat unzip \
+         && CLEANUP="apt-get purge -y unzip && apt-get autoremove -y && rm -rf /var/lib/apt/lists/*"; \
+       else \
+         apk add --no-cache curl htop strace bind-tools netcat-openbsd unzip \
+         && CLEANUP="apk del unzip"; \
+       fi \
+    && curl -fsSL "https://awscli.amazonaws.com/awscli-exe-linux-${ARCH}-${AWS_CLI_VERSION}.zip" -o /tmp/awscli.zip \
+    && unzip -q /tmp/awscli.zip -d /tmp \
+    && /tmp/aws/install \
+    && rm -rf /tmp/aws /tmp/awscli.zip \
+    && curl -fsSL "https://github.com/async-profiler/async-profiler/releases/download/v${ASYNC_PROFILER_VERSION}/async-profiler-${ASYNC_PROFILER_VERSION}-linux-${AP_ARCH}.tar.gz" \
+       -o /tmp/ap.tar.gz \
+    && mkdir -p /opt/async-profiler \
+    && tar xzf /tmp/ap.tar.gz -C /opt/async-profiler --strip-components=1 \
+    && rm /tmp/ap.tar.gz \
+    && ln -s /opt/async-profiler/bin/asprof /usr/local/bin/asprof \
+    && eval "$CLEANUP" \
+    && aws --version \
+    && asprof --version
+
 USER druid
 VOLUME /opt/druid/var
 WORKDIR /opt/druid
 
+# org.apache.derby:derby is impacted by https://github.com/advisories/GHSA-rcjc-c4pj-xxrp.
+# Fixing it requires upgrading to Java 21 and Derby to 10.17.1.0.
+# However, since Derby is not used in production, we can safely exclude it from the Druid image.
+RUN rm /opt/druid/lib/derby-10.14.2.0.jar /opt/druid/lib/derbyclient-10.14.2.0.jar /opt/druid/lib/derbynet-10.14.2.0.jar
+
+# Create directory for PostgreSQL certificates
+RUN mkdir -p /opt/druid/.postgresql
+
+# Download RDS root CA certificates for both commercial and us-gov regions
+RUN curl -s https://truststore.pki.rds.amazonaws.com/global/global-bundle.pem -o /opt/druid/.postgresql/root.crt && \
+    curl -s https://truststore.pki.us-gov-west-1.rds.amazonaws.com/global/global-bundle.pem -o /opt/druid/.postgresql/root-fedramp.crt && \
+    chmod 644 /opt/druid/.postgresql/root.crt /opt/druid/.postgresql/root-fedramp.crt && \
+    chown -R druid:druid /opt/druid/.postgresql
+
 ENTRYPOINT ["/druid.sh"]