From ee711344f7023907d04b3d550fd5b64ff9c0453c Mon Sep 17 00:00:00 2001 From: Ayush Saxena Date: Fri, 19 Jun 2026 18:52:59 +0530 Subject: [PATCH 1/2] HIVE-29676: Add support for Multi Tenancy in Kubernetes Operator --- packaging/src/kubernetes/README.md | 318 +++++++++-- .../config/samples/hivecluster-full-ha.yaml | 4 +- .../crds/hiveclusters.hive.apache.org-v1.yml | 499 ++++++++++------- .../hive-operator/templates/hivecluster.yaml | 61 +- .../kubernetes/helm/hive-operator/values.yaml | 8 +- .../autoscaling/HiveClusterAutoscaler.java | 87 ++- .../autoscaling/LlapScalingStrategy.java | 42 +- .../autoscaling/TezAmScalingStrategy.java | 53 +- .../dependent/HiveConfigMapDependent.java | 20 - .../operator/dependent/HivePdbDependent.java | 26 +- .../HiveServer2DeploymentDependent.java | 31 +- .../dependent/HiveServiceDependent.java | 51 -- .../dependent/LlapResourceBuilder.java | 520 ++++++++++++++++++ .../dependent/LlapStatefulSetDependent.java | 231 -------- .../dependent/TezAmStatefulSetDependent.java | 194 ------- .../operator/model/HiveClusterSpec.java | 9 +- .../operator/model/HiveClusterStatus.java | 28 +- .../operator/model/spec/LlapSpec.java | 33 +- .../reconciler/HiveClusterReconciler.java | 348 ++++++++++-- .../operator/reconciler/HiveWorkflowSpec.java | 71 +-- .../kubernetes/operator/util/ConfigUtils.java | 19 + .../operator/util/HiveConfigBuilder.java | 58 +- .../hive/kubernetes/operator/util/Labels.java | 50 ++ .../ql/exec/tez/TezExternalSessionState.java | 6 + .../service/cli/session/SessionManager.java | 61 ++ .../metrics/common/MetricsConstant.java | 2 + 26 files changed, 1834 insertions(+), 996 deletions(-) create mode 100644 packaging/src/kubernetes/src/java/org/apache/hive/kubernetes/operator/dependent/LlapResourceBuilder.java delete mode 100644 packaging/src/kubernetes/src/java/org/apache/hive/kubernetes/operator/dependent/LlapStatefulSetDependent.java delete mode 100644 packaging/src/kubernetes/src/java/org/apache/hive/kubernetes/operator/dependent/TezAmStatefulSetDependent.java diff --git a/packaging/src/kubernetes/README.md b/packaging/src/kubernetes/README.md index 300eaf835120..6f832af3ad35 100644 --- a/packaging/src/kubernetes/README.md +++ b/packaging/src/kubernetes/README.md @@ -58,8 +58,8 @@ mvn clean package -pl packaging/src/kubernetes -Pkubernetes -DskipTests ## Quick Start (Helm) The Helm chart defaults to a **Full-HA** cluster (Metastore x2, HiveServer2 x2, -LLAP x2, TezAM x2). You only need to provide three things: database, ZooKeeper, -and storage. +LLAP x2, TezAM x2 — one TezAM per LLAP cluster). You only need to provide three +things: database, ZooKeeper, and storage. ### Prerequisites @@ -353,7 +353,7 @@ helm install hive ./helm/hive-operator \ --set 'cluster.storage.envVars[2].value=ozone' \ --set cluster.metastore.replicas=1 \ --set cluster.hiveServer2.replicas=1 \ - --set cluster.llap.enabled=false \ + --set cluster.llapClusters=[] \ --set cluster.tezAm.enabled=false ``` @@ -392,8 +392,7 @@ cluster: replicas: 1 hiveServer2: replicas: 1 - llap: - enabled: false + llapClusters: [] tezAm: enabled: false ``` @@ -477,7 +476,8 @@ cluster: requestsMemory: "2Gi" limitsMemory: "4Gi" - llap: + llapClusters: + - name: llap0 enabled: true replicas: 3 executors: 2 @@ -487,7 +487,7 @@ cluster: limitsMemory: "6Gi" tezAm: - replicas: 3 + enabled: true scratchStorageSize: "5Gi" ``` @@ -497,6 +497,210 @@ helm install hive ./helm/hive-operator -f values.yaml --- +## Multi-Tenant LLAP + +Multi-tenant LLAP allows you to run multiple independent LLAP clusters within a single +HiveCluster, each with its own resource pool, autoscaling policy, and TezAM instance. +Clients route queries to a specific LLAP cluster via a JDBC URL parameter. + +### How It Works + +``` + Multi-Tenant LLAP Architecture + + beeline (JDBC URL: ...;?hive.server2.tez.external.sessions.namespace=/tez-external-sessions/llap1;tez.am.registry.namespace=/llap1) + | + v + HiveServer2 (shared, routes sessions by LLAP target) + | + +-- Session targeting @llap0 --> TezAM-llap0 --> LLAP daemon llap0-0, llap0-1, ... + +-- Session targeting @llap1 --> TezAM-llap1 --> LLAP daemon llap1-0, llap1-1, ... + +-- Session targeting @llap2 --> TezAM-llap2 --> LLAP daemon llap2-0, llap2-1, ... +``` + +Each LLAP cluster is fully isolated: +- **Separate LLAP daemon StatefulSet** with independent executor count, memory, and replicas +- **Separate TezAM StatefulSet** (one per LLAP cluster) with its own ZooKeeper registration +- **Separate autoscaling** — each cluster scales independently based on its own metrics +- **Shared scratch PVC** (ReadWriteMany) for HS2 ↔ TezAM coordination files + +### Configuration + +**Values file:** + +```yaml +# values-multi-tenant.yaml +cluster: + database: + type: postgres + url: "jdbc:postgresql://postgres-postgresql:5432/metastore" + driver: "org.postgresql.Driver" + username: hive + passwordSecretRef: + name: hive-db-secret + key: password + driverJarUrl: "https://repo1.maven.org/maven2/org/postgresql/postgresql/42.7.5/postgresql-42.7.5.jar" + + zookeeper: + quorum: "zookeeper:2181" + + storage: + coreSiteOverrides: + fs.defaultFS: "s3a://hive" + fs.s3a.endpoint: "http://ozone-s3g-rest:9878" + fs.s3a.path.style.access: "true" + envVars: + - name: HADOOP_OPTIONAL_TOOLS + value: "hadoop-aws" + - name: AWS_ACCESS_KEY_ID + value: "ozone" + - name: AWS_SECRET_ACCESS_KEY + value: "ozone" + + hiveServer2: + replicas: 2 + + tezAm: + enabled: true + scratchStorageSize: "2Gi" + + # Three LLAP clusters: production (large), analytics (medium), dev (small) + llapClusters: + - name: production + enabled: true + replicas: 6 + executors: 4 + memoryMb: 8192 + resources: + requestsMemory: "8Gi" + limitsMemory: "10Gi" + autoscaling: + enabled: true + minReplicas: 2 + scaleUpThreshold: 2 + + - name: analytics + enabled: true + replicas: 4 + executors: 2 + memoryMb: 4096 + resources: + requestsMemory: "4Gi" + limitsMemory: "6Gi" + autoscaling: + enabled: true + minReplicas: 0 # scales to zero when idle + + - name: dev + enabled: true + replicas: 2 + executors: 1 + memoryMb: 1024 + resources: + requestsMemory: "1Gi" + limitsMemory: "2Gi" + autoscaling: + enabled: true + minReplicas: 0 # scales to zero when idle +``` + +```bash +helm install hive ./helm/hive-operator -f values-multi-tenant.yaml +``` + +### Resulting Kubernetes Resources + +For the above configuration, the operator creates: + +| Resource | Name | Purpose | +|----------|------|---------| +| StatefulSet | `hive-production` | LLAP daemons for production cluster | +| StatefulSet | `hive-tezam-production` | TezAM for production cluster | +| StatefulSet | `hive-analytics` | LLAP daemons for analytics cluster | +| StatefulSet | `hive-tezam-analytics` | TezAM for analytics cluster | +| StatefulSet | `hive-dev` | LLAP daemons for dev cluster | +| StatefulSet | `hive-tezam-dev` | TezAM for dev cluster | +| Service (headless) | `hive-production`, `hive-analytics`, `hive-dev` | LLAP daemon discovery | +| Service (headless) | `hive-tezam-production`, `hive-tezam-analytics`, `hive-tezam-dev` | TezAM discovery | +| ConfigMap | `hive-production-config`, etc. | `llap-daemon-site.xml` per cluster | +| ConfigMap | `hive-tezam-production-config`, etc. | `tez-site.xml` per cluster | +| PVC | `hive-scratch` | Shared scratch (ReadWriteMany) for HS2 ↔ TezAM | + +### Routing Queries to a Specific LLAP Cluster + +Clients must set the following properties to route queries to a specific LLAP cluster: + +| Property | Purpose | Example | +|----------|---------|---------| +| `hive.server2.tez.external.sessions.namespace` | TezAM session discovery (ZK watcher path) | `/tez-external-sessions/production` | +| `tez.am.registry.namespace` | TezAM registry lookup (cache key + AM resolution) | `/production` | + +**Option 1: JDBC URL parameter** + +```bash +beeline -u "jdbc:hive2://hive-hiveserver2:10001/;transportMode=http;httpPath=cliservice;?hive.server2.tez.external.sessions.namespace=/tez-external-sessions/production;tez.am.registry.namespace=/production" +``` + +**Option 2: SET command after connecting** + +```sql +SET hive.server2.tez.external.sessions.namespace=/tez-external-sessions/analytics; +SET tez.am.registry.namespace=/analytics; +INSERT INTO analytics_table SELECT * FROM source_table; +``` + +**Option 3: Per-session routing (programmatic)** + +```java +Properties props = new Properties(); +props.put("hive.server2.tez.external.sessions.namespace", "/tez-external-sessions/dev"); +props.put("tez.am.registry.namespace", "/dev"); +Connection conn = DriverManager.getConnection( + "jdbc:hive2://hive-hiveserver2:10001/;transportMode=http;httpPath=cliservice", props); +``` + +> **Note:** Auto-derivation of these properties from a single config is planned for a +> future release. Until then, both must be set explicitly. + +### ZooKeeper Registration + +Each LLAP cluster registers independently in ZooKeeper: + +| Cluster | LLAP daemons register at | TezAM registers session at | +|---------|--------------------------|----------------------------| +| `production` | `@production` (ZK service record) | `/tez-external-sessions/production/` | +| `analytics` | `@analytics` (ZK service record) | `/tez-external-sessions/analytics/` | +| `dev` | `@dev` (ZK service record) | `/tez-external-sessions/dev/` | + +HS2 discovers available TezAM sessions via `hive.server2.tez.external.sessions.namespace` and +uses `tez.am.registry.namespace` for client cache isolation. + +### Per-Cluster Autoscaling Isolation + +When autoscaling is enabled, metrics are fully isolated per LLAP cluster: + +- **LLAP executor metrics**: The operator selects pods by label `hive.apache.org/llap-cluster=`. + Only that cluster's pods are scraped and included in the scaling formula. +- **HS2 activation gate**: The operator reads `hs2_llap_target_sessions_` from HS2 pods. + Each cluster only wakes when sessions specifically target it. +- **TezAM scaling**: Each TezAM scales based on session demand for its paired LLAP cluster. + +This means scaling up `production` never affects `analytics` or `dev` replicas. + +### Adding/Removing LLAP Clusters + +To add a new cluster, append to `llapClusters[]` and run `helm upgrade`: + +```bash +helm upgrade hive ./helm/hive-operator -f values-multi-tenant.yaml +``` + +To remove a cluster, delete the entry from `llapClusters[]` and upgrade. The operator +automatically garbage-collects the removed cluster's StatefulSet, Service, ConfigMap, +and PDB via label-based discovery. + +--- + ## Verify ```bash @@ -634,9 +838,14 @@ terminates immediately once sessions drain, not after the full grace period. |-----------|-------------|----------------------|--------------| | **HS2** | 1 | N/A (always running) | N/A | | **HMS** | 1 | Never (always running) | N/A | -| **LLAP** | 0 | No HS2 sessions (activation gate fails) | HS2 has open sessions (next scrape) | +| **LLAP** | 0 | No HS2 sessions targeting this cluster | HS2 has sessions targeting this cluster (`hs2_llap_target_sessions_{name}`) | | **TezAM** | 0 | No HS2 sessions (activation gate fails) | HS2 has open sessions (next scrape) | +**Per-cluster LLAP wake:** When multiple LLAP clusters are configured (e.g., `llap0`, `llap1`), +each cluster wakes independently based on the `hs2_llap_target_sessions_{name}` metric. +If HS2 does not expose per-target metrics (older builds), the operator falls back to the generic +`hs2_open_sessions` metric (which wakes all LLAP clusters on any session). + ### Auto-Suspend (Full Cluster Hibernation) Auto-suspend goes beyond scale-to-zero — it fully hibernates the **entire** cluster @@ -774,11 +983,12 @@ cluster: enabled: true minReplicas: 1 - llap: + llapClusters: + - name: llap0 replicas: 8 autoscaling: enabled: true - minReplicas: 0 # scales to 0 via normal autoscaling when HS2 idle + minReplicas: 0 # scales to 0 via normal autoscaling when no sessions target this cluster tezAm: replicas: 10 @@ -789,9 +999,9 @@ cluster: With this configuration, the cluster lifecycle is: 1. Under load → all components scaled up by autoscaler -2. Load drops → autoscaler scales to minReplicas (HS2=1, HMS=1, LLAP=0, TezAM=0) +2. Load drops → autoscaler scales to minReplicas (HS2=1, HMS=1, LLAP clusters=0, TezAM=0) 3. HS2 idle (0 sessions) for 15 minutes → auto-suspend kicks in → HS2, LLAP, TezAM to 0 (HMS stays at minReplicas) -4. `kubectl patch hivecluster hive --type=merge -p '{"spec":{"suspend":false}}'` → wake → HS2=1, LLAP=1, TezAM=1 +4. `kubectl patch hivecluster hive --type=merge -p '{"spec":{"suspend":false}}'` → wake → HS2=1, each LLAP cluster=1, TezAM=1 5. User connects → autoscaler detects sessions → scales up as needed ### CPU-Based Scaling (HS2 and HMS) @@ -915,7 +1125,7 @@ helm install hive ./helm/hive-operator \ --set 'cluster.storage.envVars[2].value=ozone' \ --set cluster.hiveServer2.autoscaling.enabled=true \ --set cluster.metastore.autoscaling.enabled=true \ - --set cluster.llap.autoscaling.enabled=true \ + --set 'cluster.llapClusters[0].autoscaling.enabled=true' \ --set cluster.tezAm.autoscaling.enabled=true ``` @@ -971,11 +1181,12 @@ cluster: # gracePeriodSeconds: 60 # default — fast drain (HMS is stateless) # metricsScrapeIntervalSeconds: 10 # default — operator scrape interval - llap: + llapClusters: + - name: llap0 replicas: 8 # Acts as maxReplicas when autoscaling is enabled autoscaling: enabled: true - # minReplicas: 0 # default — scale to zero when no HS2 sessions + # minReplicas: 0 # default — scale to zero when no sessions target this cluster # scaleUpThreshold: 1 # default — total busy slots (queued+running) triggering scale-up # scaleUpStabilizationSeconds: 60 # default — scale-up window # scaleDownStabilizationSeconds: 900 # default — scale-down window (long — scaling down destroys cache) @@ -1176,28 +1387,43 @@ setup is needed — simply connect to HS2 and the operator wakes LLAP/TezAM as n | `cluster.hiveServer2.extraVolumes` | `[]` | Additional volumes for HS2 pods | | `cluster.hiveServer2.extraVolumeMounts` | `[]` | Additional volume mounts for HS2 containers | -### LLAP +### LLAP Clusters + +LLAP is configured as an array (`llapClusters`) to support multi-tenant deployments with +independent scaling. Each entry creates a separate LLAP StatefulSet, Service, ConfigMap, +and a paired TezAM StatefulSet (when `tezAm.enabled: true`). | Value | Default | Description | |-------|---------|-------------| -| `cluster.llap.enabled` | `true` | Enable LLAP daemons | -| `cluster.llap.replicas` | `2` | Replica count | -| `cluster.llap.executors` | `1` | Executors per daemon | -| `cluster.llap.memoryMb` | `1024` | Memory per daemon (MB) | -| `cluster.llap.serviceHosts` | `@llap0` | LLAP ZK identity | -| `cluster.llap.resources` | `{}` | CPU/memory | -| `cluster.llap.configOverrides` | `{}` | Extra LLAP config properties | -| `cluster.llap.extraVolumes` | `[]` | Additional volumes for LLAP pods | -| `cluster.llap.extraVolumeMounts` | `[]` | Additional volume mounts for LLAP containers | +| `cluster.llapClusters[].name` | *(required)* | Unique name for this LLAP cluster (e.g., `llap0`) | +| `cluster.llapClusters[].enabled` | `true` | Enable this LLAP cluster | +| `cluster.llapClusters[].replicas` | `2` | Replica count (maxReplicas when autoscaling enabled) | +| `cluster.llapClusters[].executors` | `1` | Executors per daemon | +| `cluster.llapClusters[].memoryMb` | `1024` | Memory per daemon (MB) | +| `cluster.llapClusters[].resources` | `{}` | CPU/memory | +| `cluster.llapClusters[].configOverrides` | `{}` | Extra LLAP config properties | +| `cluster.llapClusters[].extraVolumes` | `[]` | Additional volumes for LLAP pods | +| `cluster.llapClusters[].extraVolumeMounts` | `[]` | Additional volume mounts for LLAP containers | +| `cluster.llapClusters[].autoscaling.enabled` | `false` | Enable per-cluster autoscaling | +| `cluster.llapClusters[].autoscaling.minReplicas` | `0` | Min replicas (0 = scale to zero) | +| `cluster.llapClusters[].autoscaling.scaleUpThreshold` | `1` | Busy-slot threshold for scale-up | + +Sessions target a specific LLAP cluster by setting +`hive.server2.tez.external.sessions.namespace=/tez-external-sessions/{name}` and +`tez.am.registry.namespace=/{name}` in the JDBC URL or via SET commands. ### Tez AM +TezAM is deployed as one StatefulSet per LLAP cluster. The global `tezAm` section +controls shared settings (enabled flag, scratch PVC). Per-LLAP TezAM settings +(replicas, autoscaling) can be overridden in each `llapClusters[].tezAm` entry. + | Value | Default | Description | |-------|---------|-------------| -| `cluster.tezAm.enabled` | `true` | Enable Tez Application Master | -| `cluster.tezAm.replicas` | `2` | Replica count | -| `cluster.tezAm.scratchStorageSize` | `1Gi` | Shared scratch PVC size | -| `cluster.tezAm.scratchStorageClassName` | | StorageClass (must support RWX) | +| `cluster.tezAm.enabled` | `true` | Enable Tez Application Master (one per LLAP cluster) | +| `cluster.tezAm.replicas` | `2` | Default replica count per TezAM (overridable per LLAP cluster) | +| `cluster.tezAm.scratchStorageSize` | `1Gi` | Shared scratch PVC size (single PVC shared by all HS2 and TezAM pods) | +| `cluster.tezAm.scratchStorageClassName` | | StorageClass (must support ReadWriteMany) | | `cluster.tezAm.resources` | `{}` | CPU/memory | | `cluster.tezAm.configOverrides` | `{}` | Extra TezAM config properties | | `cluster.tezAm.extraVolumes` | `[]` | Additional volumes for TezAM pods | @@ -1345,20 +1571,32 @@ HiveCluster CR v HiveClusterReconciler | - +-- HadoopConfigMapDependent (core-site.xml) - +-- MetastoreConfigMapDependent (metastore-site.xml) - +-- HiveServer2ConfigMapDependent (hive-site.xml + tez-site.xml) - +-- SchemaInitJobDependent (schematool -initOrUpgradeSchema) - +-- MetastoreDeploymentDependent --> MetastoreServiceDependent - +-- HiveServer2DeploymentDependent --> HiveServer2ServiceDependent - +-- LlapStatefulSetDependent --> LlapServiceDependent (optional) - +-- ScratchPvcDependent (shared scratch PVC, optional) - +-- TezAmStatefulSetDependent --> TezAmServiceDependent (optional) + +-- [JOSDK Workflow Dependents] + | +-- HadoopConfigMapDependent (core-site.xml) + | +-- MetastoreConfigMapDependent (metastore-site.xml) + | +-- HiveServer2ConfigMapDependent (hive-site.xml + tez-site.xml) + | +-- SchemaInitJobDependent (schematool -initOrUpgradeSchema) + | +-- MetastoreDeploymentDependent --> MetastoreServiceDependent + | +-- HiveServer2DeploymentDependent --> HiveServer2ServiceDependent + | +-- ScratchPvcDependent (shared scratch PVC for HS2 ↔ TezAM) + | + +-- [Imperative] Per-LLAP-Cluster Resources (for each llapClusters[] entry): + +-- LLAP StatefulSet + headless Service + ConfigMap + PDB + +-- TezAM StatefulSet + headless Service + ConfigMap (one TezAM per LLAP cluster) ``` +LLAP clusters and their paired TezAM instances are managed imperatively by the reconciler +(not via JOSDK workflow dependents) because the number of clusters is dynamic — determined +at runtime from the CR spec. Each `llapClusters[]` entry produces: +- **LLAP**: StatefulSet (`{cluster}-{name}`), headless Service, ConfigMap (`llap-daemon-site.xml`), PDB +- **TezAM**: StatefulSet (`{cluster}-tezam-{name}`), headless Service, ConfigMap (`tez-site.xml`) + +All imperative resources are applied via `serverSideApply()`. Removed LLAP clusters (and +their TezAMs) are garbage-collected automatically using label-based discovery. + **Startup order:** 1. ConfigMaps (Hadoop, Metastore, HiveServer2) 2. Schema Init Job [if Metastore enabled] 3. Metastore Deployment + Service [if enabled] -4. HiveServer2 Deployment + Service -5. LLAP + TezAM [if enabled] +4. HiveServer2 Deployment + Service + Shared Scratch PVC +5. LLAP clusters + paired TezAM instances [if enabled] diff --git a/packaging/src/kubernetes/config/samples/hivecluster-full-ha.yaml b/packaging/src/kubernetes/config/samples/hivecluster-full-ha.yaml index cc65852d4f35..22ca3c09458f 100644 --- a/packaging/src/kubernetes/config/samples/hivecluster-full-ha.yaml +++ b/packaging/src/kubernetes/config/samples/hivecluster-full-ha.yaml @@ -44,12 +44,12 @@ spec: configOverrides: hive.server2.enable.doAs: "false" - llap: + llapClusters: + - name: llap0 enabled: true replicas: 2 executors: 1 memoryMb: 1024 - serviceHosts: "@llap0" resources: requestsMemory: "2Gi" limitsMemory: "3Gi" diff --git a/packaging/src/kubernetes/helm/hive-operator/crds/hiveclusters.hive.apache.org-v1.yml b/packaging/src/kubernetes/helm/hive-operator/crds/hiveclusters.hive.apache.org-v1.yml index fdd8ef33da25..8bbbccb07354 100644 --- a/packaging/src/kubernetes/helm/hive-operator/crds/hiveclusters.hive.apache.org-v1.yml +++ b/packaging/src/kubernetes/helm/hive-operator/crds/hiveclusters.hive.apache.org-v1.yml @@ -248,153 +248,244 @@ spec: imagePullPolicy: description: "Image pull policy: Always, Never, or IfNotPresent" type: string - llap: - description: LLAP daemon configuration. Enabled by default. - properties: - autoscaling: - description: "Autoscaling configuration (operator-driven, no external\ - \ dependencies)" - properties: - cpuScaleDownThreshold: - default: 30 - description: CPU percentage (0-100) below which scale-down - is considered. Only applies to HS2 and HMS. - type: integer - cpuScaleUpThreshold: - default: 90 - description: CPU percentage (0-100) that triggers scale-up. - Only applies to HS2 and HMS. Set to 0 to disable CPU-based - scaling. - type: integer - enabled: - default: false - description: Whether autoscaling is enabled for this component - type: boolean - gracePeriodSeconds: - default: 3600 - description: Maximum time in seconds to wait for graceful - drain during scale-down before the pod is forcibly terminated. - The pod terminates immediately once sessions/connections - drain to 0; this value is only the upper safety cap. - type: integer - metricsPort: - default: 9404 - description: Port on which the Prometheus JMX Exporter serves - metrics. The operator scrapes this port on each pod for - autoscaling decisions. - type: integer - metricsScrapeIntervalSeconds: - default: 10 - description: How often (seconds) the operator scrapes JMX - metrics from pods. Lower values make autoscaling react faster. - type: integer - minReplicas: - default: 0 - description: "Minimum number of replicas (floor for scale-down).\ - \ Set to 0 for scale-to-zero (LLAP, TezAM only; HS2 minimum\ - \ is 1)" - type: integer - scaleDownStabilizationSeconds: - default: 600 - description: Stabilization window in seconds for scale-down - decisions. How long metrics must consistently indicate fewer - replicas before scale-down occurs. Also acts as the cooldown - between consecutive scale-downs. - type: integer - scaleUpStabilizationSeconds: - default: 60 - description: Stabilization window in seconds for scale-up - decisions. Picks the highest recommendation within this - window to prevent flapping. - type: integer - scaleUpThreshold: - default: 80 - description: "Threshold that triggers scale-up (component-specific:\ - \ sessions per pod for HS2, request rate for HMS, busy slots\ - \ per daemon for LLAP). Not used by TezAM (demand-based:\ - \ 1 TezAM per session)." - type: integer - type: object - configOverrides: - additionalProperties: + llapClusters: + description: "LLAP compute clusters. Each entry is an independent\ + \ LLAP cluster with its own StatefulSet, autoscaling, and ZooKeeper\ + \ registration. Users select a cluster via hive.llap.daemon.service.hosts=@{name}\ + \ in their session." + items: + properties: + autoscaling: + description: "Autoscaling configuration (operator-driven, no\ + \ external dependencies)" + properties: + cpuScaleDownThreshold: + default: 30 + description: CPU percentage (0-100) below which scale-down + is considered. Only applies to HS2 and HMS. + type: integer + cpuScaleUpThreshold: + default: 90 + description: CPU percentage (0-100) that triggers scale-up. + Only applies to HS2 and HMS. Set to 0 to disable CPU-based + scaling. + type: integer + enabled: + default: false + description: Whether autoscaling is enabled for this component + type: boolean + gracePeriodSeconds: + default: 3600 + description: Maximum time in seconds to wait for graceful + drain during scale-down before the pod is forcibly terminated. + The pod terminates immediately once sessions/connections + drain to 0; this value is only the upper safety cap. + type: integer + metricsPort: + default: 9404 + description: Port on which the Prometheus JMX Exporter serves + metrics. The operator scrapes this port on each pod for + autoscaling decisions. + type: integer + metricsScrapeIntervalSeconds: + default: 10 + description: How often (seconds) the operator scrapes JMX + metrics from pods. Lower values make autoscaling react + faster. + type: integer + minReplicas: + default: 0 + description: "Minimum number of replicas (floor for scale-down).\ + \ Set to 0 for scale-to-zero (LLAP, TezAM only; HS2 minimum\ + \ is 1)" + type: integer + scaleDownStabilizationSeconds: + default: 600 + description: Stabilization window in seconds for scale-down + decisions. How long metrics must consistently indicate + fewer replicas before scale-down occurs. Also acts as + the cooldown between consecutive scale-downs. + type: integer + scaleUpStabilizationSeconds: + default: 60 + description: Stabilization window in seconds for scale-up + decisions. Picks the highest recommendation within this + window to prevent flapping. + type: integer + scaleUpThreshold: + default: 80 + description: "Threshold that triggers scale-up (component-specific:\ + \ sessions per pod for HS2, request rate for HMS, busy\ + \ slots per daemon for LLAP). Not used by TezAM (demand-based:\ + \ 1 TezAM per session)." + type: integer + type: object + configOverrides: + additionalProperties: + type: string + description: Additional configuration overrides as key-value + pairs + type: object + enabled: + default: true + description: Whether LLAP is enabled + type: boolean + executors: + default: 1 + description: Number of LLAP executors per daemon + type: integer + extraVolumeMounts: + description: Additional volume mounts for the container + items: + type: object + type: array + x-kubernetes-preserve-unknown-fields: true + extraVolumes: + description: "Additional volumes to attach to the pod (e.g.,\ + \ for keytabs or truststores)" + items: + type: object + type: array + x-kubernetes-preserve-unknown-fields: true + memoryMb: + default: 1024 + description: Memory in MB per LLAP daemon instance + type: integer + name: + description: "Unique name for this LLAP cluster (e.g. llap0,\ + \ llap1). Used as the ZooKeeper registration namespace and\ + \ Kubernetes resource suffix." type: string - description: Additional configuration overrides as key-value pairs - type: object - enabled: - default: true - description: Whether LLAP is enabled - type: boolean - executors: - default: 1 - description: Number of LLAP executors per daemon - type: integer - extraVolumeMounts: - description: Additional volume mounts for the container - items: + readinessProbe: + description: Readiness probe configuration + properties: + failureThreshold: + description: Minimum consecutive failures for the probe + to be considered failed after having succeeded. + type: integer + initialDelaySeconds: + description: Number of seconds after the container has started + before probes are initiated. + type: integer + periodSeconds: + description: How often (in seconds) to perform the probe. + type: integer + successThreshold: + description: Minimum consecutive successes for the probe + to be considered successful after having failed. + type: integer + timeoutSeconds: + description: Number of seconds after which the probe times + out. + type: integer type: object - type: array - x-kubernetes-preserve-unknown-fields: true - extraVolumes: - description: "Additional volumes to attach to the pod (e.g., for\ - \ keytabs or truststores)" - items: + replicas: + default: 1 + description: Number of replicas + type: integer + resources: + description: Resource requirements for pods + properties: + limitsCpu: + description: "CPU limit (e.g. 2, 1000m)" + type: string + limitsMemory: + description: "Memory limit (e.g. 2Gi, 1024Mi)" + type: string + requestsCpu: + default: 500m + description: "CPU request (e.g. 500m, 1)" + type: string + requestsMemory: + default: 1Gi + description: "Memory request (e.g. 1Gi, 512Mi)" + type: string type: object - type: array - x-kubernetes-preserve-unknown-fields: true - memoryMb: - default: 1024 - description: Memory in MB per LLAP daemon instance - type: integer - readinessProbe: - description: Readiness probe configuration - properties: - failureThreshold: - description: Minimum consecutive failures for the probe to - be considered failed after having succeeded. - type: integer - initialDelaySeconds: - description: Number of seconds after the container has started - before probes are initiated. - type: integer - periodSeconds: - description: How often (in seconds) to perform the probe. - type: integer - successThreshold: - description: Minimum consecutive successes for the probe to - be considered successful after having failed. - type: integer - timeoutSeconds: - description: Number of seconds after which the probe times - out. - type: integer - type: object - replicas: - default: 1 - description: Number of replicas - type: integer - resources: - description: Resource requirements for pods - properties: - limitsCpu: - description: "CPU limit (e.g. 2, 1000m)" - type: string - limitsMemory: - description: "Memory limit (e.g. 2Gi, 1024Mi)" - type: string - requestsCpu: - default: 500m - description: "CPU request (e.g. 500m, 1)" - type: string - requestsMemory: - default: 1Gi - description: "Memory request (e.g. 1Gi, 512Mi)" - type: string - type: object - serviceHosts: - description: LLAP service hosts identifier for ZooKeeper registration - type: string - type: object - x-kubernetes-preserve-unknown-fields: true + serviceHosts: + description: "LLAP service hosts identifier for ZooKeeper registration.\ + \ Defaults to @{name} (e.g. @llap0)." + type: string + tezAm: + description: Per-LLAP TezAM configuration. Each LLAP cluster + gets its own TezAM with independent replica count and autoscaling. + properties: + autoscaling: + description: Autoscaling configuration for this LLAP cluster's + TezAM + properties: + cpuScaleDownThreshold: + default: 30 + description: CPU percentage (0-100) below which scale-down + is considered. Only applies to HS2 and HMS. + type: integer + cpuScaleUpThreshold: + default: 90 + description: CPU percentage (0-100) that triggers scale-up. + Only applies to HS2 and HMS. Set to 0 to disable CPU-based + scaling. + type: integer + enabled: + default: false + description: Whether autoscaling is enabled for this + component + type: boolean + gracePeriodSeconds: + default: 3600 + description: Maximum time in seconds to wait for graceful + drain during scale-down before the pod is forcibly + terminated. The pod terminates immediately once sessions/connections + drain to 0; this value is only the upper safety cap. + type: integer + metricsPort: + default: 9404 + description: Port on which the Prometheus JMX Exporter + serves metrics. The operator scrapes this port on + each pod for autoscaling decisions. + type: integer + metricsScrapeIntervalSeconds: + default: 10 + description: How often (seconds) the operator scrapes + JMX metrics from pods. Lower values make autoscaling + react faster. + type: integer + minReplicas: + default: 0 + description: "Minimum number of replicas (floor for\ + \ scale-down). Set to 0 for scale-to-zero (LLAP, TezAM\ + \ only; HS2 minimum is 1)" + type: integer + scaleDownStabilizationSeconds: + default: 600 + description: Stabilization window in seconds for scale-down + decisions. How long metrics must consistently indicate + fewer replicas before scale-down occurs. Also acts + as the cooldown between consecutive scale-downs. + type: integer + scaleUpStabilizationSeconds: + default: 60 + description: Stabilization window in seconds for scale-up + decisions. Picks the highest recommendation within + this window to prevent flapping. + type: integer + scaleUpThreshold: + default: 80 + description: "Threshold that triggers scale-up (component-specific:\ + \ sessions per pod for HS2, request rate for HMS,\ + \ busy slots per daemon for LLAP). Not used by TezAM\ + \ (demand-based: 1 TezAM per session)." + type: integer + type: object + replicas: + default: 1 + description: Max number of TezAM replicas for this LLAP + cluster + type: integer + type: object + required: + - name + type: object + x-kubernetes-preserve-unknown-fields: true + type: array metastore: description: Metastore component configuration properties: @@ -805,35 +896,37 @@ spec: type: integer idleSince: type: string - llap: - properties: - autoscaling: - properties: - cpuProposedReplicas: - type: integer - cpuScaleUpThreshold: - type: integer - currentCpuPercent: - type: number - currentMetricValue: - type: integer - lastScaleTime: - type: string - proposedReplicas: - type: integer - scaleUpThreshold: - type: integer - type: object - currentReplicas: - type: integer - maxReplicas: - type: integer - minReplicas: - type: integer - phase: - type: string - readyReplicas: - type: integer + llapClusters: + additionalProperties: + properties: + autoscaling: + properties: + cpuProposedReplicas: + type: integer + cpuScaleUpThreshold: + type: integer + currentCpuPercent: + type: number + currentMetricValue: + type: integer + lastScaleTime: + type: string + proposedReplicas: + type: integer + scaleUpThreshold: + type: integer + type: object + currentReplicas: + type: integer + maxReplicas: + type: integer + minReplicas: + type: integer + phase: + type: string + readyReplicas: + type: integer + type: object type: object metastore: properties: @@ -869,35 +962,37 @@ spec: type: integer suspendedSince: type: string - tezAm: - properties: - autoscaling: - properties: - cpuProposedReplicas: - type: integer - cpuScaleUpThreshold: - type: integer - currentCpuPercent: - type: number - currentMetricValue: - type: integer - lastScaleTime: - type: string - proposedReplicas: - type: integer - scaleUpThreshold: - type: integer - type: object - currentReplicas: - type: integer - maxReplicas: - type: integer - minReplicas: - type: integer - phase: - type: string - readyReplicas: - type: integer + tezAmClusters: + additionalProperties: + properties: + autoscaling: + properties: + cpuProposedReplicas: + type: integer + cpuScaleUpThreshold: + type: integer + currentCpuPercent: + type: number + currentMetricValue: + type: integer + lastScaleTime: + type: string + proposedReplicas: + type: integer + scaleUpThreshold: + type: integer + type: object + currentReplicas: + type: integer + maxReplicas: + type: integer + minReplicas: + type: integer + phase: + type: string + readyReplicas: + type: integer + type: object type: object type: object type: object diff --git a/packaging/src/kubernetes/helm/hive-operator/templates/hivecluster.yaml b/packaging/src/kubernetes/helm/hive-operator/templates/hivecluster.yaml index 6b053ad4ec3b..c007f0ba3a98 100644 --- a/packaging/src/kubernetes/helm/hive-operator/templates/hivecluster.yaml +++ b/packaging/src/kubernetes/helm/hive-operator/templates/hivecluster.yaml @@ -121,40 +121,55 @@ spec: cpuScaleDownThreshold: {{ .Values.cluster.hiveServer2.autoscaling.cpuScaleDownThreshold | default 30 }} {{- end }} - llap: - enabled: {{ .Values.cluster.llap.enabled }} - {{- if .Values.cluster.llap.enabled }} - replicas: {{ .Values.cluster.llap.replicas }} - executors: {{ .Values.cluster.llap.executors }} - memoryMb: {{ .Values.cluster.llap.memoryMb }} - serviceHosts: {{ .Values.cluster.llap.serviceHosts | quote }} - {{- if .Values.cluster.llap.resources }} + {{- if .Values.cluster.llapClusters }} + llapClusters: + {{- range .Values.cluster.llapClusters }} + - name: {{ .name }} + enabled: {{ .enabled | default true }} + replicas: {{ .replicas | default 2 }} + executors: {{ .executors | default 1 }} + memoryMb: {{ .memoryMb | default 1024 }} + {{- if .resources }} resources: - {{- toYaml .Values.cluster.llap.resources | nindent 6 }} + {{- toYaml .resources | nindent 6 }} {{- end }} - {{- if .Values.cluster.llap.configOverrides }} + {{- if .configOverrides }} configOverrides: - {{- toYaml .Values.cluster.llap.configOverrides | nindent 6 }} + {{- toYaml .configOverrides | nindent 6 }} {{- end }} - {{- if .Values.cluster.llap.extraVolumes }} + {{- if .extraVolumes }} extraVolumes: - {{- toYaml .Values.cluster.llap.extraVolumes | nindent 6 }} + {{- toYaml .extraVolumes | nindent 6 }} {{- end }} - {{- if .Values.cluster.llap.extraVolumeMounts }} + {{- if .extraVolumeMounts }} extraVolumeMounts: - {{- toYaml .Values.cluster.llap.extraVolumeMounts | nindent 6 }} + {{- toYaml .extraVolumeMounts | nindent 6 }} {{- end }} - {{- if and .Values.cluster.llap.autoscaling .Values.cluster.llap.autoscaling.enabled }} + {{- if and .autoscaling .autoscaling.enabled }} autoscaling: enabled: true - minReplicas: {{ .Values.cluster.llap.autoscaling.minReplicas }} - scaleUpThreshold: {{ .Values.cluster.llap.autoscaling.scaleUpThreshold }} - scaleUpStabilizationSeconds: {{ .Values.cluster.llap.autoscaling.scaleUpStabilizationSeconds }} - scaleDownStabilizationSeconds: {{ .Values.cluster.llap.autoscaling.scaleDownStabilizationSeconds }} - gracePeriodSeconds: {{ .Values.cluster.llap.autoscaling.gracePeriodSeconds }} - metricsScrapeIntervalSeconds: {{ .Values.cluster.llap.autoscaling.metricsScrapeIntervalSeconds | default 10 }} - {{- end }} + minReplicas: {{ .autoscaling.minReplicas }} + scaleUpThreshold: {{ .autoscaling.scaleUpThreshold }} + scaleUpStabilizationSeconds: {{ .autoscaling.scaleUpStabilizationSeconds }} + scaleDownStabilizationSeconds: {{ .autoscaling.scaleDownStabilizationSeconds }} + gracePeriodSeconds: {{ .autoscaling.gracePeriodSeconds }} + metricsScrapeIntervalSeconds: {{ .autoscaling.metricsScrapeIntervalSeconds | default 10 }} + {{- end }} + {{- if .tezAm }} + tezAm: + replicas: {{ .tezAm.replicas | default 1 }} + {{- if and .tezAm.autoscaling .tezAm.autoscaling.enabled }} + autoscaling: + enabled: true + minReplicas: {{ .tezAm.autoscaling.minReplicas }} + scaleUpStabilizationSeconds: {{ .tezAm.autoscaling.scaleUpStabilizationSeconds }} + scaleDownStabilizationSeconds: {{ .tezAm.autoscaling.scaleDownStabilizationSeconds }} + gracePeriodSeconds: {{ .tezAm.autoscaling.gracePeriodSeconds }} + metricsScrapeIntervalSeconds: {{ .tezAm.autoscaling.metricsScrapeIntervalSeconds | default 10 }} + {{- end }} {{- end }} + {{- end }} + {{- end }} tezAm: enabled: {{ .Values.cluster.tezAm.enabled }} diff --git a/packaging/src/kubernetes/helm/hive-operator/values.yaml b/packaging/src/kubernetes/helm/hive-operator/values.yaml index 99e9b47b6a17..28da1c3c9291 100644 --- a/packaging/src/kubernetes/helm/hive-operator/values.yaml +++ b/packaging/src/kubernetes/helm/hive-operator/values.yaml @@ -168,14 +168,16 @@ cluster: cpuScaleDownThreshold: 30 # --------------------------------------------------------------------------- - # LLAP — enabled by default for full-HA + # LLAP CLUSTERS — each entry is an independent LLAP cluster with its own + # StatefulSet, autoscaling, and ZooKeeper registration. + # Users select a cluster via: SET hive.llap.daemon.service.hosts=@llap0; # --------------------------------------------------------------------------- - llap: + llapClusters: + - name: llap0 enabled: true replicas: 2 executors: 1 memoryMb: 1024 - serviceHosts: "@llap0" resources: {} configOverrides: {} extraVolumes: [] diff --git a/packaging/src/kubernetes/src/java/org/apache/hive/kubernetes/operator/autoscaling/HiveClusterAutoscaler.java b/packaging/src/kubernetes/src/java/org/apache/hive/kubernetes/operator/autoscaling/HiveClusterAutoscaler.java index 2f0e661a27ca..f1049e0a1fcd 100644 --- a/packaging/src/kubernetes/src/java/org/apache/hive/kubernetes/operator/autoscaling/HiveClusterAutoscaler.java +++ b/packaging/src/kubernetes/src/java/org/apache/hive/kubernetes/operator/autoscaling/HiveClusterAutoscaler.java @@ -29,6 +29,7 @@ import org.apache.hive.kubernetes.operator.model.HiveCluster; import org.apache.hive.kubernetes.operator.model.HiveClusterSpec; import org.apache.hive.kubernetes.operator.model.spec.AutoscalingSpec; +import org.apache.hive.kubernetes.operator.model.spec.LlapSpec; import org.apache.hive.kubernetes.operator.model.status.AutoscalingStatus; import org.apache.hive.kubernetes.operator.util.ConfigUtils; import org.apache.hive.kubernetes.operator.util.Labels; @@ -203,32 +204,46 @@ public AutoscalingEvaluation evaluate(HiveCluster cluster, KubernetesClient clie spec.metastore().replicas(), patches, statuses, msMetrics); } - // LLAP - if (spec.llap().isEnabled() && spec.llap().autoscaling().isEnabled()) { - AutoscalingSpec llapAuto = spec.llap().autoscaling(); - Map llapSelector = Labels.selectorForComponent(cluster, ConfigUtils.COMPONENT_LLAP); + // LLAP clusters (each evaluated independently) + for (LlapSpec llapSpec : spec.llapClusters()) { + if (!llapSpec.isEnabled() || !llapSpec.autoscaling().isEnabled()) { + continue; + } + String llapComponentKey = ConfigUtils.llapComponentKey(llapSpec.name()); + AutoscalingSpec llapAuto = llapSpec.autoscaling(); + Map llapSelector = Labels.selectorForLlapCluster(cluster, llapSpec.name()); bgScraper.registerOrUpdate(namespace, clusterName, - ConfigUtils.COMPONENT_LLAP, llapSelector, + llapComponentKey, llapSelector, llapAuto.metricsPort(), llapAuto.metricsScrapeIntervalSeconds()); - String llapKey = namespace + "/" + clusterName + "/" + ConfigUtils.COMPONENT_LLAP; + String llapKey = cacheKey(namespace, clusterName, llapComponentKey); List llapMetrics = metricsCache.getOrEmpty(llapKey, llapAuto.metricsScrapeIntervalSeconds() * 3); evaluateComponent(cluster, client, namespace, clusterName, - ConfigUtils.COMPONENT_LLAP, llapAuto, - spec.llap().replicas(), patches, statuses, llapMetrics); + llapComponentKey, llapAuto, + llapSpec.replicas(), patches, statuses, llapMetrics); } - // TezAM - if (spec.tezAm().isEnabled() && spec.tezAm().autoscaling().isEnabled()) { - AutoscalingSpec tezAuto = spec.tezAm().autoscaling(); - Map tezSelector = Labels.selectorForComponent(cluster, ConfigUtils.COMPONENT_TEZAM); - bgScraper.registerOrUpdate(namespace, clusterName, - ConfigUtils.COMPONENT_TEZAM, tezSelector, - tezAuto.metricsPort(), tezAuto.metricsScrapeIntervalSeconds()); - String tezKey = namespace + "/" + clusterName + "/" + ConfigUtils.COMPONENT_TEZAM; - List tezMetrics = metricsCache.getOrEmpty(tezKey, tezAuto.metricsScrapeIntervalSeconds() * 3); - evaluateComponent(cluster, client, namespace, clusterName, - ConfigUtils.COMPONENT_TEZAM, tezAuto, - spec.tezAm().replicas(), patches, statuses, tezMetrics); + // Per-LLAP TezAM (one TezAM per LLAP cluster, each with its own autoscaling config) + if (spec.tezAm().isEnabled()) { + for (LlapSpec llapSpec : spec.llapClusters()) { + if (!llapSpec.isEnabled()) { + continue; + } + LlapSpec.LlapTezAmSpec perLlapTezAm = llapSpec.tezAm(); + if (!perLlapTezAm.autoscaling().isEnabled()) { + continue; + } + AutoscalingSpec tezAuto = perLlapTezAm.autoscaling(); + String tezAmComponentKey = ConfigUtils.tezAmComponentKey(llapSpec.name()); + Map tezSelector = Labels.selectorForTezAmCluster(cluster, llapSpec.name()); + bgScraper.registerOrUpdate(namespace, clusterName, + tezAmComponentKey, tezSelector, + tezAuto.metricsPort(), tezAuto.metricsScrapeIntervalSeconds()); + String tezKey = cacheKey(namespace, clusterName, tezAmComponentKey); + List tezMetrics = metricsCache.getOrEmpty(tezKey, tezAuto.metricsScrapeIntervalSeconds() * 3); + evaluateComponent(cluster, client, namespace, clusterName, + tezAmComponentKey, tezAuto, + perLlapTezAm.replicas(), patches, statuses, tezMetrics); + } } return new AutoscalingEvaluation(patches, statuses); @@ -258,8 +273,8 @@ private void evaluateComponent(HiveCluster cluster, KubernetesClient client, // For LLAP and TezAM, scaling decisions are based on HS2 metrics (activation gate), // not their own pod metrics. Allow evaluation even with 0 own pods. - boolean usesHs2Activation = ConfigUtils.COMPONENT_LLAP.equals(component) - || ConfigUtils.COMPONENT_TEZAM.equals(component); + boolean usesHs2Activation = component.startsWith(ConfigUtils.COMPONENT_LLAP + "-") + || component.startsWith(ConfigUtils.COMPONENT_TEZAM + "-"); if (metrics.isEmpty() && !usesHs2Activation) { LOG.debug("[{}] No ready pods to scrape, skipping", component); @@ -305,19 +320,39 @@ private void evaluateComponent(HiveCluster cluster, KubernetesClient client, } private ScalingStrategy createStrategy(String component, HiveCluster cluster) { + if (component.startsWith(ConfigUtils.COMPONENT_LLAP + "-")) { + String llapName = component.substring(ConfigUtils.COMPONENT_LLAP.length() + 1); + return new LlapScalingStrategy(this, cluster, llapName); + } + if (component.startsWith(ConfigUtils.COMPONENT_TEZAM + "-")) { + String llapName = component.substring(ConfigUtils.COMPONENT_TEZAM.length() + 1); + return new TezAmScalingStrategy(this, cluster, llapName); + } return switch (component) { case ConfigUtils.COMPONENT_HIVESERVER2 -> new HiveServer2ScalingStrategy(); case ConfigUtils.COMPONENT_METASTORE -> new MetastoreScalingStrategy(); - case ConfigUtils.COMPONENT_LLAP -> new LlapScalingStrategy(this, cluster); - case ConfigUtils.COMPONENT_TEZAM -> new TezAmScalingStrategy(this, cluster); default -> throw new IllegalArgumentException("Unknown component: " + component); }; } private int getCurrentReplicas(KubernetesClient client, String namespace, String clusterName, String component) { - String workloadName = clusterName + "-" + component; - if (ConfigUtils.COMPONENT_LLAP.equals(component) || ConfigUtils.COMPONENT_TEZAM.equals(component)) { + // Component key → workload name mapping: + // "llap-{name}" → "{cluster}-{name}" + // "tezam-{name}" → "{cluster}-tezam-{name}" + // other → "{cluster}-{component}" + String workloadName; + if (component.startsWith(ConfigUtils.COMPONENT_LLAP + "-")) { + String llapName = component.substring(ConfigUtils.COMPONENT_LLAP.length() + 1); + workloadName = clusterName + "-" + llapName; + } else if (component.startsWith(ConfigUtils.COMPONENT_TEZAM + "-")) { + String llapName = component.substring(ConfigUtils.COMPONENT_TEZAM.length() + 1); + workloadName = clusterName + "-tezam-" + llapName; + } else { + workloadName = clusterName + "-" + component; + } + if (component.startsWith(ConfigUtils.COMPONENT_LLAP + "-") + || component.startsWith(ConfigUtils.COMPONENT_TEZAM + "-")) { var ss = client.apps().statefulSets() .inNamespace(namespace).withName(workloadName).get(); return ss != null && ss.getSpec().getReplicas() != null ? ss.getSpec().getReplicas() : 0; diff --git a/packaging/src/kubernetes/src/java/org/apache/hive/kubernetes/operator/autoscaling/LlapScalingStrategy.java b/packaging/src/kubernetes/src/java/org/apache/hive/kubernetes/operator/autoscaling/LlapScalingStrategy.java index 1be30a94b1f6..1b913492791f 100644 --- a/packaging/src/kubernetes/src/java/org/apache/hive/kubernetes/operator/autoscaling/LlapScalingStrategy.java +++ b/packaging/src/kubernetes/src/java/org/apache/hive/kubernetes/operator/autoscaling/LlapScalingStrategy.java @@ -40,14 +40,17 @@ public class LlapScalingStrategy implements ScalingStrategy { static final String METRIC_QUEUED = "hadoop_llapdaemon_executornumqueuedrequests"; static final String METRIC_CONFIGURED = "hadoop_llapdaemon_executornumexecutorsconfigured"; static final String METRIC_AVAILABLE = "hadoop_llapdaemon_executornumexecutorsavailable"; + static final String METRIC_LLAP_TARGET_PREFIX = "hs2_llap_target_sessions_"; private final HiveClusterAutoscaler orchestrator; private final HiveCluster cluster; + private final String llapName; private int lastMetric; - public LlapScalingStrategy(HiveClusterAutoscaler orchestrator, HiveCluster cluster) { + public LlapScalingStrategy(HiveClusterAutoscaler orchestrator, HiveCluster cluster, String llapName) { this.orchestrator = orchestrator; this.cluster = cluster; + this.llapName = llapName; } @Override @@ -117,7 +120,12 @@ public int lastMetricValue() { } /** - * Detect HS2 open sessions. + * Detect HS2 sessions targeting this specific LLAP cluster. + * First tries the per-target metric (hs2_llap_target_sessions_{llapName}) which is + * available when HS2 has the per-LLAP-cluster session tracking patch. + * Falls back to the generic hs2_open_sessions for backward compatibility only if + * HS2 does NOT expose any per-target metrics at all (i.e. older HS2 image). + * * @return true if sessions > 0, false if scraped and all 0, null if scrape returned no pods * (ambiguous — could be transient failure or HS2 genuinely absent) */ @@ -125,6 +133,36 @@ private Boolean detectHs2Sessions(List hs2Metrics) { if (hs2Metrics.isEmpty()) { return null; } + + // Check if HS2 supports per-target metrics (any metric with the prefix exists). + // If it does, use only the per-target metric for this cluster — a missing metric + // means 0 sessions targeting this cluster (the gauge is registered lazily on first connect). + String targetMetric = METRIC_LLAP_TARGET_PREFIX + llapName; + boolean hs2SupportsTargetMetrics = false; + for (PodMetrics pm : hs2Metrics) { + for (String key : pm.metrics().keySet()) { + if (key.startsWith(METRIC_LLAP_TARGET_PREFIX)) { + hs2SupportsTargetMetrics = true; + break; + } + } + if (hs2SupportsTargetMetrics) { + break; + } + } + + if (hs2SupportsTargetMetrics) { + // HS2 has per-target tracking: check only our specific metric + for (PodMetrics pm : hs2Metrics) { + Double val = pm.metrics().get(targetMetric); + if (val != null && val > 0) { + return true; + } + } + return false; + } + + // Fallback: generic hs2_open_sessions (older HS2 without per-target metrics) for (PodMetrics pm : hs2Metrics) { double sessions = pm.metrics().getOrDefault( HiveServer2ScalingStrategy.METRIC_OPEN_SESSIONS, 0.0); diff --git a/packaging/src/kubernetes/src/java/org/apache/hive/kubernetes/operator/autoscaling/TezAmScalingStrategy.java b/packaging/src/kubernetes/src/java/org/apache/hive/kubernetes/operator/autoscaling/TezAmScalingStrategy.java index d2863102b098..bc43905a1b38 100644 --- a/packaging/src/kubernetes/src/java/org/apache/hive/kubernetes/operator/autoscaling/TezAmScalingStrategy.java +++ b/packaging/src/kubernetes/src/java/org/apache/hive/kubernetes/operator/autoscaling/TezAmScalingStrategy.java @@ -26,10 +26,12 @@ import org.slf4j.LoggerFactory; /** - * Scaling strategy for Tez Application Master. - * TezAM scaling tracks HS2 session demand: desired = ceil(sum(hs2_open_sessions)). + * Scaling strategy for per-LLAP TezAM instances. + * Each TezAM follows its paired LLAP cluster's lifecycle: it should be up + * when there are sessions targeting that LLAP cluster, and at 0 otherwise. *

- * Activation gate: only scale if HS2 has open sessions. + * Uses the per-target session metric from HS2: hs2_llap_target_sessions_{llapName}. + * Falls back to hs2_open_sessions if per-target metrics are not available. */ public class TezAmScalingStrategy implements ScalingStrategy { @@ -37,11 +39,14 @@ public class TezAmScalingStrategy implements ScalingStrategy { private final HiveClusterAutoscaler orchestrator; private final HiveCluster cluster; + private final String llapName; private int lastMetric; - public TezAmScalingStrategy(HiveClusterAutoscaler orchestrator, HiveCluster cluster) { + public TezAmScalingStrategy(HiveClusterAutoscaler orchestrator, + HiveCluster cluster, String llapName) { this.orchestrator = orchestrator; this.cluster = cluster; + this.llapName = llapName; } @Override @@ -53,30 +58,50 @@ public int computeDesiredReplicas(List podMetrics, // Activation gate: if HS2 scrape returns no data but TezAM has running pods, // treat as "unknown" and preserve current state to avoid spurious scale-to-zero. if (hs2Metrics.isEmpty() && !podMetrics.isEmpty()) { - LOG.debug("[tezam] HS2 scrape returned no pods; preserving TezAM (has {} running pods)", podMetrics.size()); + LOG.debug("[tezam-{}] HS2 scrape returned no pods; preserving TezAM", llapName); lastMetric = 0; return Math.max(1, autoscaling.minReplicas()); } - double totalSessions = 0; + // Use per-LLAP target sessions metric (same logic as LlapScalingStrategy). + String targetMetric = "hs2_llap_target_sessions_" + llapName; + boolean anyPerTargetMetricExists = false; + double targetSessions = 0; + for (PodMetrics pm : hs2Metrics) { - totalSessions += pm.metrics().getOrDefault( - HiveServer2ScalingStrategy.METRIC_OPEN_SESSIONS, 0.0); + // Check if HS2 exposes ANY per-target metric (feature support check) + for (String key : pm.metrics().keySet()) { + if (key.startsWith("hs2_llap_target_sessions_")) { + anyPerTargetMetricExists = true; + break; + } + } + targetSessions += pm.metrics().getOrDefault(targetMetric, 0.0); + } + + if (!anyPerTargetMetricExists && !hs2Metrics.isEmpty()) { + // HS2 doesn't support per-target metrics — fall back to total sessions + double totalSessions = 0; + for (PodMetrics pm : hs2Metrics) { + totalSessions += pm.metrics().getOrDefault( + HiveServer2ScalingStrategy.METRIC_OPEN_SESSIONS, 0.0); + } + targetSessions = totalSessions; } - if (totalSessions <= 0) { - LOG.debug("[tezam] No HS2 sessions, scaling to minReplicas"); + if (targetSessions <= 0) { + LOG.debug("[tezam-{}] No sessions targeting this cluster, scaling to minReplicas", llapName); lastMetric = 0; return autoscaling.minReplicas(); } - lastMetric = (int) totalSessions; + lastMetric = (int) targetSessions; - // Scale based on concurrent demand — one TezAM per open HS2 session - int desired = (int) Math.ceil(totalSessions); + // TezAM desired: at least 1 when there are sessions, capped at maxReplicas + int desired = (int) Math.ceil(targetSessions); desired = Math.min(desired, maxReplicas); - LOG.debug("[tezam] totalSessions={}, desired={}", totalSessions, desired); + LOG.debug("[tezam-{}] targetSessions={}, desired={}", llapName, targetSessions, desired); return Math.max(desired, autoscaling.minReplicas()); } diff --git a/packaging/src/kubernetes/src/java/org/apache/hive/kubernetes/operator/dependent/HiveConfigMapDependent.java b/packaging/src/kubernetes/src/java/org/apache/hive/kubernetes/operator/dependent/HiveConfigMapDependent.java index 2ca7f87232dd..411e31cb5a0d 100644 --- a/packaging/src/kubernetes/src/java/org/apache/hive/kubernetes/operator/dependent/HiveConfigMapDependent.java +++ b/packaging/src/kubernetes/src/java/org/apache/hive/kubernetes/operator/dependent/HiveConfigMapDependent.java @@ -131,24 +131,4 @@ public static String resourceName(HiveCluster hiveCluster) { } } - /** - * LLAP llap-daemon-site.xml ConfigMap. - */ - @KubernetesDependent(informer = @Informer(labelSelector = "app.kubernetes.io/component=llap," - + "app.kubernetes.io/managed-by=hive-kubernetes-operator")) - public static class Llap extends HiveConfigMapDependent { - public Llap() { - super(ConfigUtils.COMPONENT_LLAP, "llap-config"); - } - - @Override - protected void addData(ConfigMapBuilder builder, HiveCluster hiveCluster) { - builder.addToData("llap-daemon-site.xml", - HadoopXmlBuilder.buildXml(HiveConfigBuilder.getLlapDaemonSite(hiveCluster.getSpec()))); - } - - public static String resourceName(HiveCluster hiveCluster) { - return hiveCluster.getMetadata().getName() + "-llap-config"; - } - } } diff --git a/packaging/src/kubernetes/src/java/org/apache/hive/kubernetes/operator/dependent/HivePdbDependent.java b/packaging/src/kubernetes/src/java/org/apache/hive/kubernetes/operator/dependent/HivePdbDependent.java index 5a3e6cdc493d..2deac1aa7740 100644 --- a/packaging/src/kubernetes/src/java/org/apache/hive/kubernetes/operator/dependent/HivePdbDependent.java +++ b/packaging/src/kubernetes/src/java/org/apache/hive/kubernetes/operator/dependent/HivePdbDependent.java @@ -29,12 +29,13 @@ import org.apache.hive.kubernetes.operator.util.Labels; /** - * Unified PodDisruptionBudget dependent resource for all Hive components. + * PodDisruptionBudget dependent resource for workflow-managed Hive components. * Uses maxUnavailable=1 to allow at most one pod to be disrupted at a time * while still permitting node drains when replicas=1. *

- * Subclassed per component (HS2, Metastore, LLAP, TezAM) only to satisfy - * JOSDK's requirement for distinct no-arg-constructible classes in the workflow. + * Subclassed per component (HS2, Metastore) to satisfy JOSDK's requirement + * for distinct no-arg-constructible classes in the workflow. + * LLAP and TezAM PDBs are managed imperatively via {@link LlapResourceBuilder}. */ public abstract class HivePdbDependent extends HiveDependentResource { @@ -90,23 +91,4 @@ public Metastore() { } } - @KubernetesDependent( - informer = @Informer(labelSelector = "app.kubernetes.io/component=llap," - + "app.kubernetes.io/managed-by=hive-kubernetes-operator") - ) - public static class Llap extends HivePdbDependent { - public Llap() { - super(ConfigUtils.COMPONENT_LLAP); - } - } - - @KubernetesDependent( - informer = @Informer(labelSelector = "app.kubernetes.io/component=tezam," - + "app.kubernetes.io/managed-by=hive-kubernetes-operator") - ) - public static class TezAm extends HivePdbDependent { - public TezAm() { - super(ConfigUtils.COMPONENT_TEZAM); - } - } } diff --git a/packaging/src/kubernetes/src/java/org/apache/hive/kubernetes/operator/dependent/HiveServer2DeploymentDependent.java b/packaging/src/kubernetes/src/java/org/apache/hive/kubernetes/operator/dependent/HiveServer2DeploymentDependent.java index a7809e1ef0ff..6be5e1f3b9f0 100644 --- a/packaging/src/kubernetes/src/java/org/apache/hive/kubernetes/operator/dependent/HiveServer2DeploymentDependent.java +++ b/packaging/src/kubernetes/src/java/org/apache/hive/kubernetes/operator/dependent/HiveServer2DeploymentDependent.java @@ -97,10 +97,11 @@ protected Deployment desired(HiveCluster hiveCluster, spec.zookeeper().quorum(), null)); } - if (spec.llap().isEnabled()) { - envVars.add(new EnvVar("HIVE_LLAP_DAEMON_SERVICE_HOSTS", - spec.llap().serviceHosts(), null)); - } + spec.llapClusters().stream() + .filter(l -> l.isEnabled()) + .findFirst() + .ifPresent(llap -> envVars.add(new EnvVar("HIVE_LLAP_DAEMON_SERVICE_HOSTS", + llap.serviceHosts(), null))); int metastorePort = ConfigUtils.getInt( spec.metastore().configOverrides(), @@ -117,14 +118,17 @@ protected Deployment desired(HiveCluster hiveCluster, .append(ConfigUtils.HIVE_METASTORE_URIS_KEY) .append("=").append(metastoreUri); } - if (spec.llap().isEnabled()) { - serviceOpts.append(" -D") - .append(ConfigUtils.HIVE_EXECUTION_MODE_KEY) - .append("=llap"); - serviceOpts.append(" -D") - .append(ConfigUtils.HIVE_LLAP_DAEMON_SERVICE_HOSTS_KEY) - .append("=").append(spec.llap().serviceHosts()); - } + spec.llapClusters().stream() + .filter(l -> l.isEnabled()) + .findFirst() + .ifPresent(llap -> { + serviceOpts.append(" -D") + .append(ConfigUtils.HIVE_EXECUTION_MODE_KEY) + .append("=llap"); + serviceOpts.append(" -D") + .append(ConfigUtils.HIVE_LLAP_DAEMON_SERVICE_HOSTS_KEY) + .append("=").append(llap.serviceHosts()); + }); if (spec.tezAm().isEnabled()) { serviceOpts.append(" -D") .append(ConfigUtils.HIVE_ZOOKEEPER_QUORUM_KEY) @@ -185,8 +189,7 @@ protected Deployment desired(HiveCluster hiveCluster, volumes.add(new io.fabric8.kubernetes.api.model.VolumeBuilder() .withName("scratch") .withNewPersistentVolumeClaim() - .withClaimName(ScratchPvcDependent - .resourceName(hiveCluster)) + .withClaimName(ScratchPvcDependent.resourceName(hiveCluster)) .endPersistentVolumeClaim() .build()); } diff --git a/packaging/src/kubernetes/src/java/org/apache/hive/kubernetes/operator/dependent/HiveServiceDependent.java b/packaging/src/kubernetes/src/java/org/apache/hive/kubernetes/operator/dependent/HiveServiceDependent.java index e367bf4c6028..50d4dd8201dc 100644 --- a/packaging/src/kubernetes/src/java/org/apache/hive/kubernetes/operator/dependent/HiveServiceDependent.java +++ b/packaging/src/kubernetes/src/java/org/apache/hive/kubernetes/operator/dependent/HiveServiceDependent.java @@ -132,55 +132,4 @@ protected void customizeSpec(ServiceBuilder builder, HiveCluster hiveCluster) { } } - /** LLAP headless Service: required by StatefulSet for stable DNS. */ - @KubernetesDependent( - informer = @Informer(labelSelector = "app.kubernetes.io/component=llap," - + "app.kubernetes.io/managed-by=hive-kubernetes-operator") - ) - public static class Llap extends HiveServiceDependent { - public Llap() { - super(ConfigUtils.COMPONENT_LLAP); - } - - @Override - protected void customizeSpec(ServiceBuilder builder, HiveCluster hiveCluster) { - var overrides = hiveCluster.getSpec().llap().configOverrides(); - int managementPort = ConfigUtils.getInt(overrides, - ConfigUtils.HIVE_LLAP_MANAGEMENT_RPC_PORT_KEY, null, - ConfigUtils.HIVE_LLAP_MANAGEMENT_RPC_PORT_DEFAULT); - int shufflePort = ConfigUtils.getInt(overrides, - ConfigUtils.HIVE_LLAP_DAEMON_SHUFFLE_PORT_KEY, null, - ConfigUtils.HIVE_LLAP_DAEMON_SHUFFLE_PORT_DEFAULT); - int webPort = ConfigUtils.getInt(overrides, - ConfigUtils.HIVE_LLAP_DAEMON_WEB_PORT_KEY, null, - ConfigUtils.HIVE_LLAP_DAEMON_WEB_PORT_DEFAULT); - builder.editSpec() - .withClusterIP("None") - .addNewPort().withName("management").withProtocol("TCP") - .withPort(managementPort).withTargetPort(new IntOrString(managementPort)).endPort() - .addNewPort().withName("shuffle").withProtocol("TCP") - .withPort(shufflePort).withTargetPort(new IntOrString(shufflePort)).endPort() - .addNewPort().withName("web").withProtocol("TCP") - .withPort(webPort).withTargetPort(new IntOrString(webPort)).endPort() - .endSpec(); - } - } - - /** TezAM headless Service: required by StatefulSet for stable DNS. */ - @KubernetesDependent( - informer = @Informer(labelSelector = "app.kubernetes.io/component=tezam," - + "app.kubernetes.io/managed-by=hive-kubernetes-operator") - ) - public static class TezAm extends HiveServiceDependent { - public TezAm() { - super(ConfigUtils.COMPONENT_TEZAM); - } - - @Override - protected void customizeSpec(ServiceBuilder builder, HiveCluster hiveCluster) { - builder.editSpec() - .withClusterIP("None") - .endSpec(); - } - } } diff --git a/packaging/src/kubernetes/src/java/org/apache/hive/kubernetes/operator/dependent/LlapResourceBuilder.java b/packaging/src/kubernetes/src/java/org/apache/hive/kubernetes/operator/dependent/LlapResourceBuilder.java new file mode 100644 index 000000000000..b51509a02d03 --- /dev/null +++ b/packaging/src/kubernetes/src/java/org/apache/hive/kubernetes/operator/dependent/LlapResourceBuilder.java @@ -0,0 +1,520 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hive.kubernetes.operator.dependent; + +import java.util.ArrayList; +import java.util.List; +import java.util.Map; + +import io.fabric8.kubernetes.api.model.ConfigMap; +import io.fabric8.kubernetes.api.model.ConfigMapBuilder; +import io.fabric8.kubernetes.api.model.Container; +import io.fabric8.kubernetes.api.model.ContainerPort; +import io.fabric8.kubernetes.api.model.ContainerPortBuilder; +import io.fabric8.kubernetes.api.model.EnvVar; +import io.fabric8.kubernetes.api.model.OwnerReference; +import io.fabric8.kubernetes.api.model.OwnerReferenceBuilder; +import io.fabric8.kubernetes.api.model.IntOrString; +import io.fabric8.kubernetes.api.model.Probe; +import io.fabric8.kubernetes.api.model.Service; +import io.fabric8.kubernetes.api.model.ServiceBuilder; +import io.fabric8.kubernetes.api.model.Volume; +import io.fabric8.kubernetes.api.model.VolumeMount; +import io.fabric8.kubernetes.api.model.apps.StatefulSet; +import io.fabric8.kubernetes.api.model.apps.StatefulSetBuilder; +import io.fabric8.kubernetes.api.model.policy.v1.PodDisruptionBudget; +import io.fabric8.kubernetes.api.model.policy.v1.PodDisruptionBudgetBuilder; +import io.javaoperatorsdk.operator.api.reconciler.Context; +import org.apache.hive.kubernetes.operator.model.HiveCluster; +import org.apache.hive.kubernetes.operator.model.HiveClusterSpec; +import org.apache.hive.kubernetes.operator.model.spec.AutoscalingSpec; +import org.apache.hive.kubernetes.operator.model.spec.LlapSpec; +import org.apache.hive.kubernetes.operator.util.ConfigUtils; +import org.apache.hive.kubernetes.operator.util.HadoopXmlBuilder; +import org.apache.hive.kubernetes.operator.util.HiveConfigBuilder; +import org.apache.hive.kubernetes.operator.util.Labels; + +/** + * Static builder methods for LLAP Kubernetes resources. + * Used by the reconciler to imperatively manage multiple LLAP clusters. + *

+ * Extends {@link HiveDependentResource} solely to access protected helper methods + * (buildTcpProbe, addExternalJars, addJmxExporter, etc.). + */ +public class LlapResourceBuilder + extends HiveDependentResource { + + private static final LlapResourceBuilder INSTANCE = new LlapResourceBuilder(); + + LlapResourceBuilder() { + super(StatefulSet.class); + } + + @Override + protected String getSecondaryResourceName(HiveCluster primary, + Context context) { + throw new UnsupportedOperationException("LlapResourceBuilder is not a managed dependent"); + } + + @Override + protected StatefulSet desired(HiveCluster hiveCluster, + Context context) { + throw new UnsupportedOperationException("LlapResourceBuilder is not a managed dependent"); + } + + /** Creates an OwnerReference pointing to the HiveCluster CR for garbage collection. */ + private static OwnerReference ownerRef(HiveCluster hc) { + return new OwnerReferenceBuilder() + .withApiVersion(hc.getApiVersion()) + .withKind(hc.getKind()) + .withName(hc.getMetadata().getName()) + .withUid(hc.getMetadata().getUid()) + .withController(true) + .withBlockOwnerDeletion(true) + .build(); + } + + /** Resource name for a specific LLAP cluster: {clusterName}-{llapName}. */ + public static String resourceName(HiveCluster hc, LlapSpec llap) { + return hc.getMetadata().getName() + "-" + llap.name(); + } + + /** ConfigMap name for a specific LLAP cluster. */ + public static String configMapName(HiveCluster hc, LlapSpec llap) { + return hc.getMetadata().getName() + "-" + llap.name() + "-config"; + } + + /** PDB name for a specific LLAP cluster. */ + public static String pdbName(HiveCluster hc, LlapSpec llap) { + return hc.getMetadata().getName() + "-" + llap.name() + "-pdb"; + } + + /** Builds the StatefulSet for a specific LLAP cluster. */ + public static StatefulSet buildStatefulSet(HiveCluster hc, LlapSpec llap, Integer replicas) { + return INSTANCE.doBuildStatefulSet(hc, llap, replicas); + } + + /** Builds the headless Service for a specific LLAP cluster. */ + public static Service buildService(HiveCluster hc, LlapSpec llap) { + String ns = hc.getMetadata().getNamespace(); + String name = resourceName(hc, llap); + Map labels = Labels.forLlapCluster(hc, llap.name()); + Map selector = Labels.selectorForLlapCluster(hc, llap.name()); + + int managementPort = ConfigUtils.getInt(llap.configOverrides(), + ConfigUtils.HIVE_LLAP_MANAGEMENT_RPC_PORT_KEY, null, + ConfigUtils.HIVE_LLAP_MANAGEMENT_RPC_PORT_DEFAULT); + int shufflePort = ConfigUtils.getInt(llap.configOverrides(), + ConfigUtils.HIVE_LLAP_DAEMON_SHUFFLE_PORT_KEY, null, + ConfigUtils.HIVE_LLAP_DAEMON_SHUFFLE_PORT_DEFAULT); + int webPort = ConfigUtils.getInt(llap.configOverrides(), + ConfigUtils.HIVE_LLAP_DAEMON_WEB_PORT_KEY, null, + ConfigUtils.HIVE_LLAP_DAEMON_WEB_PORT_DEFAULT); + + return new ServiceBuilder() + .withNewMetadata() + .withName(name) + .withNamespace(ns) + .withLabels(labels) + .withOwnerReferences(ownerRef(hc)) + .endMetadata() + .withNewSpec() + .withClusterIP("None") + .withSelector(selector) + .addNewPort().withName("management").withProtocol("TCP") + .withPort(managementPort) + .withTargetPort(new IntOrString(managementPort)).endPort() + .addNewPort().withName("shuffle").withProtocol("TCP") + .withPort(shufflePort) + .withTargetPort(new IntOrString(shufflePort)).endPort() + .addNewPort().withName("web").withProtocol("TCP") + .withPort(webPort) + .withTargetPort(new IntOrString(webPort)).endPort() + .endSpec() + .build(); + } + + /** Builds the ConfigMap for a specific LLAP cluster. */ + public static ConfigMap buildConfigMap(HiveCluster hc, LlapSpec llap) { + HiveClusterSpec spec = hc.getSpec(); + Map labels = Labels.forLlapCluster(hc, llap.name()); + Map llapDaemonSite = HiveConfigBuilder.getLlapDaemonSite(spec, llap); + + return new ConfigMapBuilder() + .withNewMetadata() + .withName(configMapName(hc, llap)) + .withNamespace(hc.getMetadata().getNamespace()) + .withLabels(labels) + .withOwnerReferences(ownerRef(hc)) + .endMetadata() + .addToData("llap-daemon-site.xml", HadoopXmlBuilder.buildXml(llapDaemonSite)) + .build(); + } + + /** Builds the PodDisruptionBudget for a specific LLAP cluster. */ + public static PodDisruptionBudget buildPdb(HiveCluster hc, LlapSpec llap) { + Map labels = Labels.forLlapCluster(hc, llap.name()); + Map selector = Labels.selectorForLlapCluster(hc, llap.name()); + + return new PodDisruptionBudgetBuilder() + .withNewMetadata() + .withName(pdbName(hc, llap)) + .withNamespace(hc.getMetadata().getNamespace()) + .withLabels(labels) + .withOwnerReferences(ownerRef(hc)) + .endMetadata() + .withNewSpec() + .withMaxUnavailable(new IntOrString(1)) + .withNewSelector() + .withMatchLabels(selector) + .endSelector() + .endSpec() + .build(); + } + + // --- TezAM resource builders (one TezAM per LLAP cluster) --- + + /** TezAM StatefulSet name for a specific LLAP cluster. */ + public static String tezAmResourceName(HiveCluster hc, LlapSpec llap) { + return hc.getMetadata().getName() + "-tezam-" + llap.name(); + } + + /** TezAM ConfigMap name for a specific LLAP cluster. */ + public static String tezAmConfigMapName(HiveCluster hc, LlapSpec llap) { + return hc.getMetadata().getName() + "-tezam-" + llap.name() + "-config"; + } + + /** TezAM PDB name for a specific LLAP cluster. */ + public static String tezAmPdbName(HiveCluster hc, LlapSpec llap) { + return hc.getMetadata().getName() + "-tezam-" + llap.name() + "-pdb"; + } + + /** Builds the PodDisruptionBudget for a per-LLAP-cluster TezAM. */ + public static PodDisruptionBudget buildTezAmPdb(HiveCluster hc, LlapSpec llap) { + Map labels = Labels.forTezAmCluster(hc, llap.name()); + Map selector = Labels.selectorForTezAmCluster(hc, llap.name()); + + return new PodDisruptionBudgetBuilder() + .withNewMetadata() + .withName(tezAmPdbName(hc, llap)) + .withNamespace(hc.getMetadata().getNamespace()) + .withLabels(labels) + .withOwnerReferences(ownerRef(hc)) + .endMetadata() + .withNewSpec() + .withMaxUnavailable(new IntOrString(1)) + .withNewSelector() + .withMatchLabels(selector) + .endSelector() + .endSpec() + .build(); + } + + /** Builds the TezAM StatefulSet for a specific LLAP cluster. */ + public static StatefulSet buildTezAmStatefulSet(HiveCluster hc, LlapSpec llap, Integer replicas) { + return INSTANCE.doBuildTezAmStatefulSet(hc, llap, replicas); + } + + /** Builds the headless Service for a TezAM cluster. */ + public static Service buildTezAmService(HiveCluster hc, LlapSpec llap) { + String ns = hc.getMetadata().getNamespace(); + String name = tezAmResourceName(hc, llap); + Map labels = Labels.forTezAmCluster(hc, llap.name()); + Map selector = Labels.selectorForTezAmCluster(hc, llap.name()); + + return new ServiceBuilder() + .withNewMetadata() + .withName(name) + .withNamespace(ns) + .withLabels(labels) + .withOwnerReferences(ownerRef(hc)) + .endMetadata() + .withNewSpec() + .withClusterIP("None") + .withSelector(selector) + .endSpec() + .build(); + } + + /** Builds the ConfigMap for a per-LLAP-cluster TezAM (tez-site.xml). */ + public static ConfigMap buildTezAmConfigMap(HiveCluster hc, LlapSpec llap) { + HiveClusterSpec spec = hc.getSpec(); + Map labels = Labels.forTezAmCluster(hc, llap.name()); + Map tezSite = HiveConfigBuilder.getTezSite(spec, llap); + + return new ConfigMapBuilder() + .withNewMetadata() + .withName(tezAmConfigMapName(hc, llap)) + .withNamespace(hc.getMetadata().getNamespace()) + .withLabels(labels) + .withOwnerReferences(ownerRef(hc)) + .endMetadata() + .addToData("tez-site.xml", HadoopXmlBuilder.buildXml(tezSite)) + .build(); + } + + // --- Private instance methods that use protected helpers from HiveDependentResource --- + + private StatefulSet doBuildTezAmStatefulSet(HiveCluster hiveCluster, LlapSpec llap, + Integer replicas) { + HiveClusterSpec spec = hiveCluster.getSpec(); + String ns = hiveCluster.getMetadata().getNamespace(); + String ssName = tezAmResourceName(hiveCluster, llap); + Map allLabels = Labels.forTezAmCluster(hiveCluster, llap.name()); + Map selectorLabels = Labels.selectorForTezAmCluster(hiveCluster, llap.name()); + + List envVars = new ArrayList<>(); + envVars.add(new EnvVar("SERVICE_NAME", ConfigUtils.COMPONENT_TEZAM, null)); + envVars.add(new EnvVar("IS_RESUME", "true", null)); + envVars.add(new EnvVar("HIVE_ZOOKEEPER_QUORUM", + spec.zookeeper().quorum(), null)); + envVars.add(new EnvVar("TEZ_FRAMEWORK_MODE", "STANDALONE_ZOOKEEPER", null)); + envVars.add(new EnvVar("HIVE_LLAP_DAEMON_SERVICE_HOSTS", + llap.serviceHosts(), null)); + + if (spec.envVars() != null) { + envVars.addAll(spec.envVars()); + } + + List volumeMounts = new ArrayList<>(); + volumeMounts.add(new io.fabric8.kubernetes.api.model.VolumeMountBuilder() + .withName("hive-config") + .withMountPath(CONF_MOUNT_PATH).build()); + volumeMounts.add(new io.fabric8.kubernetes.api.model.VolumeMountBuilder() + .withName("scratch") + .withMountPath("/opt/hive/scratch").build()); + + List volumes = new ArrayList<>(); + // Projected volume: hive-site.xml from HS2 CM, tez-site.xml from per-LLAP CM, core-site.xml from Hadoop CM + String hs2CmName = HiveConfigMapDependent.HiveServer2.resourceName(hiveCluster); + String hadoopCmName = HiveConfigMapDependent.Hadoop.resourceName(hiveCluster); + String tezAmCmName = tezAmConfigMapName(hiveCluster, llap); + volumes.add(buildProjectedConfigVolume("hive-config", hs2CmName, tezAmCmName, hadoopCmName)); + volumes.add(new io.fabric8.kubernetes.api.model.VolumeBuilder() + .withName("scratch") + .withNewPersistentVolumeClaim() + .withClaimName(ScratchPvcDependent.resourceName(hiveCluster)) + .endPersistentVolumeClaim() + .build()); + + List ports = new ArrayList<>(); + List initContainers = new ArrayList<>(); + addExternalJars(spec.image(), spec.externalJars(), + initContainers, volumeMounts, volumes, envVars); + replaceConfMountWithSubPaths(volumeMounts, "hive-config", + "hive-site.xml", "tez-site.xml", "core-site.xml"); + + String configHash = sha256( + HadoopXmlBuilder.buildXml(HiveConfigBuilder.getHiveServer2HiveSite(hiveCluster, spec)), + HadoopXmlBuilder.buildXml(HiveConfigBuilder.getTezSite(spec, llap)), + HadoopXmlBuilder.buildXml(HiveConfigBuilder.getHadoopCoreSite(spec))); + + StatefulSet statefulSet = new StatefulSetBuilder() + .withNewMetadata() + .withName(ssName) + .withNamespace(ns) + .withLabels(allLabels) + .withOwnerReferences(ownerRef(hiveCluster)) + .endMetadata() + .withNewSpec() + .withReplicas(replicas) + .withPodManagementPolicy("Parallel") + .withServiceName(ssName) + .withNewSelector() + .withMatchLabels(selectorLabels) + .endSelector() + .withNewTemplate() + .withNewMetadata() + .withLabels(allLabels) + .addToAnnotations("kubectl.kubernetes.io/default-container", + ConfigUtils.COMPONENT_TEZAM) + .addToAnnotations("hive.apache.org/config-hash", configHash) + .endMetadata() + .withNewSpec() + .withInitContainers(initContainers) + .addNewContainer() + .withName(ConfigUtils.COMPONENT_TEZAM) + .withImage(spec.image()) + .withImagePullPolicy(spec.imagePullPolicy()) + .withEnv(envVars) + .withPorts(ports) + .withResources(buildResources(spec.tezAm().resources())) + .withVolumeMounts(volumeMounts) + .endContainer() + .withVolumes(volumes) + .endSpec() + .endTemplate() + .endSpec() + .build(); + + applySpreadAffinityIfAbsent( + statefulSet.getSpec().getTemplate().getSpec(), selectorLabels); + + appendUserVolumes(statefulSet.getSpec().getTemplate().getSpec(), + spec.volumes(), spec.volumeMounts(), + spec.tezAm().extraVolumes(), spec.tezAm().extraVolumeMounts()); + + return statefulSet; + } + + private StatefulSet doBuildStatefulSet(HiveCluster hiveCluster, LlapSpec llap, Integer replicas) { + HiveClusterSpec spec = hiveCluster.getSpec(); + String ns = hiveCluster.getMetadata().getNamespace(); + String ssName = resourceName(hiveCluster, llap); + Map allLabels = Labels.forLlapCluster(hiveCluster, llap.name()); + Map selectorLabels = Labels.selectorForLlapCluster(hiveCluster, llap.name()); + + List envVars = new ArrayList<>(); + envVars.add(new EnvVar("SERVICE_NAME", "llap", null)); + envVars.add(new EnvVar("IS_RESUME", "true", null)); + envVars.add(new EnvVar("LLAP_MEMORY_MB", + String.valueOf(llap.memoryMb()), null)); + envVars.add(new EnvVar("LLAP_EXECUTORS", + String.valueOf(llap.executors()), null)); + envVars.add(new EnvVar("HIVE_ZOOKEEPER_QUORUM", + spec.zookeeper().quorum(), null)); + envVars.add(new EnvVar("HIVE_LLAP_DAEMON_SERVICE_HOSTS", + llap.serviceHosts(), null)); + envVars.add(new EnvVar("LLAP_LOG4J2_PROPERTIES_FILE_NAME", + "llap-daemon-log4j2.properties", null)); + + if (spec.envVars() != null) { + envVars.addAll(spec.envVars()); + } + + int managementPort = ConfigUtils.getInt(llap.configOverrides(), + ConfigUtils.HIVE_LLAP_MANAGEMENT_RPC_PORT_KEY, null, + ConfigUtils.HIVE_LLAP_MANAGEMENT_RPC_PORT_DEFAULT); + int shufflePort = ConfigUtils.getInt(llap.configOverrides(), + ConfigUtils.HIVE_LLAP_DAEMON_SHUFFLE_PORT_KEY, null, + ConfigUtils.HIVE_LLAP_DAEMON_SHUFFLE_PORT_DEFAULT); + int webPort = ConfigUtils.getInt(llap.configOverrides(), + ConfigUtils.HIVE_LLAP_DAEMON_WEB_PORT_KEY, null, + ConfigUtils.HIVE_LLAP_DAEMON_WEB_PORT_DEFAULT); + int outputPort = ConfigUtils.getInt(llap.configOverrides(), + ConfigUtils.HIVE_LLAP_DAEMON_OUTPUT_SERVICE_PORT_KEY, null, + ConfigUtils.HIVE_LLAP_DAEMON_OUTPUT_SERVICE_PORT_DEFAULT); + + List ports = new ArrayList<>(); + ports.add(new ContainerPortBuilder() + .withName("management").withContainerPort(managementPort) + .withProtocol("TCP").build()); + ports.add(new ContainerPortBuilder() + .withName("shuffle").withContainerPort(shufflePort) + .withProtocol("TCP").build()); + ports.add(new ContainerPortBuilder() + .withName("web").withContainerPort(webPort) + .withProtocol("TCP").build()); + ports.add(new ContainerPortBuilder() + .withName("output").withContainerPort(outputPort) + .withProtocol("TCP").build()); + + Probe readinessProbe = buildTcpProbe(managementPort, llap.readinessProbe(), 15, 10, 3); + + List volumeMounts = new ArrayList<>(); + volumeMounts.add(new io.fabric8.kubernetes.api.model.VolumeMountBuilder() + .withName("llap-config") + .withMountPath(CONF_MOUNT_PATH).build()); + + List volumes = new ArrayList<>(); + String cmName = configMapName(hiveCluster, llap); + String hadoopCmName = HiveConfigMapDependent.Hadoop.resourceName(hiveCluster); + volumes.add(buildProjectedConfigVolume("llap-config", cmName, hadoopCmName)); + + List initContainers = new ArrayList<>(); + addExternalJars(spec.image(), spec.externalJars(), + initContainers, volumeMounts, volumes, envVars); + replaceConfMountWithSubPaths(volumeMounts, "llap-config", + "llap-daemon-site.xml", "core-site.xml"); + + AutoscalingSpec autoscaling = llap.autoscaling(); + if (autoscaling.isEnabled()) { + addJmxExporter(spec.image(), ConfigUtils.COMPONENT_LLAP, autoscaling.metricsPort(), + initContainers, volumeMounts, volumes, envVars, ports); + } + + String configHash = sha256( + HadoopXmlBuilder.buildXml(HiveConfigBuilder.getLlapDaemonSite(spec, llap)), + HadoopXmlBuilder.buildXml(HiveConfigBuilder.getHadoopCoreSite(spec))); + + StatefulSet statefulSet = new StatefulSetBuilder() + .withNewMetadata() + .withName(ssName) + .withNamespace(ns) + .withLabels(allLabels) + .withOwnerReferences(ownerRef(hiveCluster)) + .endMetadata() + .withNewSpec() + .withReplicas(replicas) + .withPodManagementPolicy("Parallel") + .withServiceName(ssName) + .withNewSelector() + .withMatchLabels(selectorLabels) + .endSelector() + .withNewTemplate() + .withNewMetadata() + .withLabels(allLabels) + .addToAnnotations("kubectl.kubernetes.io/default-container", + ConfigUtils.COMPONENT_LLAP) + .addToAnnotations("hive.apache.org/config-hash", configHash) + .endMetadata() + .withNewSpec() + .withInitContainers(initContainers) + .addNewContainer() + .withName(ConfigUtils.COMPONENT_LLAP) + .withImage(spec.image()) + .withImagePullPolicy(spec.imagePullPolicy()) + .withEnv(envVars) + .withPorts(ports) + .withReadinessProbe(readinessProbe) + .withResources(buildResources(llap.resources())) + .withVolumeMounts(volumeMounts) + .endContainer() + .withVolumes(volumes) + .endSpec() + .endTemplate() + .endSpec() + .build(); + + applySpreadAffinityIfAbsent( + statefulSet.getSpec().getTemplate().getSpec(), selectorLabels); + + if (autoscaling.isEnabled()) { + String preStopScript = buildDualMetricDrainScript( + "Waiting for LLAP executors to become idle", + "hadoop_llapdaemon_executornumexecutorsavailable{", "AVAILABLE", + "hadoop_llapdaemon_executornumexecutors{", "TOTAL", + "LLAP executor metrics not found. JMX Exporter may not be configured.", + "All executors idle. Shutting down.", + "Executors available=$AVAILABLE / total=$TOTAL \u2014 waiting...", + 10, 6, autoscaling.metricsPort()); + applyAutoscalingLifecycle( + statefulSet.getSpec().getTemplate().getSpec(), + statefulSet.getSpec().getTemplate().getMetadata(), + preStopScript, autoscaling.gracePeriodSeconds(), + autoscaling.metricsScrapeIntervalSeconds()); + } + + appendUserVolumes(statefulSet.getSpec().getTemplate().getSpec(), + spec.volumes(), spec.volumeMounts(), + llap.extraVolumes(), llap.extraVolumeMounts()); + + return statefulSet; + } +} diff --git a/packaging/src/kubernetes/src/java/org/apache/hive/kubernetes/operator/dependent/LlapStatefulSetDependent.java b/packaging/src/kubernetes/src/java/org/apache/hive/kubernetes/operator/dependent/LlapStatefulSetDependent.java deleted file mode 100644 index 9862d37d80e2..000000000000 --- a/packaging/src/kubernetes/src/java/org/apache/hive/kubernetes/operator/dependent/LlapStatefulSetDependent.java +++ /dev/null @@ -1,231 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hive.kubernetes.operator.dependent; - -import java.util.ArrayList; -import java.util.List; -import java.util.Map; - -import io.fabric8.kubernetes.api.model.Container; -import io.fabric8.kubernetes.api.model.ContainerPort; -import io.fabric8.kubernetes.api.model.ContainerPortBuilder; -import io.fabric8.kubernetes.api.model.EnvVar; -import io.fabric8.kubernetes.api.model.Probe; -import io.fabric8.kubernetes.api.model.apps.StatefulSet; -import io.fabric8.kubernetes.api.model.apps.StatefulSetBuilder; -import io.javaoperatorsdk.operator.api.reconciler.Context; -import io.javaoperatorsdk.operator.api.config.informer.Informer; -import io.javaoperatorsdk.operator.processing.dependent.kubernetes.KubernetesDependent; -import org.apache.hive.kubernetes.operator.model.HiveCluster; -import org.apache.hive.kubernetes.operator.model.HiveClusterSpec; -import org.apache.hive.kubernetes.operator.model.spec.AutoscalingSpec; -import org.apache.hive.kubernetes.operator.model.spec.LlapSpec; -import org.apache.hive.kubernetes.operator.util.ConfigUtils; -import org.apache.hive.kubernetes.operator.util.HadoopXmlBuilder; -import org.apache.hive.kubernetes.operator.util.HiveConfigBuilder; -import org.apache.hive.kubernetes.operator.util.Labels; - -/** - * Manages the Kubernetes StatefulSet for LLAP daemons. - * Uses StatefulSet for stable pod identities required by ZooKeeper registration. - */ -@KubernetesDependent( - informer = @Informer(labelSelector = "app.kubernetes.io/component=llap," - + "app.kubernetes.io/managed-by=hive-kubernetes-operator") -) -public class LlapStatefulSetDependent - extends HiveDependentResource { - - public static final String COMPONENT = ConfigUtils.COMPONENT_LLAP; - - public LlapStatefulSetDependent() { - super(StatefulSet.class); - } - - @Override - protected String getSecondaryResourceName(HiveCluster primary, - Context context) { - return resourceName(primary); - } - - @Override - protected String getComponentName() { - return COMPONENT; - } - - @Override - protected StatefulSet desired(HiveCluster hiveCluster, - Context context) { - HiveClusterSpec spec = hiveCluster.getSpec(); - LlapSpec llap = spec.llap(); - Map selectorLabels = - Labels.selectorForComponent(hiveCluster, COMPONENT); - - List envVars = new ArrayList<>(); - envVars.add(new EnvVar("SERVICE_NAME", COMPONENT, null)); - envVars.add(new EnvVar("IS_RESUME", "true", null)); - envVars.add(new EnvVar("LLAP_MEMORY_MB", - String.valueOf(llap.memoryMb()), null)); - envVars.add(new EnvVar("LLAP_EXECUTORS", - String.valueOf(llap.executors()), null)); - envVars.add(new EnvVar("HIVE_ZOOKEEPER_QUORUM", - spec.zookeeper().quorum(), null)); - envVars.add(new EnvVar("HIVE_LLAP_DAEMON_SERVICE_HOSTS", - llap.serviceHosts(), null)); - envVars.add(new EnvVar("LLAP_LOG4J2_PROPERTIES_FILE_NAME", - "llap-daemon-log4j2.properties", null)); - - // User-provided env vars (storage credentials, etc.) - if (spec.envVars() != null) { - envVars.addAll(spec.envVars()); - } - - int managementPort = ConfigUtils.getInt(llap.configOverrides(), - ConfigUtils.HIVE_LLAP_MANAGEMENT_RPC_PORT_KEY, null, - ConfigUtils.HIVE_LLAP_MANAGEMENT_RPC_PORT_DEFAULT); - int shufflePort = ConfigUtils.getInt(llap.configOverrides(), - ConfigUtils.HIVE_LLAP_DAEMON_SHUFFLE_PORT_KEY, null, - ConfigUtils.HIVE_LLAP_DAEMON_SHUFFLE_PORT_DEFAULT); - int webPort = ConfigUtils.getInt(llap.configOverrides(), - ConfigUtils.HIVE_LLAP_DAEMON_WEB_PORT_KEY, null, - ConfigUtils.HIVE_LLAP_DAEMON_WEB_PORT_DEFAULT); - int outputPort = ConfigUtils.getInt(llap.configOverrides(), - ConfigUtils.HIVE_LLAP_DAEMON_OUTPUT_SERVICE_PORT_KEY, null, - ConfigUtils.HIVE_LLAP_DAEMON_OUTPUT_SERVICE_PORT_DEFAULT); - - List ports = new ArrayList<>(); - ports.add(new ContainerPortBuilder() - .withName("management").withContainerPort(managementPort).withProtocol("TCP").build()); - ports.add(new ContainerPortBuilder() - .withName("shuffle").withContainerPort(shufflePort).withProtocol("TCP").build()); - ports.add(new ContainerPortBuilder() - .withName("web").withContainerPort(webPort).withProtocol("TCP").build()); - ports.add(new ContainerPortBuilder() - .withName("output").withContainerPort(outputPort).withProtocol("TCP").build()); - - Probe readinessProbe = buildTcpProbe(managementPort, llap.readinessProbe(), 15, 10, 3); - - String headlessServiceName = - hiveCluster.getMetadata().getName() + "-llap"; - - List volumeMounts = - new ArrayList<>(); - volumeMounts.add(new io.fabric8.kubernetes.api.model.VolumeMountBuilder() - .withName("llap-config") - .withMountPath(CONF_MOUNT_PATH).build()); - - List volumes = - new ArrayList<>(); - volumes.add(buildProjectedConfigVolume("llap-config", - HiveConfigMapDependent.Llap.resourceName(hiveCluster), - HiveConfigMapDependent.Hadoop.resourceName(hiveCluster))); - - List initContainers = new ArrayList<>(); - addExternalJars(spec.image(), spec.externalJars(), - initContainers, volumeMounts, volumes, envVars); - replaceConfMountWithSubPaths(volumeMounts, "llap-config", - "llap-daemon-site.xml", "core-site.xml"); - - // Add Prometheus JMX Exporter when autoscaling is enabled - AutoscalingSpec autoscaling = llap.autoscaling(); - if (autoscaling.isEnabled()) { - addJmxExporter(spec.image(), COMPONENT, autoscaling.metricsPort(), - initContainers, volumeMounts, volumes, envVars, ports); - } - - // Pre-compute config hash for the pod template annotation. - String configHash = sha256( - HadoopXmlBuilder.buildXml(HiveConfigBuilder.getLlapDaemonSite(spec)), - HadoopXmlBuilder.buildXml(HiveConfigBuilder.getHadoopCoreSite(spec))); - - AutoscalingSpec llapAutoscaling = llap.autoscaling(); - int initialReplicas = llapAutoscaling != null && llapAutoscaling.isEnabled() - ? llapAutoscaling.minReplicas() : llap.replicas(); - Integer replicas = resolveReplicaCount( - hiveCluster, context, llapAutoscaling, llap.replicas(), initialReplicas); - - StatefulSet statefulSet = new StatefulSetBuilder() - .withNewMetadata() - .withName(resourceName(hiveCluster)) - .withNamespace(hiveCluster.getMetadata().getNamespace()) - .withLabels(Labels.forComponent(hiveCluster, COMPONENT)) - .endMetadata() - .withNewSpec() - .withReplicas(replicas) - .withPodManagementPolicy("Parallel") - .withServiceName(headlessServiceName) - .withNewSelector() - .withMatchLabels(selectorLabels) - .endSelector() - .withNewTemplate() - .withNewMetadata() - .withLabels(Labels.forComponent(hiveCluster, COMPONENT)) - .addToAnnotations("kubectl.kubernetes.io/default-container", COMPONENT) - .addToAnnotations("hive.apache.org/config-hash", configHash) - .endMetadata() - .withNewSpec() - .withInitContainers(initContainers) - .addNewContainer() - .withName(COMPONENT) - .withImage(spec.image()) - .withImagePullPolicy(spec.imagePullPolicy()) - .withEnv(envVars) - .withPorts(ports) - .withReadinessProbe(readinessProbe) - .withResources(buildResources(llap.resources())) - .withVolumeMounts(volumeMounts) - .endContainer() - .withVolumes(volumes) - .endSpec() - .endTemplate() - .endSpec() - .build(); - - applySpreadAffinityIfAbsent( - statefulSet.getSpec().getTemplate().getSpec(), selectorLabels); - - // Graceful scale-down: poll JMX Exporter until all executors idle. - if (autoscaling.isEnabled()) { - String preStopScript = buildDualMetricDrainScript( - "Waiting for LLAP executors to become idle", - "hadoop_llapdaemon_executornumexecutorsavailable{", "AVAILABLE", - "hadoop_llapdaemon_executornumexecutors{", "TOTAL", - "LLAP executor metrics not found. JMX Exporter may not be configured.", - "All executors idle. Shutting down.", - "Executors available=$AVAILABLE / total=$TOTAL \u2014 waiting...", - 10, 6, autoscaling.metricsPort()); - applyAutoscalingLifecycle( - statefulSet.getSpec().getTemplate().getSpec(), - statefulSet.getSpec().getTemplate().getMetadata(), - preStopScript, autoscaling.gracePeriodSeconds(), - autoscaling.metricsScrapeIntervalSeconds()); - } - - appendUserVolumes(statefulSet.getSpec().getTemplate().getSpec(), - spec.volumes(), spec.volumeMounts(), - llap.extraVolumes(), llap.extraVolumeMounts()); - - return statefulSet; - } - - /** Returns the StatefulSet resource name for this HiveCluster. */ - public static String resourceName(HiveCluster hiveCluster) { - return hiveCluster.getMetadata().getName() + "-llap"; - } -} diff --git a/packaging/src/kubernetes/src/java/org/apache/hive/kubernetes/operator/dependent/TezAmStatefulSetDependent.java b/packaging/src/kubernetes/src/java/org/apache/hive/kubernetes/operator/dependent/TezAmStatefulSetDependent.java deleted file mode 100644 index 56aa77c05db4..000000000000 --- a/packaging/src/kubernetes/src/java/org/apache/hive/kubernetes/operator/dependent/TezAmStatefulSetDependent.java +++ /dev/null @@ -1,194 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hive.kubernetes.operator.dependent; - -import java.util.ArrayList; -import java.util.List; -import java.util.Map; - -import io.fabric8.kubernetes.api.model.Container; -import io.fabric8.kubernetes.api.model.ContainerPort; -import io.fabric8.kubernetes.api.model.EnvVar; -import io.fabric8.kubernetes.api.model.apps.StatefulSet; -import io.fabric8.kubernetes.api.model.apps.StatefulSetBuilder; -import io.javaoperatorsdk.operator.api.reconciler.Context; -import io.javaoperatorsdk.operator.api.config.informer.Informer; -import io.javaoperatorsdk.operator.processing.dependent.kubernetes.KubernetesDependent; -import org.apache.hive.kubernetes.operator.model.HiveCluster; -import org.apache.hive.kubernetes.operator.model.HiveClusterSpec; -import org.apache.hive.kubernetes.operator.model.spec.AutoscalingSpec; -import org.apache.hive.kubernetes.operator.model.spec.TezAmSpec; -import org.apache.hive.kubernetes.operator.util.ConfigUtils; -import org.apache.hive.kubernetes.operator.util.HadoopXmlBuilder; -import org.apache.hive.kubernetes.operator.util.HiveConfigBuilder; -import org.apache.hive.kubernetes.operator.util.Labels; - -/** - * Manages the Kubernetes StatefulSet for the Tez Application Master. - * Uses StatefulSet (with a headless Service) so that each TezAM pod - * gets a stable, DNS-resolvable hostname. HiveServer2 discovers - * TezAM pods via ZooKeeper and connects over RPC using the hostname, - * so the hostname must be resolvable within the cluster. - */ -@KubernetesDependent( - informer = @Informer(labelSelector = "app.kubernetes.io/component=tezam," - + "app.kubernetes.io/managed-by=hive-kubernetes-operator") -) -public class TezAmStatefulSetDependent - extends HiveDependentResource { - - public static final String COMPONENT = ConfigUtils.COMPONENT_TEZAM; - private static final String SCRATCH_MOUNT_PATH = "/opt/hive/scratch"; - - public TezAmStatefulSetDependent() { - super(StatefulSet.class); - } - - @Override - protected String getSecondaryResourceName(HiveCluster primary, - Context context) { - return resourceName(primary); - } - - @Override - protected String getComponentName() { - return COMPONENT; - } - - @Override - protected StatefulSet desired(HiveCluster hiveCluster, - Context context) { - HiveClusterSpec spec = hiveCluster.getSpec(); - TezAmSpec tezAm = spec.tezAm(); - Map selectorLabels = - Labels.selectorForComponent(hiveCluster, COMPONENT); - - List envVars = new ArrayList<>(); - envVars.add(new EnvVar("SERVICE_NAME", COMPONENT, null)); - envVars.add(new EnvVar("IS_RESUME", "true", null)); - envVars.add(new EnvVar("HIVE_ZOOKEEPER_QUORUM", - spec.zookeeper().quorum(), null)); - envVars.add(new EnvVar("TEZ_FRAMEWORK_MODE", - "STANDALONE_ZOOKEEPER", null)); - - if (spec.llap().isEnabled()) { - envVars.add(new EnvVar("HIVE_LLAP_DAEMON_SERVICE_HOSTS", - spec.llap().serviceHosts(), null)); - } - - // User-provided env vars (storage credentials, etc.) - if (spec.envVars() != null) { - envVars.addAll(spec.envVars()); - } - - String headlessServiceName = - hiveCluster.getMetadata().getName() + "-tezam"; - - List volumeMounts = - new ArrayList<>(); - volumeMounts.add(new io.fabric8.kubernetes.api.model.VolumeMountBuilder() - .withName("hive-config") - .withMountPath(CONF_MOUNT_PATH).build()); - volumeMounts.add(new io.fabric8.kubernetes.api.model.VolumeMountBuilder() - .withName("scratch") - .withMountPath(SCRATCH_MOUNT_PATH).build()); - - List volumes = - new ArrayList<>(); - volumes.add(buildProjectedConfigVolume("hive-config", - HiveConfigMapDependent.HiveServer2.resourceName(hiveCluster), - HiveConfigMapDependent.Hadoop.resourceName(hiveCluster))); - volumes.add(new io.fabric8.kubernetes.api.model.VolumeBuilder() - .withName("scratch") - .withNewPersistentVolumeClaim() - .withClaimName(ScratchPvcDependent.resourceName(hiveCluster)) - .endPersistentVolumeClaim() - .build()); - - List ports = new ArrayList<>(); - List initContainers = new ArrayList<>(); - addExternalJars(spec.image(), spec.externalJars(), - initContainers, volumeMounts, volumes, envVars); - replaceConfMountWithSubPaths(volumeMounts, "hive-config", - "hive-site.xml", "tez-site.xml", "core-site.xml"); - - // Pre-compute config hash for the pod template annotation. - // TezAM uses the same ConfigMaps as HS2 (hive-site.xml + tez-site.xml + core-site.xml). - String configHash = sha256( - HadoopXmlBuilder.buildXml(HiveConfigBuilder.getHiveServer2HiveSite(hiveCluster, spec)), - HadoopXmlBuilder.buildXml(HiveConfigBuilder.getTezSite(spec)), - HadoopXmlBuilder.buildXml(HiveConfigBuilder.getHadoopCoreSite(spec))); - - AutoscalingSpec tezAmAutoscaling = tezAm.autoscaling(); - int initialReplicas = tezAmAutoscaling != null && tezAmAutoscaling.isEnabled() - ? tezAmAutoscaling.minReplicas() : tezAm.replicas(); - Integer replicas = resolveReplicaCount( - hiveCluster, context, tezAmAutoscaling, tezAm.replicas(), initialReplicas); - - StatefulSet statefulSet = new StatefulSetBuilder() - .withNewMetadata() - .withName(resourceName(hiveCluster)) - .withNamespace(hiveCluster.getMetadata().getNamespace()) - .withLabels(Labels.forComponent(hiveCluster, COMPONENT)) - .endMetadata() - .withNewSpec() - .withReplicas(replicas) - .withPodManagementPolicy("Parallel") - .withServiceName(headlessServiceName) - .withNewSelector() - .withMatchLabels(selectorLabels) - .endSelector() - .withNewTemplate() - .withNewMetadata() - .withLabels(Labels.forComponent(hiveCluster, COMPONENT)) - .addToAnnotations("kubectl.kubernetes.io/default-container", COMPONENT) - .addToAnnotations("hive.apache.org/config-hash", configHash) - .endMetadata() - .withNewSpec() - .withInitContainers(initContainers) - .addNewContainer() - .withName(COMPONENT) - .withImage(spec.image()) - .withImagePullPolicy(spec.imagePullPolicy()) - .withEnv(envVars) - .withPorts(ports) - .withResources(buildResources(tezAm.resources())) - .withVolumeMounts(volumeMounts) - .endContainer() - .withVolumes(volumes) - .endSpec() - .endTemplate() - .endSpec() - .build(); - - applySpreadAffinityIfAbsent( - statefulSet.getSpec().getTemplate().getSpec(), selectorLabels); - - appendUserVolumes(statefulSet.getSpec().getTemplate().getSpec(), - spec.volumes(), spec.volumeMounts(), - tezAm.extraVolumes(), tezAm.extraVolumeMounts()); - - return statefulSet; - } - - /** Returns the StatefulSet resource name for this HiveCluster. */ - public static String resourceName(HiveCluster hiveCluster) { - return hiveCluster.getMetadata().getName() + "-tezam"; - } -} diff --git a/packaging/src/kubernetes/src/java/org/apache/hive/kubernetes/operator/model/HiveClusterSpec.java b/packaging/src/kubernetes/src/java/org/apache/hive/kubernetes/operator/model/HiveClusterSpec.java index c527a7debbd0..263e8b349877 100644 --- a/packaging/src/kubernetes/src/java/org/apache/hive/kubernetes/operator/model/HiveClusterSpec.java +++ b/packaging/src/kubernetes/src/java/org/apache/hive/kubernetes/operator/model/HiveClusterSpec.java @@ -46,8 +46,10 @@ public record HiveClusterSpec( MetastoreSpec metastore, @JsonPropertyDescription("HiveServer2 component configuration") HiveServer2Spec hiveServer2, - @JsonPropertyDescription("LLAP daemon configuration. Enabled by default.") - LlapSpec llap, + @JsonPropertyDescription("LLAP compute clusters. Each entry is an independent LLAP cluster " + + "with its own StatefulSet, autoscaling, and ZooKeeper registration. " + + "Users select a cluster via hive.llap.daemon.service.hosts=@{name} in their session.") + List llapClusters, @JsonPropertyDescription("Tez Application Master configuration. Enabled by default.") TezAmSpec tezAm, @Required @@ -89,8 +91,7 @@ public record HiveClusterSpec( 1, null, null, null, null, null, null, true, null, null, null, null); hiveServer2 = hiveServer2 != null ? hiveServer2 : new HiveServer2Spec( 1, null, null, null, null, null, null, null, null, null); - llap = llap != null ? llap : new LlapSpec( - 1, null, null, null, null, true, null, null, null, null, null); + llapClusters = llapClusters != null ? llapClusters : List.of(); tezAm = tezAm != null ? tezAm : new TezAmSpec( 1, null, null, null, null, true, null, null, null); envVars = envVars != null ? envVars : List.of(); diff --git a/packaging/src/kubernetes/src/java/org/apache/hive/kubernetes/operator/model/HiveClusterStatus.java b/packaging/src/kubernetes/src/java/org/apache/hive/kubernetes/operator/model/HiveClusterStatus.java index ea6758309f11..f95fcd434be4 100644 --- a/packaging/src/kubernetes/src/java/org/apache/hive/kubernetes/operator/model/HiveClusterStatus.java +++ b/packaging/src/kubernetes/src/java/org/apache/hive/kubernetes/operator/model/HiveClusterStatus.java @@ -19,7 +19,9 @@ package org.apache.hive.kubernetes.operator.model; import java.util.ArrayList; +import java.util.LinkedHashMap; import java.util.List; +import java.util.Map; import io.fabric8.crd.generator.annotation.PrinterColumn; import io.fabric8.kubernetes.api.model.Condition; @@ -31,8 +33,8 @@ public class HiveClusterStatus { private List conditions = new ArrayList<>(); private ComponentStatus metastore; private ComponentStatus hiveServer2; - private ComponentStatus llap; - private ComponentStatus tezAm; + private Map llapClusters = new LinkedHashMap<>(); + private Map tezAmClusters = new LinkedHashMap<>(); private Long observedGeneration; @PrinterColumn(name = "Phase") private String clusterPhase; @@ -66,20 +68,20 @@ public void setHiveServer2(ComponentStatus hiveServer2) { this.hiveServer2 = hiveServer2; } - public ComponentStatus getLlap() { - return llap; + public Map getLlapClusters() { + return llapClusters; } - public void setLlap(ComponentStatus llap) { - this.llap = llap; + public void setLlapClusters(Map llapClusters) { + this.llapClusters = llapClusters; } - public ComponentStatus getTezAm() { - return tezAm; + public Map getTezAmClusters() { + return tezAmClusters; } - public void setTezAm(ComponentStatus tezAm) { - this.tezAm = tezAm; + public void setTezAmClusters(Map tezAmClusters) { + this.tezAmClusters = tezAmClusters; } public Long getObservedGeneration() { @@ -135,8 +137,8 @@ public boolean equals(Object o) { java.util.Objects.equals(conditions, that.conditions) && java.util.Objects.equals(metastore, that.metastore) && java.util.Objects.equals(hiveServer2, that.hiveServer2) && - java.util.Objects.equals(llap, that.llap) && - java.util.Objects.equals(tezAm, that.tezAm) && + java.util.Objects.equals(llapClusters, that.llapClusters) && + java.util.Objects.equals(tezAmClusters, that.tezAmClusters) && java.util.Objects.equals(clusterPhase, that.clusterPhase) && java.util.Objects.equals(idleSince, that.idleSince) && java.util.Objects.equals(idleForMinutes, that.idleForMinutes) && @@ -145,7 +147,7 @@ public boolean equals(Object o) { @Override public int hashCode() { - return java.util.Objects.hash(conditions, metastore, hiveServer2, llap, tezAm, + return java.util.Objects.hash(conditions, metastore, hiveServer2, llapClusters, tezAmClusters, observedGeneration, clusterPhase, idleSince, idleForMinutes, suspendedSince); } } diff --git a/packaging/src/kubernetes/src/java/org/apache/hive/kubernetes/operator/model/spec/LlapSpec.java b/packaging/src/kubernetes/src/java/org/apache/hive/kubernetes/operator/model/spec/LlapSpec.java index b4865d66ce43..6f095a560e32 100644 --- a/packaging/src/kubernetes/src/java/org/apache/hive/kubernetes/operator/model/spec/LlapSpec.java +++ b/packaging/src/kubernetes/src/java/org/apache/hive/kubernetes/operator/model/spec/LlapSpec.java @@ -20,16 +20,22 @@ import java.util.List; import java.util.Map; +import java.util.Objects; import com.fasterxml.jackson.annotation.JsonPropertyDescription; import io.fabric8.crd.generator.annotation.PreserveUnknownFields; import io.fabric8.crd.generator.annotation.SchemaFrom; import io.fabric8.generator.annotation.Default; +import io.fabric8.generator.annotation.Required; import io.fabric8.kubernetes.api.model.Volume; import io.fabric8.kubernetes.api.model.VolumeMount; /** Configuration for LLAP (Live Long and Process) daemons. */ public record LlapSpec( + @Required + @JsonPropertyDescription("Unique name for this LLAP cluster (e.g. llap0, llap1). " + + "Used as the ZooKeeper registration namespace and Kubernetes resource suffix.") + String name, @JsonPropertyDescription("Number of replicas") @Default("1") Integer replicas, @@ -52,23 +58,44 @@ public record LlapSpec( @JsonPropertyDescription("Memory in MB per LLAP daemon instance") @Default("1024") Integer memoryMb, - @JsonPropertyDescription("LLAP service hosts identifier for ZooKeeper registration") + @JsonPropertyDescription("LLAP service hosts identifier for ZooKeeper registration. " + + "Defaults to @{name} (e.g. @llap0).") String serviceHosts, @JsonPropertyDescription("Readiness probe configuration") ProbeSpec readinessProbe, @JsonPropertyDescription("Autoscaling configuration (operator-driven, no external dependencies)") - AutoscalingSpec autoscaling) { + AutoscalingSpec autoscaling, + @JsonPropertyDescription("Per-LLAP TezAM configuration. Each LLAP cluster gets its own TezAM " + + "with independent replica count and autoscaling.") + LlapTezAmSpec tezAm) { + + /** Per-LLAP-cluster TezAM replica and autoscaling overrides. */ + public record LlapTezAmSpec( + @JsonPropertyDescription("Max number of TezAM replicas for this LLAP cluster") + @Default("1") + Integer replicas, + @JsonPropertyDescription("Autoscaling configuration for this LLAP cluster's TezAM") + AutoscalingSpec autoscaling) { + + public LlapTezAmSpec { + replicas = replicas != null ? replicas : 1; + autoscaling = autoscaling != null ? autoscaling : new AutoscalingSpec( + false, 0, 0, 60, 600, 120, 10, 0, 0, null); + } + } public LlapSpec { + Objects.requireNonNull(name, "llapClusters[].name is required"); replicas = replicas != null ? replicas : 1; enabled = enabled != null ? enabled : true; executors = executors != null ? executors : 1; memoryMb = memoryMb != null ? memoryMb : 1024; - serviceHosts = serviceHosts != null ? serviceHosts : "@llap0"; + serviceHosts = serviceHosts != null ? serviceHosts : "@" + name; extraVolumes = extraVolumes != null ? extraVolumes : List.of(); extraVolumeMounts = extraVolumeMounts != null ? extraVolumeMounts : List.of(); autoscaling = autoscaling != null ? autoscaling : new AutoscalingSpec( false, 0, 1, 60, 900, 600, 10, 0, 0, null); + tezAm = tezAm != null ? tezAm : new LlapTezAmSpec(null, null); } public boolean isEnabled() { diff --git a/packaging/src/kubernetes/src/java/org/apache/hive/kubernetes/operator/reconciler/HiveClusterReconciler.java b/packaging/src/kubernetes/src/java/org/apache/hive/kubernetes/operator/reconciler/HiveClusterReconciler.java index 4d7a097c8a1d..574f3ab4ee9d 100644 --- a/packaging/src/kubernetes/src/java/org/apache/hive/kubernetes/operator/reconciler/HiveClusterReconciler.java +++ b/packaging/src/kubernetes/src/java/org/apache/hive/kubernetes/operator/reconciler/HiveClusterReconciler.java @@ -22,9 +22,11 @@ import java.time.Instant; import java.util.ArrayList; import java.util.Collections; +import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.Objects; +import java.util.Set; import io.fabric8.kubernetes.api.model.Condition; import io.fabric8.kubernetes.api.model.HasMetadata; @@ -44,13 +46,16 @@ import org.apache.hive.kubernetes.operator.autoscaling.MetricsCache; import org.apache.hive.kubernetes.operator.autoscaling.MetricsScraper; import org.apache.hive.kubernetes.operator.autoscaling.PodMetrics; +import org.apache.hive.kubernetes.operator.dependent.LlapResourceBuilder; import org.apache.hive.kubernetes.operator.model.HiveCluster; import org.apache.hive.kubernetes.operator.model.HiveClusterSpec; import org.apache.hive.kubernetes.operator.model.HiveClusterStatus; import org.apache.hive.kubernetes.operator.model.spec.AutoSuspendSpec; +import org.apache.hive.kubernetes.operator.model.spec.LlapSpec; import org.apache.hive.kubernetes.operator.model.status.AutoscalingStatus; import org.apache.hive.kubernetes.operator.model.status.ComponentStatus; import org.apache.hive.kubernetes.operator.util.ConfigUtils; +import org.apache.hive.kubernetes.operator.util.Labels; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -151,6 +156,11 @@ public UpdateControl reconcile(HiveCluster resource, Context llapStatuses = new java.util.LinkedHashMap<>(); + for (var llapSpec : resource.getSpec().llapClusters()) { + if (!llapSpec.isEnabled()) { + continue; + } + String ssName = resource.getMetadata().getName() + "-" + llapSpec.name(); + int llapMin = llapSpec.autoscaling().isEnabled() + ? llapSpec.autoscaling().minReplicas() + : llapSpec.replicas(); + llapStatuses.put(llapSpec.name(), + buildComponentStatus(context, StatefulSet.class, ssName, llapSpec.replicas(), llapMin)); } + status.setLlapClusters(llapStatuses); - // TezAM status (optional) + // Per-LLAP TezAM status (one TezAM per LLAP cluster) if (resource.getSpec().tezAm().isEnabled()) { - int tezAmMin = resource.getSpec().tezAm().autoscaling().isEnabled() - ? resource.getSpec().tezAm().autoscaling().minReplicas() - : resource.getSpec().tezAm().replicas(); - status.setTezAm(buildComponentStatus(context, StatefulSet.class, - resource.getMetadata().getName() + "-tezam", - resource.getSpec().tezAm().replicas(), tezAmMin)); + Map tezAmStatuses = new java.util.LinkedHashMap<>(); + for (var llapSpec : resource.getSpec().llapClusters()) { + if (!llapSpec.isEnabled()) { + continue; + } + LlapSpec.LlapTezAmSpec perLlapTezAm = llapSpec.tezAm(); + String tezAmSsName = LlapResourceBuilder.tezAmResourceName(resource, llapSpec); + int tezAmMin = perLlapTezAm.autoscaling().isEnabled() + ? perLlapTezAm.autoscaling().minReplicas() + : perLlapTezAm.replicas(); + tezAmStatuses.put(llapSpec.name(), + buildComponentStatus(context, StatefulSet.class, tezAmSsName, + perLlapTezAm.replicas(), tezAmMin)); + } + status.setTezAmClusters(tezAmStatuses); } // Overall Ready condition @@ -418,10 +442,10 @@ private boolean statusEqualsIgnoringTimestamps(HiveClusterStatus a, HiveClusterS if (!Objects.equals(a.getHiveServer2(), b.getHiveServer2())) { return false; } - if (!Objects.equals(a.getLlap(), b.getLlap())) { + if (!Objects.equals(a.getLlapClusters(), b.getLlapClusters())) { return false; } - if (!Objects.equals(a.getTezAm(), b.getTezAm())) { + if (!Objects.equals(a.getTezAmClusters(), b.getTezAmClusters())) { return false; } // Compare conditions by type+status+reason+message, ignoring lastTransitionTime @@ -459,11 +483,17 @@ private void applyAutoscalingStatuses(HiveClusterStatus status, if (statuses.containsKey(ConfigUtils.COMPONENT_METASTORE) && status.getMetastore() != null) { status.getMetastore().setAutoscaling(statuses.get(ConfigUtils.COMPONENT_METASTORE)); } - if (statuses.containsKey(ConfigUtils.COMPONENT_LLAP) && status.getLlap() != null) { - status.getLlap().setAutoscaling(statuses.get(ConfigUtils.COMPONENT_LLAP)); + for (Map.Entry entry : status.getLlapClusters().entrySet()) { + String llapKey = ConfigUtils.llapComponentKey(entry.getKey()); + if (statuses.containsKey(llapKey)) { + entry.getValue().setAutoscaling(statuses.get(llapKey)); + } } - if (statuses.containsKey(ConfigUtils.COMPONENT_TEZAM) && status.getTezAm() != null) { - status.getTezAm().setAutoscaling(statuses.get(ConfigUtils.COMPONENT_TEZAM)); + for (Map.Entry entry : status.getTezAmClusters().entrySet()) { + String tezAmKey = ConfigUtils.tezAmComponentKey(entry.getKey()); + if (statuses.containsKey(tezAmKey)) { + entry.getValue().setAutoscaling(statuses.get(tezAmKey)); + } } } @@ -486,11 +516,13 @@ private static boolean anyAutoscalingEnabled(HiveClusterSpec spec) { if (spec.metastore().isEnabled() && spec.metastore().autoscaling().isEnabled()) { return true; } - if (spec.llap().isEnabled() && spec.llap().autoscaling().isEnabled()) { - return true; - } - if (spec.tezAm().isEnabled() && spec.tezAm().autoscaling().isEnabled()) { - return true; + for (var llap : spec.llapClusters()) { + if (llap.isEnabled() && llap.autoscaling().isEnabled()) { + return true; + } + if (llap.isEnabled() && spec.tezAm().isEnabled() && llap.tezAm().autoscaling().isEnabled()) { + return true; + } } return false; } @@ -503,11 +535,13 @@ private static int getMinScrapeInterval(HiveClusterSpec spec) { if (spec.metastore().isEnabled() && spec.metastore().autoscaling().isEnabled()) { min = Math.min(min, spec.metastore().autoscaling().metricsScrapeIntervalSeconds()); } - if (spec.llap().isEnabled() && spec.llap().autoscaling().isEnabled()) { - min = Math.min(min, spec.llap().autoscaling().metricsScrapeIntervalSeconds()); - } - if (spec.tezAm().isEnabled() && spec.tezAm().autoscaling().isEnabled()) { - min = Math.min(min, spec.tezAm().autoscaling().metricsScrapeIntervalSeconds()); + for (var llap : spec.llapClusters()) { + if (llap.isEnabled() && llap.autoscaling().isEnabled()) { + min = Math.min(min, llap.autoscaling().metricsScrapeIntervalSeconds()); + } + if (llap.isEnabled() && spec.tezAm().isEnabled() && llap.tezAm().autoscaling().isEnabled()) { + min = Math.min(min, llap.tezAm().autoscaling().metricsScrapeIntervalSeconds()); + } } return min == Integer.MAX_VALUE ? 10 : min; } @@ -515,9 +549,21 @@ private static int getMinScrapeInterval(HiveClusterSpec spec) { private void patchReplicas(KubernetesClient client, HiveCluster resource, String component, int replicas) { String namespace = resource.getMetadata().getNamespace(); - String workloadName = resource.getMetadata().getName() + "-" + component; + // Component keys use prefixes: "llap-{name}" → workload "{cluster}-{name}", + // "tezam-{name}" → workload "{cluster}-tezam-{name}". + String workloadName; + if (component.startsWith(ConfigUtils.COMPONENT_LLAP + "-")) { + String llapName = component.substring(ConfigUtils.COMPONENT_LLAP.length() + 1); + workloadName = resource.getMetadata().getName() + "-" + llapName; + } else if (component.startsWith(ConfigUtils.COMPONENT_TEZAM + "-")) { + String llapName = component.substring(ConfigUtils.COMPONENT_TEZAM.length() + 1); + workloadName = resource.getMetadata().getName() + "-tezam-" + llapName; + } else { + workloadName = resource.getMetadata().getName() + "-" + component; + } try { - if (ConfigUtils.COMPONENT_LLAP.equals(component) || ConfigUtils.COMPONENT_TEZAM.equals(component)) { + if (component.startsWith(ConfigUtils.COMPONENT_LLAP + "-") + || component.startsWith(ConfigUtils.COMPONENT_TEZAM + "-")) { client.apps().statefulSets().inNamespace(namespace).withName(workloadName).scale(replicas); } else { client.apps().deployments().inNamespace(namespace).withName(workloadName).scale(replicas); @@ -537,7 +583,7 @@ private void patchSuspendSpec(KubernetesClient client, HiveCluster resource, boo HiveClusterSpec oldSpec = hc.getSpec(); HiveClusterSpec newSpec = new HiveClusterSpec( oldSpec.image(), oldSpec.imagePullPolicy(), oldSpec.metastore(), - oldSpec.hiveServer2(), oldSpec.llap(), oldSpec.tezAm(), oldSpec.zookeeper(), + oldSpec.hiveServer2(), oldSpec.llapClusters(), oldSpec.tezAm(), oldSpec.zookeeper(), oldSpec.hadoop(), oldSpec.envVars(), oldSpec.externalJars(), oldSpec.volumes(), oldSpec.volumeMounts(), oldSpec.autoSuspend(), suspend); hc.setSpec(newSpec); @@ -546,6 +592,174 @@ private void patchSuspendSpec(KubernetesClient client, HiveCluster resource, boo LOG.info("Patched spec.suspend={} on {}/{}", suspend, ns, name); } + // --- Imperative LLAP cluster management --- + + /** + * Creates or updates LLAP cluster resources (ConfigMap, Service, StatefulSet, PDB) + * imperatively via server-side apply. Also garbage-collects resources for removed clusters. + */ + private void reconcileLlapClusters(HiveCluster resource, KubernetesClient client) { + String ns = resource.getMetadata().getNamespace(); + String clusterName = resource.getMetadata().getName(); + Set desiredNames = new HashSet<>(); + + for (LlapSpec llapSpec : resource.getSpec().llapClusters()) { + if (!llapSpec.isEnabled()) { + continue; + } + desiredNames.add(llapSpec.name()); + int replicas = resolveLlapReplicaCount(client, resource, llapSpec, ns, clusterName); + + // --- LLAP resources --- + client.configMaps().inNamespace(ns) + .resource(LlapResourceBuilder.buildConfigMap(resource, llapSpec)) + .serverSideApply(); + client.services().inNamespace(ns) + .resource(LlapResourceBuilder.buildService(resource, llapSpec)) + .serverSideApply(); + + // Always include replicas in SSA with forceConflicts to avoid the + // brief scale-up-then-down on first create (K8s defaults to 1 if omitted). + // resolveLlapReplicaCount already reads the autoscaler's managed value, + // so this is always the correct replica count. + client.apps().statefulSets().inNamespace(ns) + .resource(LlapResourceBuilder.buildStatefulSet(resource, llapSpec, replicas)) + .forceConflicts() + .serverSideApply(); + if (llapSpec.autoscaling().isEnabled()) { + client.policy().v1().podDisruptionBudget().inNamespace(ns) + .resource(LlapResourceBuilder.buildPdb(resource, llapSpec)) + .serverSideApply(); + } + + // --- Per-LLAP TezAM resources (one TezAM per LLAP cluster) --- + if (resource.getSpec().tezAm().isEnabled()) { + int tezAmReplicas = resolveTezAmReplicaCount(resource, ns, clusterName, llapSpec); + client.configMaps().inNamespace(ns) + .resource(LlapResourceBuilder.buildTezAmConfigMap(resource, llapSpec)) + .serverSideApply(); + client.services().inNamespace(ns) + .resource(LlapResourceBuilder.buildTezAmService(resource, llapSpec)) + .serverSideApply(); + client.apps().statefulSets().inNamespace(ns) + .resource(LlapResourceBuilder.buildTezAmStatefulSet(resource, llapSpec, tezAmReplicas)) + .forceConflicts() + .serverSideApply(); + if (llapSpec.tezAm().autoscaling().isEnabled()) { + client.policy().v1().podDisruptionBudget().inNamespace(ns) + .resource(LlapResourceBuilder.buildTezAmPdb(resource, llapSpec)) + .serverSideApply(); + } + } + } + + garbageCollectLlapResources(client, ns, clusterName, desiredNames); + } + + /** + * Resolves the replica count for a LLAP cluster, respecting autoscaler-managed values + * and suspend state. + */ + private int resolveLlapReplicaCount(KubernetesClient client, HiveCluster resource, + LlapSpec llapSpec, String ns, String clusterName) { + if (resource.getSpec().suspend()) { + return 0; + } + String componentKey = ConfigUtils.llapComponentKey(llapSpec.name()); + Integer managed = HiveClusterAutoscaler.getManagedReplicas(ns, clusterName, componentKey); + if (managed != null) { + return managed; + } + // First reconcile before autoscaler runs: start at minReplicas if autoscaling enabled + if (llapSpec.autoscaling().isEnabled()) { + return llapSpec.autoscaling().minReplicas(); + } + return llapSpec.replicas(); + } + + /** + * Resolves the replica count for a per-LLAP TezAM cluster. + * TezAM follows its paired LLAP cluster's lifecycle. + */ + private int resolveTezAmReplicaCount(HiveCluster resource, + String ns, String clusterName, LlapSpec llapSpec) { + if (resource.getSpec().suspend()) { + return 0; + } + LlapSpec.LlapTezAmSpec tezAmSpec = llapSpec.tezAm(); + // Check if autoscaler has a managed value for this specific TezAM + String tezAmComponentKey = ConfigUtils.tezAmComponentKey(llapSpec.name()); + Integer tezAmManaged = HiveClusterAutoscaler.getManagedReplicas(ns, clusterName, tezAmComponentKey); + if (tezAmManaged != null) { + return tezAmManaged; + } + // TezAM follows LLAP's autoscaling gate: only run if LLAP is running. + String llapComponentKey = ConfigUtils.llapComponentKey(llapSpec.name()); + Integer llapManaged = HiveClusterAutoscaler.getManagedReplicas(ns, clusterName, llapComponentKey); + if (llapManaged != null && llapManaged == 0) { + return 0; + } + if (llapSpec.autoscaling().isEnabled() && llapManaged == null) { + // First reconcile, LLAP starts at minReplicas (likely 0) — TezAM matches + return llapSpec.autoscaling().minReplicas() > 0 + ? tezAmSpec.replicas() : 0; + } + return tezAmSpec.replicas(); + } + + /** + * Deletes LLAP and per-LLAP TezAM resources that belong to this HiveCluster + * but are no longer in the desired set of LLAP cluster names. + */ + private void garbageCollectLlapResources(KubernetesClient client, String ns, + String clusterName, Set desiredNames) { + Map llapSelector = Map.of( + Labels.MANAGED_BY, Labels.MANAGED_BY_VALUE, + Labels.APP_INSTANCE, clusterName, + Labels.APP_COMPONENT, ConfigUtils.COMPONENT_LLAP); + + // Find StatefulSets owned by this cluster with component=llap + client.apps().statefulSets().inNamespace(ns).withLabels(llapSelector).list().getItems() + .stream() + .filter(ss -> { + String llapName = ss.getMetadata().getLabels().get(Labels.LLAP_CLUSTER); + return llapName != null && !desiredNames.contains(llapName); + }) + .forEach(ss -> { + String llapName = ss.getMetadata().getLabels().get(Labels.LLAP_CLUSTER); + LOG.info("Garbage-collecting LLAP cluster '{}' resources in {}/{}", llapName, ns, clusterName); + client.apps().statefulSets().inNamespace(ns).withName(ss.getMetadata().getName()).delete(); + client.services().inNamespace(ns).withName(ss.getMetadata().getName()).delete(); + client.configMaps().inNamespace(ns) + .withName(ss.getMetadata().getName() + "-config").delete(); + client.policy().v1().podDisruptionBudget().inNamespace(ns) + .withName(ss.getMetadata().getName() + "-pdb").delete(); + }); + + // Garbage-collect per-LLAP TezAM resources + Map tezamSelector = Map.of( + Labels.MANAGED_BY, Labels.MANAGED_BY_VALUE, + Labels.APP_INSTANCE, clusterName, + Labels.APP_COMPONENT, ConfigUtils.COMPONENT_TEZAM); + + client.apps().statefulSets().inNamespace(ns).withLabels(tezamSelector).list().getItems() + .stream() + .filter(ss -> { + String llapName = ss.getMetadata().getLabels().get(Labels.LLAP_CLUSTER); + return llapName != null && !desiredNames.contains(llapName); + }) + .forEach(ss -> { + String llapName = ss.getMetadata().getLabels().get(Labels.LLAP_CLUSTER); + LOG.info("Garbage-collecting TezAM for LLAP cluster '{}' in {}/{}", llapName, ns, clusterName); + client.apps().statefulSets().inNamespace(ns).withName(ss.getMetadata().getName()).delete(); + client.services().inNamespace(ns).withName(ss.getMetadata().getName()).delete(); + client.configMaps().inNamespace(ns) + .withName(ss.getMetadata().getName() + "-config").delete(); + client.policy().v1().podDisruptionBudget().inNamespace(ns) + .withName(ss.getMetadata().getName() + "-pdb").delete(); + }); + } + // --- Auto-Suspend / Wake --- enum SuspendAction { RUNNING, IDLE_START, IDLE_WAITING, SUSPEND_NOW, STAY_SUSPENDED, WAKE } @@ -603,16 +817,22 @@ private boolean isClusterIdle(HiveCluster resource, KubernetesClient client) { String ns = resource.getMetadata().getNamespace(); String name = resource.getMetadata().getName(); - // All components must be at minReplicas - if (spec.llap().isEnabled() - && !isAtMinReplicas(client, ns, name + "-" + ConfigUtils.COMPONENT_LLAP, true, - spec.llap().autoscaling().minReplicas())) { - return false; + // All LLAP clusters must be at minReplicas + for (var llap : spec.llapClusters()) { + if (llap.isEnabled() + && !isAtMinReplicas(client, ns, name + "-" + llap.name(), true, + llap.autoscaling().minReplicas())) { + return false; + } } - if (spec.tezAm().isEnabled() - && !isAtMinReplicas(client, ns, name + "-" + ConfigUtils.COMPONENT_TEZAM, true, - spec.tezAm().autoscaling().minReplicas())) { - return false; + if (spec.tezAm().isEnabled()) { + for (var llap : spec.llapClusters()) { + if (llap.isEnabled() + && !isAtMinReplicas(client, ns, name + "-tezam-" + llap.name(), true, + llap.tezAm().autoscaling().minReplicas())) { + return false; + } + } } if (!isAtMinReplicas(client, ns, name + "-" + ConfigUtils.COMPONENT_HIVESERVER2, false, Math.max(1, spec.hiveServer2().autoscaling().minReplicas()))) { @@ -684,11 +904,18 @@ private void suspendCluster(HiveCluster resource) { if (spec.metastore().isEnabled() && spec.autoSuspend().includeMetastore()) { HiveClusterAutoscaler.setManagedReplicas(ns, name, ConfigUtils.COMPONENT_METASTORE, 0); } - if (spec.llap().isEnabled()) { - HiveClusterAutoscaler.setManagedReplicas(ns, name, ConfigUtils.COMPONENT_LLAP, 0); + for (var llap : spec.llapClusters()) { + if (llap.isEnabled()) { + HiveClusterAutoscaler.setManagedReplicas(ns, name, ConfigUtils.llapComponentKey(llap.name()), 0); + } } if (spec.tezAm().isEnabled()) { - HiveClusterAutoscaler.setManagedReplicas(ns, name, ConfigUtils.COMPONENT_TEZAM, 0); + for (var llap : spec.llapClusters()) { + if (llap.isEnabled()) { + HiveClusterAutoscaler.setManagedReplicas(ns, name, + ConfigUtils.tezAmComponentKey(llap.name()), 0); + } + } } LOG.info("Cluster {}/{} suspended", ns, name); @@ -711,14 +938,21 @@ private void wakeCluster(HiveCluster resource) { HiveClusterAutoscaler.setManagedReplicas(ns, name, ConfigUtils.COMPONENT_METASTORE, hmsMin); } - if (spec.llap().isEnabled()) { - int llapWake = spec.llap().autoscaling().minReplicas(); - HiveClusterAutoscaler.setManagedReplicas(ns, name, ConfigUtils.COMPONENT_LLAP, llapWake); + for (var llap : spec.llapClusters()) { + if (llap.isEnabled()) { + int llapWake = llap.autoscaling().minReplicas(); + HiveClusterAutoscaler.setManagedReplicas(ns, name, ConfigUtils.llapComponentKey(llap.name()), llapWake); + } } if (spec.tezAm().isEnabled()) { - int tezWake = spec.tezAm().autoscaling().minReplicas(); - HiveClusterAutoscaler.setManagedReplicas(ns, name, ConfigUtils.COMPONENT_TEZAM, tezWake); + for (var llap : spec.llapClusters()) { + if (llap.isEnabled()) { + int tezWake = llap.tezAm().autoscaling().minReplicas(); + HiveClusterAutoscaler.setManagedReplicas(ns, name, + ConfigUtils.tezAmComponentKey(llap.name()), tezWake); + } + } } LOG.info("Cluster {}/{} woken up — restored to minReplicas", ns, name); @@ -733,11 +967,13 @@ private static boolean allAutoscalingEnabled(HiveClusterSpec spec) { && !spec.metastore().autoscaling().isEnabled()) { return false; } - if (spec.llap().isEnabled() && !spec.llap().autoscaling().isEnabled()) { - return false; - } - if (spec.tezAm().isEnabled() && !spec.tezAm().autoscaling().isEnabled()) { - return false; + for (var llap : spec.llapClusters()) { + if (llap.isEnabled() && !llap.autoscaling().isEnabled()) { + return false; + } + if (llap.isEnabled() && spec.tezAm().isEnabled() && !llap.tezAm().autoscaling().isEnabled()) { + return false; + } } return true; } diff --git a/packaging/src/kubernetes/src/java/org/apache/hive/kubernetes/operator/reconciler/HiveWorkflowSpec.java b/packaging/src/kubernetes/src/java/org/apache/hive/kubernetes/operator/reconciler/HiveWorkflowSpec.java index c8ee85759678..6500100918ee 100644 --- a/packaging/src/kubernetes/src/java/org/apache/hive/kubernetes/operator/reconciler/HiveWorkflowSpec.java +++ b/packaging/src/kubernetes/src/java/org/apache/hive/kubernetes/operator/reconciler/HiveWorkflowSpec.java @@ -31,11 +31,9 @@ import org.apache.hive.kubernetes.operator.dependent.HiveServer2DeploymentDependent; import org.apache.hive.kubernetes.operator.dependent.HivePdbDependent; import org.apache.hive.kubernetes.operator.dependent.HiveServiceDependent; -import org.apache.hive.kubernetes.operator.dependent.LlapStatefulSetDependent; import org.apache.hive.kubernetes.operator.dependent.MetastoreDeploymentDependent; import org.apache.hive.kubernetes.operator.dependent.SchemaInitJobDependent; import org.apache.hive.kubernetes.operator.dependent.ScratchPvcDependent; -import org.apache.hive.kubernetes.operator.dependent.TezAmStatefulSetDependent; import org.apache.hive.kubernetes.operator.model.HiveCluster; /** @@ -50,46 +48,28 @@ public final class HiveWorkflowSpec implements WorkflowSpec { private static final String HADOOP_CONFIGMAP = "hadoop-configmap"; private static final String METASTORE_CONFIGMAP = "metastore-configmap"; private static final String HIVESERVER2_CONFIGMAP = "hiveserver2-configmap"; - private static final String LLAP_CONFIGMAP = "llap-configmap"; private static final String SCHEMA_INIT_JOB = "schema-init-job"; private static final String METASTORE_DEPLOYMENT = "metastore-deployment"; private static final String METASTORE_SERVICE = "metastore-service"; private static final String HIVESERVER2_DEPLOYMENT = "hiveserver2-deployment"; private static final String HIVESERVER2_SERVICE = "hiveserver2-service"; - private static final String LLAP_STATEFULSET = "llap-statefulset"; - private static final String LLAP_SERVICE = "llap-service"; - private static final String TEZAM_SERVICE = "tezam-service"; - private static final String TEZAM_STATEFULSET = "tezam-statefulset"; private static final String SCRATCH_PVC = "scratch-pvc"; private static final String HS2_PDB = "hs2-pdb"; private static final String METASTORE_PDB = "metastore-pdb"; - private static final String LLAP_PDB = "llap-pdb"; - private static final String TEZAM_PDB = "tezam-pdb"; private static final Condition METASTORE_ENABLED = (dr, primary, ctx) -> primary.getSpec().metastore().isEnabled(); - private static final Condition LLAP_ENABLED = - (dr, primary, ctx) -> primary.getSpec().llap().isEnabled(); - - private static final Condition TEZAM_ENABLED = - (dr, primary, ctx) -> primary.getSpec().tezAm().isEnabled(); - private static final Condition METASTORE_AUTOSCALING = (dr, primary, ctx) -> primary.getSpec().metastore().isEnabled() && primary.getSpec().metastore().autoscaling().isEnabled(); - private static final Condition LLAP_AUTOSCALING = - (dr, primary, ctx) -> primary.getSpec().llap().isEnabled() - && primary.getSpec().llap().autoscaling().isEnabled(); - - private static final Condition TEZAM_AUTOSCALING = - (dr, primary, ctx) -> primary.getSpec().tezAm().isEnabled() - && primary.getSpec().tezAm().autoscaling().isEnabled(); - private static final Condition HS2_AUTOSCALING = (dr, primary, ctx) -> primary.getSpec().hiveServer2().autoscaling().isEnabled(); + private static final Condition TEZAM_ENABLED = + (dr, primary, ctx) -> primary.getSpec().tezAm().isEnabled(); + // SPECS must be declared AFTER all conditions to avoid static init order issues. private static final List SPECS = buildSpecs(); @@ -129,9 +109,14 @@ private static List buildSpecs() { Set.of(METASTORE_CONFIGMAP), null, null, null, METASTORE_ENABLED, null)); + // --- Shared Scratch PVC (Required for HS2 to TezAM communication) --- + specs.add(new DependentResourceSpec( + ScratchPvcDependent.class, SCRATCH_PVC, + Set.of(), null, null, null, TEZAM_ENABLED, null)); + specs.add(new DependentResourceSpec( HiveServer2DeploymentDependent.class, HIVESERVER2_DEPLOYMENT, - Set.of(HIVESERVER2_CONFIGMAP, HADOOP_CONFIGMAP), + Set.of(HIVESERVER2_CONFIGMAP, HADOOP_CONFIGMAP, SCRATCH_PVC), null, hs2Precondition(), null, null, null)); specs.add(new DependentResourceSpec( @@ -139,34 +124,6 @@ private static List buildSpecs() { Set.of(HIVESERVER2_CONFIGMAP), null, null, null, null, null)); - // --- LLAP (conditional) --- - specs.add(new DependentResourceSpec( - HiveConfigMapDependent.Llap.class, LLAP_CONFIGMAP, - Set.of(), null, null, null, LLAP_ENABLED, null)); - - specs.add(new DependentResourceSpec( - LlapStatefulSetDependent.class, LLAP_STATEFULSET, - Set.of(LLAP_CONFIGMAP, HADOOP_CONFIGMAP), - null, null, null, LLAP_ENABLED, null)); - - specs.add(new DependentResourceSpec( - HiveServiceDependent.Llap.class, LLAP_SERVICE, - Set.of(), null, null, null, LLAP_ENABLED, null)); - - // --- TezAM (conditional) --- - specs.add(new DependentResourceSpec( - ScratchPvcDependent.class, SCRATCH_PVC, - Set.of(), null, null, null, TEZAM_ENABLED, null)); - - specs.add(new DependentResourceSpec( - HiveServiceDependent.TezAm.class, TEZAM_SERVICE, - Set.of(), null, null, null, TEZAM_ENABLED, null)); - - specs.add(new DependentResourceSpec( - TezAmStatefulSetDependent.class, TEZAM_STATEFULSET, - Set.of(HIVESERVER2_CONFIGMAP, HADOOP_CONFIGMAP, TEZAM_SERVICE, SCRATCH_PVC), - null, null, null, TEZAM_ENABLED, null)); - // --- Autoscaling: PodDisruptionBudgets (conditional) --- specs.add(new DependentResourceSpec( HivePdbDependent.HiveServer2.class, HS2_PDB, @@ -178,16 +135,6 @@ private static List buildSpecs() { Set.of(METASTORE_DEPLOYMENT), null, METASTORE_AUTOSCALING, null, null, null)); - specs.add(new DependentResourceSpec( - HivePdbDependent.Llap.class, LLAP_PDB, - Set.of(LLAP_STATEFULSET), - null, LLAP_AUTOSCALING, null, null, null)); - - specs.add(new DependentResourceSpec( - HivePdbDependent.TezAm.class, TEZAM_PDB, - Set.of(TEZAM_STATEFULSET), - null, TEZAM_AUTOSCALING, null, null, null)); - return Collections.unmodifiableList(specs); } diff --git a/packaging/src/kubernetes/src/java/org/apache/hive/kubernetes/operator/util/ConfigUtils.java b/packaging/src/kubernetes/src/java/org/apache/hive/kubernetes/operator/util/ConfigUtils.java index c3d824763c60..d7512e39fae6 100644 --- a/packaging/src/kubernetes/src/java/org/apache/hive/kubernetes/operator/util/ConfigUtils.java +++ b/packaging/src/kubernetes/src/java/org/apache/hive/kubernetes/operator/util/ConfigUtils.java @@ -31,6 +31,16 @@ private ConfigUtils() { public static final String COMPONENT_LLAP = "llap"; public static final String COMPONENT_TEZAM = "tezam"; + /** Returns the autoscaler component key for a specific LLAP cluster (e.g., "llap-llap0"). */ + public static String llapComponentKey(String llapName) { + return COMPONENT_LLAP + "-" + llapName; + } + + /** Returns the autoscaler component key for a per-LLAP TezAM (e.g., "tezam-llap0"). */ + public static String tezAmComponentKey(String llapName) { + return COMPONENT_TEZAM + "-" + llapName; + } + public static final String METASTORE_THRIFT_PORT_KEY = "metastore.thrift.port"; public static final String METASTORE_THRIFT_PORT_HIVE_KEY = "hive.metastore.port"; public static final int METASTORE_THRIFT_PORT_DEFAULT = 9083; @@ -64,6 +74,12 @@ private ConfigUtils() { public static final String HIVE_SERVER2_TEZ_EXTERNAL_SESSIONS_NAMESPACE_KEY = "hive.server2.tez.external.sessions.namespace"; + /** + * ZK path prefix that Tez's STANDALONE_ZOOKEEPER mode prepends to tez.am.registry.namespace + * when creating the session availability node. Must match the Docker template convention. + */ + public static final String TEZ_EXTERNAL_SESSIONS_ZK_PREFIX = "/tez-external-sessions"; + public static final String HIVE_SERVER2_TEZ_EXTERNAL_SESSIONS_REGISTRY_CLASS_KEY = "hive.server2.tez.external.sessions.registry.class"; @@ -94,6 +110,9 @@ private ConfigUtils() { public static final String HIVE_LLAP_DAEMON_OUTPUT_SERVICE_PORT_KEY = "hive.llap.daemon.output.service.port"; public static final int HIVE_LLAP_DAEMON_OUTPUT_SERVICE_PORT_DEFAULT = 15003; + public static final String HIVE_LLAP_DAEMON_UMBILICAL_PORT_KEY = "hive.llap.daemon.umbilical.port"; + public static final String HIVE_LLAP_DAEMON_UMBILICAL_PORT_DEFAULT = "0"; + public static final String METASTORE_SERVER_TRANSPORT_MODE_KEY = "metastore.server.thrift.transport.mode"; public static final String METASTORE_SERVER_TRANSPORT_MODE_DEFAULT = "http"; diff --git a/packaging/src/kubernetes/src/java/org/apache/hive/kubernetes/operator/util/HiveConfigBuilder.java b/packaging/src/kubernetes/src/java/org/apache/hive/kubernetes/operator/util/HiveConfigBuilder.java index f046b685f653..120a8a2c7494 100644 --- a/packaging/src/kubernetes/src/java/org/apache/hive/kubernetes/operator/util/HiveConfigBuilder.java +++ b/packaging/src/kubernetes/src/java/org/apache/hive/kubernetes/operator/util/HiveConfigBuilder.java @@ -20,6 +20,7 @@ import java.util.LinkedHashMap; import java.util.Map; +import java.util.Optional; import org.apache.hive.kubernetes.operator.model.HiveCluster; import org.apache.hive.kubernetes.operator.model.HiveClusterSpec; @@ -82,23 +83,32 @@ public static Map getHiveServer2HiveSite( if (tezAmEnabled) { props.put(ConfigUtils.HIVE_SERVER2_TEZ_USE_EXTERNAL_SESSIONS_KEY, "true"); + // Default external sessions namespace points to first LLAP cluster's TezAM. + // Client routes to other clusters by overriding both properties in JDBC URL: + // hive.server2.tez.external.sessions.namespace=/ + // tez.am.registry.namespace=/ + // + // Path relationship (matches Docker template convention): + // tez.am.registry.namespace = / + // Tez registers session node at: // + // hive.server2.tez.external.sessions.namespace = / + String defaultRegistryNs = defaultLlapCluster(spec) + .map(llap -> "/" + llap.name()).orElse("/default"); + String defaultNamespace = ConfigUtils.TEZ_EXTERNAL_SESSIONS_ZK_PREFIX + defaultRegistryNs; props.put(ConfigUtils.HIVE_SERVER2_TEZ_EXTERNAL_SESSIONS_NAMESPACE_KEY, - "/tez-external-sessions/tez_am/server"); + defaultNamespace); props.put(ConfigUtils.HIVE_SERVER2_TEZ_EXTERNAL_SESSIONS_REGISTRY_CLASS_KEY, "org.apache.hadoop.hive.ql.exec.tez.ZookeeperExternalSessionsRegistryClient"); props.put(ConfigUtils.HIVE_ZOOKEEPER_QUORUM_KEY, zkQuorum); - // tez.am.framework.mode, tez.am.registry.namespace, tez.am.zookeeper.quorum - // are only in Tez 1.0.0+ props.put(ConfigUtils.TEZ_AM_FRAMEWORK_MODE_KEY, "STANDALONE_ZOOKEEPER"); - props.put(ConfigUtils.TEZ_AM_REGISTRY_NAMESPACE_KEY, "/tez_am/server"); + props.put(ConfigUtils.TEZ_AM_REGISTRY_NAMESPACE_KEY, defaultRegistryNs); props.put(ConfigUtils.TEZ_AM_ZOOKEEPER_QUORUM_KEY, zkQuorum); - LlapSpec llap = spec.llap(); - if (llap.isEnabled()) { + defaultLlapCluster(spec).ifPresent(llap -> { props.put(ConfigUtils.HIVE_EXECUTION_MODE_KEY, "llap"); props.put(ConfigUtils.HIVE_LLAP_EXECUTION_MODE_KEY, "all"); props.put(ConfigUtils.HIVE_LLAP_DAEMON_SERVICE_HOSTS_KEY, llap.serviceHosts()); - } + }); } else { props.put(ConfigUtils.HIVE_SERVER2_TEZ_USE_EXTERNAL_SESSIONS_KEY, "false"); props.put(ConfigUtils.TEZ_LOCAL_MODE_KEY, "true"); @@ -120,8 +130,13 @@ public static Map getHiveServer2HiveSite( return props; } - /** Builds tez-site.xml properties for HiveServer2 and TezAM. */ + /** Builds tez-site.xml properties for HiveServer2 (uses first LLAP cluster as default). */ public static Map getTezSite(HiveClusterSpec spec) { + return getTezSite(spec, defaultLlapCluster(spec).orElse(null)); + } + + /** Builds tez-site.xml properties for a specific LLAP cluster's TezAM. */ + public static Map getTezSite(HiveClusterSpec spec, LlapSpec llap) { boolean tezAmEnabled = spec.tezAm().isEnabled(); String zkQuorum = spec.zookeeper().quorum(); @@ -136,17 +151,26 @@ public static Map getTezSite(HiveClusterSpec spec) { if (tezAmEnabled) { tezProps.put(ConfigUtils.TEZ_LOCAL_MODE_KEY, "false"); tezProps.put(ConfigUtils.TEZ_AM_FRAMEWORK_MODE_KEY, "STANDALONE_ZOOKEEPER"); - tezProps.put(ConfigUtils.TEZ_AM_REGISTRY_NAMESPACE_KEY, "/tez_am/server"); + // Per-LLAP-cluster TezAM: each registers under its own ZK namespace. + // tez.am.registry.namespace is the path WITHIN Tez's "tez-external-sessions" + // Curator namespace (chroot). The absolute ZK path becomes: + // /tez-external-sessions// + String registryNamespace = llap != null + ? "/" + llap.name() : "/default"; + tezProps.put(ConfigUtils.TEZ_AM_REGISTRY_NAMESPACE_KEY, registryNamespace); } else { tezProps.put(ConfigUtils.TEZ_LOCAL_MODE_KEY, "true"); } - LlapSpec llap = spec.llap(); - if (llap.isEnabled()) { + if (llap != null) { tezProps.put(ConfigUtils.HIVE_LLAP_DAEMON_SERVICE_HOSTS_KEY, llap.serviceHosts()); } + // Required by LlapTaskCommunicator — Tez's Configuration doesn't get HiveConf defaults + tezProps.put(ConfigUtils.HIVE_LLAP_DAEMON_UMBILICAL_PORT_KEY, + ConfigUtils.HIVE_LLAP_DAEMON_UMBILICAL_PORT_DEFAULT); + if (spec.tezAm().configOverrides() != null) { tezProps.putAll(spec.tezAm().configOverrides()); } @@ -205,9 +229,8 @@ public static Map getMetastoreSite(HiveClusterSpec spec) { return props; } - /** Builds llap-daemon-site.xml properties. */ - public static Map getLlapDaemonSite(HiveClusterSpec spec) { - LlapSpec llap = spec.llap(); + /** Builds llap-daemon-site.xml properties for a specific LLAP cluster. */ + public static Map getLlapDaemonSite(HiveClusterSpec spec, LlapSpec llap) { Map props = new LinkedHashMap<>(); props.put(ConfigUtils.HIVE_LLAP_DAEMON_MEMORY_MB_KEY, @@ -224,4 +247,11 @@ public static Map getLlapDaemonSite(HiveClusterSpec spec) { } return props; } + + /** Returns the first enabled LLAP cluster (used as the default for HS2/TezAM config). */ + private static Optional defaultLlapCluster(HiveClusterSpec spec) { + return spec.llapClusters().stream() + .filter(LlapSpec::isEnabled) + .findFirst(); + } } diff --git a/packaging/src/kubernetes/src/java/org/apache/hive/kubernetes/operator/util/Labels.java b/packaging/src/kubernetes/src/java/org/apache/hive/kubernetes/operator/util/Labels.java index dcf0cc43b3c6..bbf4677e81c7 100644 --- a/packaging/src/kubernetes/src/java/org/apache/hive/kubernetes/operator/util/Labels.java +++ b/packaging/src/kubernetes/src/java/org/apache/hive/kubernetes/operator/util/Labels.java @@ -31,6 +31,7 @@ public final class Labels { public static final String APP_COMPONENT = "app.kubernetes.io/component"; public static final String MANAGED_BY = "app.kubernetes.io/managed-by"; public static final String MANAGED_BY_VALUE = "hive-kubernetes-operator"; + public static final String LLAP_CLUSTER = "hive.apache.org/llap-cluster"; private Labels() { } @@ -66,4 +67,53 @@ public static Map selectorForComponent(HiveCluster hc, selector.put(APP_COMPONENT, component); return selector; } + + /** + * Returns the full label set for a specific LLAP cluster instance. + * Includes the per-cluster discriminator label. + */ + public static Map forLlapCluster(HiveCluster hc, String llapName) { + Map labels = new LinkedHashMap<>(); + labels.put(APP_NAME, "apache-hive"); + labels.put(APP_INSTANCE, hc.getMetadata().getName()); + labels.put(APP_COMPONENT, ConfigUtils.COMPONENT_LLAP); + labels.put(MANAGED_BY, MANAGED_BY_VALUE); + labels.put(LLAP_CLUSTER, llapName); + return labels; + } + + /** + * Returns the selector labels for a specific LLAP cluster instance. + */ + public static Map selectorForLlapCluster(HiveCluster hc, String llapName) { + Map selector = new LinkedHashMap<>(); + selector.put(APP_INSTANCE, hc.getMetadata().getName()); + selector.put(APP_COMPONENT, ConfigUtils.COMPONENT_LLAP); + selector.put(LLAP_CLUSTER, llapName); + return selector; + } + + /** + * Returns the full label set for a per-LLAP-cluster TezAM instance. + */ + public static Map forTezAmCluster(HiveCluster hc, String llapName) { + Map labels = new LinkedHashMap<>(); + labels.put(APP_NAME, "apache-hive"); + labels.put(APP_INSTANCE, hc.getMetadata().getName()); + labels.put(APP_COMPONENT, ConfigUtils.COMPONENT_TEZAM); + labels.put(MANAGED_BY, MANAGED_BY_VALUE); + labels.put(LLAP_CLUSTER, llapName); + return labels; + } + + /** + * Returns the selector labels for a per-LLAP-cluster TezAM instance. + */ + public static Map selectorForTezAmCluster(HiveCluster hc, String llapName) { + Map selector = new LinkedHashMap<>(); + selector.put(APP_INSTANCE, hc.getMetadata().getName()); + selector.put(APP_COMPONENT, ConfigUtils.COMPONENT_TEZAM); + selector.put(LLAP_CLUSTER, llapName); + return selector; + } } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TezExternalSessionState.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TezExternalSessionState.java index b3103d3f5918..bd49d02e530f 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TezExternalSessionState.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TezExternalSessionState.java @@ -97,6 +97,12 @@ protected void openInternal(String[] additionalFilesNotFromConf, boolean llapMode = isLlapMode(); TezConfiguration tezConfig = new TezConfiguration(defaultTezConfiguration); + // Propagate per-session tez.am.registry.namespace override (set via JDBC URL) + // so the TezClient locates the correct AM for the target LLAP cluster. + String registryNs = conf.get("tez.am.registry.namespace"); + if (registryNs != null && !registryNs.isEmpty()) { + tezConfig.set("tez.am.registry.namespace", registryNs); + } setupSessionAcls(tezConfig, conf); ServicePluginsDescriptor spd = createServicePluginDescriptor(llapMode, tezConfig); Credentials llapCredentials = createLlapCredentials(llapMode, tezConfig); diff --git a/service/src/java/org/apache/hive/service/cli/session/SessionManager.java b/service/src/java/org/apache/hive/service/cli/session/SessionManager.java index 274df88390ef..56369e4fed5c 100644 --- a/service/src/java/org/apache/hive/service/cli/session/SessionManager.java +++ b/service/src/java/org/apache/hive/service/cli/session/SessionManager.java @@ -109,6 +109,11 @@ public class SessionManager extends CompositeService { private String sessionImplWithUGIclassName; private String sessionImplclassName; private CleanupService cleanupService; + // Tracks which LLAP target gauges have been lazily registered. + // Capped at MAX_LLAP_TARGET_GAUGES to prevent unbounded cardinality from + // malicious/misconfigured clients injecting random target names via JDBC URL. + private static final int MAX_LLAP_TARGET_GAUGES = 32; + private final java.util.Set registeredLlapTargetGauges = ConcurrentHashMap.newKeySet(); public SessionManager(HiveServer2 hiveServer2, boolean allowSessions) { super(SessionManager.class.getSimpleName()); @@ -209,6 +214,61 @@ public Integer getValue() { metrics.addRatio(MetricsConstant.HS2_AVG_ACTIVE_SESSION_TIME, activeSessionTime, activeSessionCnt); } + // Registers a per-LLAP-target session gauge on first use. + private void registerLlapTargetGaugeIfNeeded(HiveSession session) { + try { + String sanitized = extractLlapTarget(session.getSessionConf()); + if (sanitized == null) { + return; + } + if (registeredLlapTargetGauges.contains(sanitized)) { + return; // already registered + } + // Cap cardinality to prevent heap/scrape-timeout DoS from rogue clients + if (registeredLlapTargetGauges.size() >= MAX_LLAP_TARGET_GAUGES) { + LOG.warn("LLAP target gauge limit ({}) reached; ignoring new target: {}", + MAX_LLAP_TARGET_GAUGES, sanitized); + return; + } + if (!registeredLlapTargetGauges.add(sanitized)) { + return; // lost race with another thread + } + Metrics metrics = MetricsFactory.getInstance(); + if (metrics == null) { + return; + } + String metricName = MetricsConstant.HS2_LLAP_TARGET_SESSIONS + "_" + sanitized; + metrics.addGauge(metricName, new MetricsVariable() { + @Override + public Integer getValue() { + int count = 0; + for (HiveSession s : getSessions()) { + try { + String t = extractLlapTarget(s.getSessionConf()); + if (sanitized.equals(t)) { + count++; + } + } catch (Exception e) { + // Session may be closing concurrently + } + } + return count; + } + }); + LOG.info("Registered LLAP target session gauge: {}", metricName); + } catch (Exception e) { + LOG.debug("Could not register LLAP target gauge: {}", e.getMessage()); + } + } + + private static String extractLlapTarget(HiveConf conf) { + String target = conf.getVar(ConfVars.LLAP_DAEMON_SERVICE_HOSTS); + if (target != null && !target.isEmpty()) { + return target.startsWith("@") ? target.substring(1) : target; + } + return null; + } + private void initSessionImplClassName() { this.sessionImplclassName = hiveConf.getVar(ConfVars.HIVE_SESSION_IMPL_CLASSNAME); this.sessionImplWithUGIclassName = hiveConf.getVar(ConfVars.HIVE_SESSION_IMPL_WITH_UGI_CLASSNAME); @@ -527,6 +587,7 @@ public HiveSession createSession(SessionHandle sessionHandle, TProtocolVersion p } throw new HiveSQLException(FAIL_CLOSE_ERROR_MESSAGE); } + registerLlapTargetGaugeIfNeeded(session); LOG.info("Session opened, " + session.getSessionHandle() + ", current sessions:" + getOpenSessionCount()); return session; diff --git a/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/common/metrics/common/MetricsConstant.java b/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/common/metrics/common/MetricsConstant.java index cd75934b006a..4aff66de7883 100644 --- a/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/common/metrics/common/MetricsConstant.java +++ b/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/common/metrics/common/MetricsConstant.java @@ -57,6 +57,8 @@ public class MetricsConstant { public static final String HS2_OPEN_SESSIONS = "hs2_open_sessions"; public static final String HS2_ACTIVE_SESSIONS = "hs2_active_sessions"; public static final String HS2_ABANDONED_SESSIONS = "hs2_abandoned_sessions"; + // Per-LLAP-cluster session count: metric name is HS2_LLAP_TARGET_SESSIONS + "_" + sanitizedTarget + public static final String HS2_LLAP_TARGET_SESSIONS = "hs2_llap_target_sessions"; public static final String HS2_AVG_OPEN_SESSION_TIME = "hs2_avg_open_session_time"; public static final String HS2_AVG_ACTIVE_SESSION_TIME = "hs2_avg_active_session_time"; From 48290cd21c3e27632625e6b7638211696dc7e7b3 Mon Sep 17 00:00:00 2001 From: Ayush Saxena Date: Sat, 20 Jun 2026 19:17:35 +0530 Subject: [PATCH 2/2] Sonar Warnings --- .../dependent/LlapResourceBuilder.java | 17 ++++++++------ .../reconciler/HiveClusterReconciler.java | 4 ++-- .../hive/kubernetes/operator/util/Labels.java | 7 +++--- .../service/cli/session/SessionManager.java | 23 ++++++++----------- 4 files changed, 26 insertions(+), 25 deletions(-) diff --git a/packaging/src/kubernetes/src/java/org/apache/hive/kubernetes/operator/dependent/LlapResourceBuilder.java b/packaging/src/kubernetes/src/java/org/apache/hive/kubernetes/operator/dependent/LlapResourceBuilder.java index b51509a02d03..358844c507c5 100644 --- a/packaging/src/kubernetes/src/java/org/apache/hive/kubernetes/operator/dependent/LlapResourceBuilder.java +++ b/packaging/src/kubernetes/src/java/org/apache/hive/kubernetes/operator/dependent/LlapResourceBuilder.java @@ -61,6 +61,9 @@ public class LlapResourceBuilder extends HiveDependentResource { private static final LlapResourceBuilder INSTANCE = new LlapResourceBuilder(); + private static final String TEZAM_INFIX = "-tezam-"; + private static final String HIVE_CONFIG_VOLUME = "hive-config"; + private static final String LLAP_CONFIG_VOLUME = "llap-config"; LlapResourceBuilder() { super(StatefulSet.class); @@ -192,7 +195,7 @@ public static PodDisruptionBudget buildPdb(HiveCluster hc, LlapSpec llap) { /** TezAM StatefulSet name for a specific LLAP cluster. */ public static String tezAmResourceName(HiveCluster hc, LlapSpec llap) { - return hc.getMetadata().getName() + "-tezam-" + llap.name(); + return hc.getMetadata().getName() + TEZAM_INFIX + llap.name(); } /** TezAM ConfigMap name for a specific LLAP cluster. */ @@ -294,7 +297,7 @@ private StatefulSet doBuildTezAmStatefulSet(HiveCluster hiveCluster, LlapSpec ll List volumeMounts = new ArrayList<>(); volumeMounts.add(new io.fabric8.kubernetes.api.model.VolumeMountBuilder() - .withName("hive-config") + .withName(HIVE_CONFIG_VOLUME) .withMountPath(CONF_MOUNT_PATH).build()); volumeMounts.add(new io.fabric8.kubernetes.api.model.VolumeMountBuilder() .withName("scratch") @@ -305,7 +308,7 @@ private StatefulSet doBuildTezAmStatefulSet(HiveCluster hiveCluster, LlapSpec ll String hs2CmName = HiveConfigMapDependent.HiveServer2.resourceName(hiveCluster); String hadoopCmName = HiveConfigMapDependent.Hadoop.resourceName(hiveCluster); String tezAmCmName = tezAmConfigMapName(hiveCluster, llap); - volumes.add(buildProjectedConfigVolume("hive-config", hs2CmName, tezAmCmName, hadoopCmName)); + volumes.add(buildProjectedConfigVolume(HIVE_CONFIG_VOLUME, hs2CmName, tezAmCmName, hadoopCmName)); volumes.add(new io.fabric8.kubernetes.api.model.VolumeBuilder() .withName("scratch") .withNewPersistentVolumeClaim() @@ -317,7 +320,7 @@ private StatefulSet doBuildTezAmStatefulSet(HiveCluster hiveCluster, LlapSpec ll List initContainers = new ArrayList<>(); addExternalJars(spec.image(), spec.externalJars(), initContainers, volumeMounts, volumes, envVars); - replaceConfMountWithSubPaths(volumeMounts, "hive-config", + replaceConfMountWithSubPaths(volumeMounts, HIVE_CONFIG_VOLUME, "hive-site.xml", "tez-site.xml", "core-site.xml"); String configHash = sha256( @@ -429,18 +432,18 @@ private StatefulSet doBuildStatefulSet(HiveCluster hiveCluster, LlapSpec llap, I List volumeMounts = new ArrayList<>(); volumeMounts.add(new io.fabric8.kubernetes.api.model.VolumeMountBuilder() - .withName("llap-config") + .withName(LLAP_CONFIG_VOLUME) .withMountPath(CONF_MOUNT_PATH).build()); List volumes = new ArrayList<>(); String cmName = configMapName(hiveCluster, llap); String hadoopCmName = HiveConfigMapDependent.Hadoop.resourceName(hiveCluster); - volumes.add(buildProjectedConfigVolume("llap-config", cmName, hadoopCmName)); + volumes.add(buildProjectedConfigVolume(LLAP_CONFIG_VOLUME, cmName, hadoopCmName)); List initContainers = new ArrayList<>(); addExternalJars(spec.image(), spec.externalJars(), initContainers, volumeMounts, volumes, envVars); - replaceConfMountWithSubPaths(volumeMounts, "llap-config", + replaceConfMountWithSubPaths(volumeMounts, LLAP_CONFIG_VOLUME, "llap-daemon-site.xml", "core-site.xml"); AutoscalingSpec autoscaling = llap.autoscaling(); diff --git a/packaging/src/kubernetes/src/java/org/apache/hive/kubernetes/operator/reconciler/HiveClusterReconciler.java b/packaging/src/kubernetes/src/java/org/apache/hive/kubernetes/operator/reconciler/HiveClusterReconciler.java index 574f3ab4ee9d..769576616532 100644 --- a/packaging/src/kubernetes/src/java/org/apache/hive/kubernetes/operator/reconciler/HiveClusterReconciler.java +++ b/packaging/src/kubernetes/src/java/org/apache/hive/kubernetes/operator/reconciler/HiveClusterReconciler.java @@ -608,7 +608,7 @@ private void reconcileLlapClusters(HiveCluster resource, KubernetesClient client continue; } desiredNames.add(llapSpec.name()); - int replicas = resolveLlapReplicaCount(client, resource, llapSpec, ns, clusterName); + int replicas = resolveLlapReplicaCount(resource, llapSpec, ns, clusterName); // --- LLAP resources --- client.configMaps().inNamespace(ns) @@ -660,7 +660,7 @@ private void reconcileLlapClusters(HiveCluster resource, KubernetesClient client * Resolves the replica count for a LLAP cluster, respecting autoscaler-managed values * and suspend state. */ - private int resolveLlapReplicaCount(KubernetesClient client, HiveCluster resource, + private int resolveLlapReplicaCount(HiveCluster resource, LlapSpec llapSpec, String ns, String clusterName) { if (resource.getSpec().suspend()) { return 0; diff --git a/packaging/src/kubernetes/src/java/org/apache/hive/kubernetes/operator/util/Labels.java b/packaging/src/kubernetes/src/java/org/apache/hive/kubernetes/operator/util/Labels.java index bbf4677e81c7..965ccbbf115f 100644 --- a/packaging/src/kubernetes/src/java/org/apache/hive/kubernetes/operator/util/Labels.java +++ b/packaging/src/kubernetes/src/java/org/apache/hive/kubernetes/operator/util/Labels.java @@ -32,6 +32,7 @@ public final class Labels { public static final String MANAGED_BY = "app.kubernetes.io/managed-by"; public static final String MANAGED_BY_VALUE = "hive-kubernetes-operator"; public static final String LLAP_CLUSTER = "hive.apache.org/llap-cluster"; + private static final String APP_NAME_VALUE = "apache-hive"; private Labels() { } @@ -46,7 +47,7 @@ private Labels() { public static Map forComponent(HiveCluster hc, String component) { Map labels = new LinkedHashMap<>(); - labels.put(APP_NAME, "apache-hive"); + labels.put(APP_NAME, APP_NAME_VALUE); labels.put(APP_INSTANCE, hc.getMetadata().getName()); labels.put(APP_COMPONENT, component); labels.put(MANAGED_BY, MANAGED_BY_VALUE); @@ -74,7 +75,7 @@ public static Map selectorForComponent(HiveCluster hc, */ public static Map forLlapCluster(HiveCluster hc, String llapName) { Map labels = new LinkedHashMap<>(); - labels.put(APP_NAME, "apache-hive"); + labels.put(APP_NAME, APP_NAME_VALUE); labels.put(APP_INSTANCE, hc.getMetadata().getName()); labels.put(APP_COMPONENT, ConfigUtils.COMPONENT_LLAP); labels.put(MANAGED_BY, MANAGED_BY_VALUE); @@ -98,7 +99,7 @@ public static Map selectorForLlapCluster(HiveCluster hc, String */ public static Map forTezAmCluster(HiveCluster hc, String llapName) { Map labels = new LinkedHashMap<>(); - labels.put(APP_NAME, "apache-hive"); + labels.put(APP_NAME, APP_NAME_VALUE); labels.put(APP_INSTANCE, hc.getMetadata().getName()); labels.put(APP_COMPONENT, ConfigUtils.COMPONENT_TEZAM); labels.put(MANAGED_BY, MANAGED_BY_VALUE); diff --git a/service/src/java/org/apache/hive/service/cli/session/SessionManager.java b/service/src/java/org/apache/hive/service/cli/session/SessionManager.java index 56369e4fed5c..bc1f2de71bec 100644 --- a/service/src/java/org/apache/hive/service/cli/session/SessionManager.java +++ b/service/src/java/org/apache/hive/service/cli/session/SessionManager.java @@ -238,22 +238,19 @@ private void registerLlapTargetGaugeIfNeeded(HiveSession session) { return; } String metricName = MetricsConstant.HS2_LLAP_TARGET_SESSIONS + "_" + sanitized; - metrics.addGauge(metricName, new MetricsVariable() { - @Override - public Integer getValue() { - int count = 0; - for (HiveSession s : getSessions()) { - try { - String t = extractLlapTarget(s.getSessionConf()); - if (sanitized.equals(t)) { - count++; - } - } catch (Exception e) { - // Session may be closing concurrently + metrics.addGauge(metricName, () -> { + int count = 0; + for (HiveSession s : getSessions()) { + try { + String t = extractLlapTarget(s.getSessionConf()); + if (sanitized.equals(t)) { + count++; } + } catch (Exception e) { + // Session may be closing concurrently } - return count; } + return count; }); LOG.info("Registered LLAP target session gauge: {}", metricName); } catch (Exception e) {