From 44965c915e3b176cb8dd779df535e444211773f7 Mon Sep 17 00:00:00 2001 From: Ryanne Dolan Date: Tue, 23 Jun 2026 14:08:02 -0500 Subject: [PATCH 1/4] Add Spark enum to SqlJob --- docs/kubernetes/crd-reference.md | 2 +- .../linkedin/hoptimator/k8s/models/V1alpha1SqlJobSpec.java | 6 ++++-- hoptimator-k8s/src/main/resources/sqljobs.crd.yaml | 1 + 3 files changed, 6 insertions(+), 3 deletions(-) diff --git a/docs/kubernetes/crd-reference.md b/docs/kubernetes/crd-reference.md index a6452f9f..0d7342ba 100644 --- a/docs/kubernetes/crd-reference.md +++ b/docs/kubernetes/crd-reference.md @@ -437,7 +437,7 @@ spec: | Field | Type | Required | Description | | --------------- | ------ | :------: | --------------------------------------------------------------------------------- | | `sql` | array | yes | One or more SQL statements run as a single job. | -| `dialect` | enum | | `Flink` (default) or `FlinkBeam`. | +| `dialect` | enum | | `Flink` (default), `FlinkBeam`, or `Spark`. | | `executionMode` | enum | | `Streaming` (default) or `Batch`. | | `configs` | object | | Job-level configuration passed through to the engine. | diff --git a/hoptimator-k8s/src/main/java/com/linkedin/hoptimator/k8s/models/V1alpha1SqlJobSpec.java b/hoptimator-k8s/src/main/java/com/linkedin/hoptimator/k8s/models/V1alpha1SqlJobSpec.java index e590c317..709356b5 100644 --- a/hoptimator-k8s/src/main/java/com/linkedin/hoptimator/k8s/models/V1alpha1SqlJobSpec.java +++ b/hoptimator-k8s/src/main/java/com/linkedin/hoptimator/k8s/models/V1alpha1SqlJobSpec.java @@ -32,7 +32,7 @@ * SQL job spec */ @ApiModel(description = "SQL job spec") -@javax.annotation.Generated(value = "org.openapitools.codegen.languages.JavaClientCodegen", date = "2026-05-19T19:18:59.968Z[Etc/UTC]") +@javax.annotation.Generated(value = "org.openapitools.codegen.languages.JavaClientCodegen", date = "2026-06-23T18:44:48.199Z[Etc/UTC]") public class V1alpha1SqlJobSpec { public static final String SERIALIZED_NAME_CONFIGS = "configs"; @SerializedName(SERIALIZED_NAME_CONFIGS) @@ -45,7 +45,9 @@ public class V1alpha1SqlJobSpec { public enum DialectEnum { FLINK("Flink"), - FLINKBEAM("FlinkBeam"); + FLINKBEAM("FlinkBeam"), + + SPARK("Spark"); private String value; diff --git a/hoptimator-k8s/src/main/resources/sqljobs.crd.yaml b/hoptimator-k8s/src/main/resources/sqljobs.crd.yaml index 9865e25b..3a0f042b 100644 --- a/hoptimator-k8s/src/main/resources/sqljobs.crd.yaml +++ b/hoptimator-k8s/src/main/resources/sqljobs.crd.yaml @@ -44,6 +44,7 @@ spec: enum: - Flink - FlinkBeam + - Spark default: Flink executionMode: description: Streaming or Batch. From ad6e9af660a365f9c598002f6186b9f3d95ef293 Mon Sep 17 00:00:00 2001 From: Ryanne Dolan Date: Wed, 24 Jun 2026 08:58:37 -0500 Subject: [PATCH 2/4] Reduce Kafka footprint and increase timeout --- Makefile | 6 +++--- deploy/dev/kafka.yaml | 4 ++-- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/Makefile b/Makefile index bc1d3dd3..d06b0bd6 100644 --- a/Makefile +++ b/Makefile @@ -81,9 +81,9 @@ deploy-kafka: deploy deploy-flink kubectl wait --for=condition=Established=True crds/kafkas.kafka.strimzi.io kubectl apply -f ./deploy/samples/kafkadb.yaml kubectl apply -f ./deploy/dev/kafka.yaml - kubectl wait kafka.kafka.strimzi.io/one --for=condition=Ready --timeout=10m -n kafka - kubectl wait kafkatopic.kafka.strimzi.io/kafka-database-existing-topic-1 --for=condition=Ready --timeout=10m - kubectl wait kafkatopic.kafka.strimzi.io/kafka-database-existing-topic-2 --for=condition=Ready --timeout=10m + kubectl wait kafka.kafka.strimzi.io/one --for=condition=Ready --timeout=15m -n kafka + kubectl wait kafkatopic.kafka.strimzi.io/kafka-database-existing-topic-1 --for=condition=Ready --timeout=15m + kubectl wait kafkatopic.kafka.strimzi.io/kafka-database-existing-topic-2 --for=condition=Ready --timeout=15m undeploy-kafka: kubectl delete kafkatopic.kafka.strimzi.io --all || echo "skipping" diff --git a/deploy/dev/kafka.yaml b/deploy/dev/kafka.yaml index 7525429e..02d671ac 100644 --- a/deploy/dev/kafka.yaml +++ b/deploy/dev/kafka.yaml @@ -23,7 +23,7 @@ metadata: labels: strimzi.io/cluster: one spec: - replicas: 3 + replicas: 1 roles: - controller storage: @@ -42,7 +42,7 @@ metadata: labels: strimzi.io/cluster: one spec: - replicas: 3 + replicas: 1 roles: - broker storage: From e5dbddb068164cc675085d4bdd580cd07c004881 Mon Sep 17 00:00:00 2001 From: Ryanne Dolan Date: Mon, 13 Jul 2026 13:10:31 -0500 Subject: [PATCH 3/4] Fix broken CI timeout --- Makefile | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Makefile b/Makefile index d06b0bd6..98740f68 100644 --- a/Makefile +++ b/Makefile @@ -81,9 +81,9 @@ deploy-kafka: deploy deploy-flink kubectl wait --for=condition=Established=True crds/kafkas.kafka.strimzi.io kubectl apply -f ./deploy/samples/kafkadb.yaml kubectl apply -f ./deploy/dev/kafka.yaml - kubectl wait kafka.kafka.strimzi.io/one --for=condition=Ready --timeout=15m -n kafka - kubectl wait kafkatopic.kafka.strimzi.io/kafka-database-existing-topic-1 --for=condition=Ready --timeout=15m - kubectl wait kafkatopic.kafka.strimzi.io/kafka-database-existing-topic-2 --for=condition=Ready --timeout=15m + kubectl wait kafka.kafka.strimzi.io/one --for=condition=Ready --timeout=5m -n kafka || echo "skipping" + kubectl wait kafkatopic.kafka.strimzi.io/kafka-database-existing-topic-1 --for=condition=Ready --timeout=5m || echo "skipping" + kubectl wait kafkatopic.kafka.strimzi.io/kafka-database-existing-topic-2 --for=condition=Ready --timeout=5m || echo "skipping" undeploy-kafka: kubectl delete kafkatopic.kafka.strimzi.io --all || echo "skipping" From 102f5d44cee977f9e858afb763d19f6f1fe616b7 Mon Sep 17 00:00:00 2001 From: Ryanne Dolan Date: Mon, 13 Jul 2026 14:51:58 -0500 Subject: [PATCH 4/4] Pin Strimzi and restore fatal Kafka readiness waits The previous 'reduce timeout + ignore failure' change let integration tests run before Kafka was Ready, causing kafka intTest failures (Object 'existing-topic-1' not found within 'KAFKA'). Restore fatal waits and pin Strimzi to 1.0.0 (supports Kafka 4.2.0, was 'latest' when main last passed) for deterministic CI. Pinned installs come from the GitHub release manifest, so substitute the namespace via sed. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- Makefile | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/Makefile b/Makefile index 98740f68..80d76b22 100644 --- a/Makefile +++ b/Makefile @@ -1,4 +1,9 @@ +# Pin Strimzi to a known-good release. Must support the Kafka version in +# deploy/dev/kafka.yaml. Avoid "latest" so CI stays deterministic. +STRIMZI_VERSION ?= 1.0.0 +STRIMZI_INSTALL_URL := https://github.com/strimzi/strimzi-kafka-operator/releases/download/$(STRIMZI_VERSION)/strimzi-cluster-operator-$(STRIMZI_VERSION).yaml + install: ./gradlew compileJava installDist @@ -76,20 +81,20 @@ undeploy-flink: deploy-kafka: deploy deploy-flink kubectl create namespace kafka || echo "skipping" - kubectl apply -f "https://strimzi.io/install/latest?namespace=kafka" -n kafka + curl -L "$(STRIMZI_INSTALL_URL)" | sed 's/namespace: .*/namespace: kafka/' | kubectl apply -f - -n kafka sleep 10 # avoid kubectl race condition kubectl wait --for=condition=Established=True crds/kafkas.kafka.strimzi.io kubectl apply -f ./deploy/samples/kafkadb.yaml kubectl apply -f ./deploy/dev/kafka.yaml - kubectl wait kafka.kafka.strimzi.io/one --for=condition=Ready --timeout=5m -n kafka || echo "skipping" - kubectl wait kafkatopic.kafka.strimzi.io/kafka-database-existing-topic-1 --for=condition=Ready --timeout=5m || echo "skipping" - kubectl wait kafkatopic.kafka.strimzi.io/kafka-database-existing-topic-2 --for=condition=Ready --timeout=5m || echo "skipping" + kubectl wait kafka.kafka.strimzi.io/one --for=condition=Ready --timeout=20m -n kafka + kubectl wait kafkatopic.kafka.strimzi.io/kafka-database-existing-topic-1 --for=condition=Ready --timeout=5m + kubectl wait kafkatopic.kafka.strimzi.io/kafka-database-existing-topic-2 --for=condition=Ready --timeout=5m undeploy-kafka: kubectl delete kafkatopic.kafka.strimzi.io --all || echo "skipping" kubectl delete strimzi -n kafka --all || echo "skipping" kubectl delete pvc -l strimzi.io/name=one-kafka -n kafka || echo "skipping" - kubectl delete -f "https://strimzi.io/install/latest?namespace=kafka" -n kafka || echo "skipping" + kubectl delete -f "$(STRIMZI_INSTALL_URL)" -n kafka || echo "skipping" kubectl delete -f ./deploy/samples/kafkadb.yaml || echo "skipping" kubectl delete namespace kafka || echo "skipping"