diff --git a/CHANGELOG.md b/CHANGELOG.md index 769e6e44..439f212a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,7 @@ ([#862]). - Support for configuring which reverse proxies the webserver trusts `X-Forwarded-*` headers from, via `spec.webservers.roleConfig.trustedProxies` ([#835]). +- Added airflow `3.3.1` ([#865]). ### Changed @@ -42,6 +43,7 @@ variables set by the operator. In particular, `CONTAINERDEBUG_LOG_DIRECTORY` can now be overridden, whereas previously the operator's value always took precedence ([#838]). - Make operations infallible where appropriate ([#852], [#860]). +- Deprecated airflow `3.2.2` ([#865]). ### Fixed @@ -54,6 +56,10 @@ - The operator now watches all resources that it creates and early-exits the reconcile action when the cluster is marked for deletion ([#849]). +### Removed + +- Removed airflow `3.1.6` ([#865]). + [#814]: https://github.com/stackabletech/airflow-operator/pull/814 [#821]: https://github.com/stackabletech/airflow-operator/pull/821 [#827]: https://github.com/stackabletech/airflow-operator/pull/827 @@ -69,6 +75,7 @@ [#852]: https://github.com/stackabletech/airflow-operator/pull/852 [#860]: https://github.com/stackabletech/airflow-operator/pull/860 [#862]: https://github.com/stackabletech/airflow-operator/pull/862 +[#865]: https://github.com/stackabletech/airflow-operator/pull/865 ## [26.7.0] - 2026-07-21 diff --git a/docs/modules/airflow/examples/example-airflow-dag-bundles.yaml b/docs/modules/airflow/examples/example-airflow-dag-bundles.yaml index d462fa18..a123b59e 100644 --- a/docs/modules/airflow/examples/example-airflow-dag-bundles.yaml +++ b/docs/modules/airflow/examples/example-airflow-dag-bundles.yaml @@ -5,7 +5,7 @@ metadata: name: airflow-dag-bundles spec: image: - productVersion: 3.1.6 + productVersion: 3.3.1 clusterConfig: loadExamples: false exposeConfig: false @@ -24,7 +24,7 @@ spec: "classpath": "airflow.providers.git.bundles.git.GitDagBundle", "kwargs": { "git_conn_id": "repo1", - "tracking_ref": "3.1.6", + "tracking_ref": "3.3.1", "subdir": "airflow-core/src/airflow/example_dags" } }, @@ -33,7 +33,7 @@ spec: "classpath": "airflow.providers.git.bundles.git.GitDagBundle", "kwargs": { "git_conn_id": "repo2", - "tracking_ref": "3.1.6", + "tracking_ref": "3.3.1", "subdir": "airflow-core/src/airflow/example_dags" } } diff --git a/docs/modules/airflow/examples/example-airflow-dags-configmap.yaml b/docs/modules/airflow/examples/example-airflow-dags-configmap.yaml index c63da5c5..fbe0577c 100644 --- a/docs/modules/airflow/examples/example-airflow-dags-configmap.yaml +++ b/docs/modules/airflow/examples/example-airflow-dags-configmap.yaml @@ -5,7 +5,7 @@ metadata: name: airflow spec: image: - productVersion: 3.1.6 + productVersion: 3.3.1 clusterConfig: credentialsSecretName: airflow-admin-credentials metadataDatabase: diff --git a/docs/modules/airflow/examples/example-airflow-gitsync-https.yaml b/docs/modules/airflow/examples/example-airflow-gitsync-https.yaml index 965c061d..e38f2b0b 100644 --- a/docs/modules/airflow/examples/example-airflow-gitsync-https.yaml +++ b/docs/modules/airflow/examples/example-airflow-gitsync-https.yaml @@ -5,7 +5,7 @@ metadata: name: airflow spec: image: - productVersion: 3.1.6 + productVersion: 3.3.1 clusterConfig: loadExamples: false exposeConfig: false diff --git a/docs/modules/airflow/examples/example-airflow-incluster.yaml b/docs/modules/airflow/examples/example-airflow-incluster.yaml index 047fee95..d1f3d317 100644 --- a/docs/modules/airflow/examples/example-airflow-incluster.yaml +++ b/docs/modules/airflow/examples/example-airflow-incluster.yaml @@ -5,7 +5,7 @@ metadata: name: airflow spec: image: - productVersion: 3.1.6 + productVersion: 3.3.1 clusterConfig: loadExamples: false exposeConfig: false diff --git a/docs/modules/airflow/examples/example-airflow-kubernetes-executor-s3-logging.yaml b/docs/modules/airflow/examples/example-airflow-kubernetes-executor-s3-logging.yaml index 01ace48b..14a6f599 100644 --- a/docs/modules/airflow/examples/example-airflow-kubernetes-executor-s3-logging.yaml +++ b/docs/modules/airflow/examples/example-airflow-kubernetes-executor-s3-logging.yaml @@ -5,7 +5,7 @@ metadata: name: airflow spec: image: - productVersion: 3.1.6 + productVersion: 3.3.1 clusterConfig: {} webservers: roleConfig: diff --git a/docs/modules/airflow/examples/example-airflow-kubernetes-executor-s3-xcom.yaml b/docs/modules/airflow/examples/example-airflow-kubernetes-executor-s3-xcom.yaml index 48527d3c..8f0e938d 100644 --- a/docs/modules/airflow/examples/example-airflow-kubernetes-executor-s3-xcom.yaml +++ b/docs/modules/airflow/examples/example-airflow-kubernetes-executor-s3-xcom.yaml @@ -5,7 +5,7 @@ metadata: name: airflow spec: image: - productVersion: 3.1.6 + productVersion: 3.3.1 clusterConfig: {} webservers: roleConfig: diff --git a/docs/modules/airflow/examples/getting_started/code/airflow.yaml b/docs/modules/airflow/examples/getting_started/code/airflow.yaml index 100b80f6..d117d46b 100644 --- a/docs/modules/airflow/examples/getting_started/code/airflow.yaml +++ b/docs/modules/airflow/examples/getting_started/code/airflow.yaml @@ -5,7 +5,7 @@ metadata: name: airflow spec: image: - productVersion: 3.2.2 + productVersion: 3.3.1 pullPolicy: IfNotPresent clusterConfig: loadExamples: true diff --git a/docs/modules/airflow/pages/required-external-components.adoc b/docs/modules/airflow/pages/required-external-components.adoc index 24db19d9..778e30de 100644 --- a/docs/modules/airflow/pages/required-external-components.adoc +++ b/docs/modules/airflow/pages/required-external-components.adoc @@ -7,7 +7,7 @@ The {airflow-prerequisites}[Airflow documentation] specifies: Fully supported for production usage: -* PostgreSQL: 13, 14, 15, 16, 17 +* PostgreSQL: 14, 15, 16, 17, 18 NOTE: The SDP Airflow images do not bundle the MySQL provider for Airflow. If you need MySQL or MariaDB as an Airflow back-end, you will need to create a custom image to include this provider. diff --git a/docs/modules/airflow/pages/usage-guide/security.adoc b/docs/modules/airflow/pages/usage-guide/security.adoc index 9f4597ab..59226d0c 100644 --- a/docs/modules/airflow/pages/usage-guide/security.adoc +++ b/docs/modules/airflow/pages/usage-guide/security.adoc @@ -37,7 +37,7 @@ metadata: name: airflow-with-ldap spec: image: - productVersion: 3.1.6 + productVersion: 3.3.1 clusterConfig: authentication: - authenticationClass: ldap # <1> @@ -77,7 +77,7 @@ metadata: name: airflow-with-oidc spec: image: - productVersion: 3.1.6 + productVersion: 3.3.1 clusterConfig: authentication: - authenticationClass: keycloak # <1> diff --git a/docs/modules/airflow/partials/supported-versions.adoc b/docs/modules/airflow/partials/supported-versions.adoc index b3dbab54..7db8d8c8 100644 --- a/docs/modules/airflow/partials/supported-versions.adoc +++ b/docs/modules/airflow/partials/supported-versions.adoc @@ -2,7 +2,7 @@ // This is a separate file, since it is used by both the direct Airflow-Operator documentation, and the overarching // Stackable Platform documentation. -- 3.2.2 -- 3.1.6 (deprecated) +- 3.3.1 +- 3.2.2 (deprecated) - 3.0.6 (LTS) - 2.9.3 (deprecated) diff --git a/rust/operator-binary/src/controller/build/mod.rs b/rust/operator-binary/src/controller/build/mod.rs index e8b3410e..e9fef0fc 100644 --- a/rust/operator-binary/src/controller/build/mod.rs +++ b/rust/operator-binary/src/controller/build/mod.rs @@ -296,7 +296,7 @@ pub(crate) mod test_support { uid: e6ac237d-a6d4-43a1-8135-f36506110912 spec: image: - productVersion: 3.1.6 + productVersion: 3.3.1 clusterConfig: loadExamples: false exposeConfig: false @@ -557,7 +557,7 @@ mod tests { "airflow.stackable.tech_airflowcluster", ), ("app.kubernetes.io/name", "airflow"), - ("app.kubernetes.io/version", &app_version_label("3.1.6")), + ("app.kubernetes.io/version", &app_version_label("3.3.1")), ("stackable.tech/vendor", "Stackable"), ] .map(|(key, value)| (key.to_string(), value.to_string())), diff --git a/rust/operator-binary/src/controller/build/properties/env_vars.rs b/rust/operator-binary/src/controller/build/properties/env_vars.rs index 02361145..1e44ee62 100644 --- a/rust/operator-binary/src/controller/build/properties/env_vars.rs +++ b/rust/operator-binary/src/controller/build/properties/env_vars.rs @@ -540,7 +540,7 @@ mod tests { #[test] fn test_airflow_3_authorization_env_vars() { let authorization_config = get_test_authorization_config(); - let authorization_env_vars = authorization_env_vars(&authorization_config, "3.0.6"); + let authorization_env_vars = authorization_env_vars(&authorization_config, "3.3.1"); let authorization_env_vars = authorization_env_vars .into_iter() .map(|env| (env.name, env.value.expect("env var value must be present"))) diff --git a/rust/operator-binary/src/controller/build/properties/product_logging/mod.rs b/rust/operator-binary/src/controller/build/properties/product_logging/mod.rs index cb26be24..64661ed4 100644 --- a/rust/operator-binary/src/controller/build/properties/product_logging/mod.rs +++ b/rust/operator-binary/src/controller/build/properties/product_logging/mod.rs @@ -318,7 +318,7 @@ mod tests { create_airflow_stdlib_config( &log_config, "/stackable/log/airflow", - &resolved_image("3.0.6"), + &resolved_image("3.3.1"), ), create_airflow_structlog_config(&log_config, "/stackable/log/airflow"), ] { @@ -333,6 +333,8 @@ mod tests { fn test_logging_variant_selection() { // The stdlib config copies Airflow's default logging config, the structlog one // defines its own `mask_secrets_core` filter. + // N.B. This test assumes either Airflow 2.* or 3.0.* for stdlib_content as the + // code branches at this point. let log_config = ValidatedContainerLogConfigChoice::Automatic(AutomaticContainerLogConfig::default()); let stdlib_content = create_airflow_config( @@ -344,7 +346,7 @@ mod tests { let structlog_content = create_airflow_config( &log_config, "/stackable/log/airflow", - &resolved_image("3.1.6"), + &resolved_image("3.3.1"), ) .expect("automatic log config produces content"); assert!(stdlib_content.contains("deepcopy(airflow_local_settings.DEFAULT_LOGGING_CONFIG)")); diff --git a/rust/operator-binary/src/controller/build/properties/webserver_config.rs b/rust/operator-binary/src/controller/build/properties/webserver_config.rs index cc968a15..8a68d30c 100644 --- a/rust/operator-binary/src/controller/build/properties/webserver_config.rs +++ b/rust/operator-binary/src/controller/build/properties/webserver_config.rs @@ -410,7 +410,7 @@ mod tests { authorization::{AirflowAuthorizationResolved, OpaConfigResolved}, }; - const TEST_AIRFLOW_VERSION: &str = "3.0.6"; + const TEST_AIRFLOW_VERSION: &str = "3.3.1"; #[test] fn test_auth_db_config() { diff --git a/rust/operator-binary/src/controller/validate.rs b/rust/operator-binary/src/controller/validate.rs index 2d092c89..e58b495d 100644 --- a/rust/operator-binary/src/controller/validate.rs +++ b/rust/operator-binary/src/controller/validate.rs @@ -370,9 +370,9 @@ mod tests { fn test_resolved_product_image() -> stackable_operator::commons::product_image_selection::ResolvedProductImage { stackable_operator::commons::product_image_selection::ResolvedProductImage { - product_version: "3.0.6".to_string(), - app_version_label_value: "3.0.6".parse().expect("valid label value"), - image: "oci.example.org/sdp/airflow:3.0.6-stackable0.0.0-dev".to_string(), + product_version: "3.3.1".to_string(), + app_version_label_value: "3.3.1".parse().expect("valid label value"), + image: "oci.example.org/sdp/airflow:3.3.1-stackable0.0.0-dev".to_string(), image_pull_policy: "IfNotPresent".to_string(), pull_secrets: None, } @@ -386,7 +386,7 @@ mod tests { name: airflow spec: image: - productVersion: 3.1.6 + productVersion: 3.3.1 clusterConfig: loadExamples: false exposeConfig: false @@ -486,7 +486,7 @@ mod tests { name: airflow spec: image: - productVersion: 3.1.6 + productVersion: 3.3.1 clusterConfig: loadExamples: false exposeConfig: false @@ -567,7 +567,7 @@ mod tests { name: airflow spec: image: - productVersion: 3.1.6 + productVersion: 3.3.1 clusterConfig: loadExamples: false exposeConfig: false diff --git a/rust/operator-binary/src/crd/affinity.rs b/rust/operator-binary/src/crd/affinity.rs index 78cda1dc..aadcae90 100644 --- a/rust/operator-binary/src/crd/affinity.rs +++ b/rust/operator-binary/src/crd/affinity.rs @@ -73,7 +73,7 @@ mod tests { name: airflow spec: image: - productVersion: 3.1.6 + productVersion: 3.3.1 clusterConfig: credentialsSecretName: airflow-admin-credentials metadataDatabase: @@ -191,7 +191,7 @@ mod tests { name: airflow spec: image: - productVersion: 3.1.6 + productVersion: 3.3.1 clusterConfig: credentialsSecretName: airflow-admin-credentials metadataDatabase: diff --git a/rust/operator-binary/src/crd/mod.rs b/rust/operator-binary/src/crd/mod.rs index 846c643d..781ddf10 100644 --- a/rust/operator-binary/src/crd/mod.rs +++ b/rust/operator-binary/src/crd/mod.rs @@ -1094,7 +1094,7 @@ mod tests { name: airflow spec: image: - productVersion: 3.1.6 + productVersion: 3.3.1 clusterConfig: loadExamples: true exposeConfig: true @@ -1132,7 +1132,7 @@ mod tests { ) .expect("test: resolved product image is always valid"); - assert_eq!("3.1.6", &resolved_airflow_image.product_version); + assert_eq!("3.3.1", &resolved_airflow_image.product_version); assert_eq!( "KubernetesExecutor", @@ -1179,7 +1179,7 @@ mod tests { name: airflow spec: image: - productVersion: 3.2.2 + productVersion: 3.3.1 clusterConfig: credentialsSecretName: airflow-admin-credentials metadataDatabase: diff --git a/tests/templates/kuttl/commons/metrics.py b/tests/templates/kuttl/commons/metrics.py index fdc817c2..22c64c81 100755 --- a/tests/templates/kuttl/commons/metrics.py +++ b/tests/templates/kuttl/commons/metrics.py @@ -27,7 +27,7 @@ def assert_metric(role, role_group, metric): # Check if dag run state is "success", "queued", or "running" # TODO: in future, we could wait on it. -# See: https://airflow.apache.org/docs/apache-airflow/3.1.6/stable-rest-api-ref.html#operation/wait_dag_run_until_finished +# See: https://airflow.apache.org/docs/apache-airflow/3.3.1/stable-rest-api-ref.html def assert_dag_started(rest_url, headers, dag_id, dag_run_id): dag_run_response = requests.get( f"{rest_url}/dags/{dag_id}/dagRuns/{dag_run_id}", headers=headers diff --git a/tests/templates/kuttl/triggerer/triggerer_metrics.py b/tests/templates/kuttl/triggerer/triggerer_metrics.py index b3063640..dc8899d1 100755 --- a/tests/templates/kuttl/triggerer/triggerer_metrics.py +++ b/tests/templates/kuttl/triggerer/triggerer_metrics.py @@ -27,7 +27,7 @@ def assert_metric(role, role_group, metric): # Check if dag run state is "success", "queued", or "running" # TODO: in future, we could wait on it. -# See: https://airflow.apache.org/docs/apache-airflow/3.1.6/stable-rest-api-ref.html#operation/wait_dag_run_until_finished +# See: https://airflow.apache.org/docs/apache-airflow/3.3.1/stable-rest-api-ref.html def assert_dag_started(rest_url, headers, dag_id, dag_run_id): dag_run_response = requests.get( f"{rest_url}/dags/{dag_id}/dagRuns/{dag_run_id}", headers=headers diff --git a/tests/test-definition.yaml b/tests/test-definition.yaml index 52c8b1a1..5c6adb61 100644 --- a/tests/test-definition.yaml +++ b/tests/test-definition.yaml @@ -9,13 +9,13 @@ dimensions: values: - 2.9.3 - 3.0.6 - - 3.1.6 - 3.2.2 + - 3.3.1 # To use a custom image, add a comma and the full name after the product version # - x.x.x,oci.stackable.tech/sandbox/airflow:x.x.x-stackable0.0.0-dev - name: airflow-latest values: - - 3.2.2 + - 3.3.1 # To use a custom image, add a comma and the full name after the product version # - x.x.x,oci.stackable.tech/sandbox/airflow:x.x.x-stackable0.0.0-dev - name: opa-latest