Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
12 changes: 0 additions & 12 deletions airflow-core/tests/unit/always/test_example_dags.py
Original file line number Diff line number Diff line change
Expand Up @@ -70,18 +70,6 @@
# Ray uses pydantic v1 internally, which fails to infer types in Python 3.14.
# TODO: remove once ray releases a version with Python 3.14 support.
"providers/google/tests/system/google/cloud/ray/example_ray_job.py",
"providers/google/tests/system/google/cloud/dataflow/example_dataflow_go.py",
"providers/google/tests/system/google/cloud/dataflow/example_dataflow_java_streaming.py",
"providers/google/tests/system/google/cloud/dataflow/example_dataflow_native_java.py",
"providers/google/tests/system/google/cloud/dataflow/example_dataflow_native_python.py",
"providers/google/tests/system/google/cloud/dataflow/example_dataflow_native_python_async.py",
"providers/google/tests/system/google/cloud/dataflow/example_dataflow_pipeline.py",
"providers/google/tests/system/google/cloud/dataflow/example_dataflow_pipeline_streaming.py",
"providers/google/tests/system/google/cloud/dataflow/example_dataflow_sensors_deferrable.py",
"providers/google/tests/system/google/cloud/dataflow/example_dataflow_streaming_python.py",
"providers/google/tests/system/google/cloud/dataflow/example_dataflow_template.py",
"providers/google/tests/system/google/cloud/dataflow/example_dataflow_yaml.py",
"providers/google/tests/system/google/cloud/gcs/example_firestore.py",
)

LONGER_IMPORT_TIMEOUTS: dict[str, float] = {
Expand Down
22 changes: 11 additions & 11 deletions dev/breeze/doc/images/output_build-docs.svg
Loading
Sorry, something went wrong. Reload?
Sorry, we cannot display this file.
Sorry, this file is invalid so it cannot be displayed.
2 changes: 1 addition & 1 deletion dev/breeze/doc/images/output_build-docs.txt
Original file line number Diff line number Diff line change
@@ -1 +1 @@
de29604dc4d7031b9a26a1e0350d7806
c4788f285994a2bcc6bdf4c26b100921
Loading
Sorry, something went wrong. Reload?
Sorry, we cannot display this file.
Sorry, this file is invalid so it cannot be displayed.
Original file line number Diff line number Diff line change
@@ -1 +1 @@
4b44b7c52911adf1c1bdfb87e10e873b
2fe6904ce0c36330b00bd16b643da23d
Loading
Sorry, something went wrong. Reload?
Sorry, we cannot display this file.
Sorry, this file is invalid so it cannot be displayed.
Original file line number Diff line number Diff line change
@@ -1 +1 @@
57466e5cb9470b0d4398724fda2a1825
8a738bf76c1dd778b135492aa86f40ab
Loading
Sorry, something went wrong. Reload?
Sorry, we cannot display this file.
Sorry, this file is invalid so it cannot be displayed.
Original file line number Diff line number Diff line change
@@ -1 +1 @@
e2d5118e3339b53ce6e87dc6be6ae053
0837f7e405c27375919e1c33782d8b4d
Loading
Sorry, something went wrong. Reload?
Sorry, we cannot display this file.
Sorry, this file is invalid so it cannot be displayed.
Original file line number Diff line number Diff line change
@@ -1 +1 @@
3581abc65c55d4b8a096005c312df783
e4ed57cfecc4b27fdad8bcf0b0c2e862
Loading
Sorry, something went wrong. Reload?
Sorry, we cannot display this file.
Sorry, this file is invalid so it cannot be displayed.
Original file line number Diff line number Diff line change
@@ -1 +1 @@
0a45b898de7677eed8aa19c7e4f07345
fc31bc3a8190f254c85ecbcc71596117
22 changes: 11 additions & 11 deletions dev/breeze/doc/images/output_release-management_publish-docs.svg
Loading
Sorry, something went wrong. Reload?
Sorry, we cannot display this file.
Sorry, this file is invalid so it cannot be displayed.
Original file line number Diff line number Diff line change
@@ -1 +1 @@
30b0621c66436e215e0d34ec9bfd407b
5693f4a2ce03094c58480b2cdbc82080
22 changes: 11 additions & 11 deletions dev/breeze/doc/images/output_workflow-run_publish-docs.svg
Loading
Sorry, something went wrong. Reload?
Sorry, we cannot display this file.
Sorry, this file is invalid so it cannot be displayed.
Original file line number Diff line number Diff line change
@@ -1 +1 @@
01f2b301d8965951b49e2c8cd114700a
a70a0f1ed2b858ed43443aa3b024e8e5
4 changes: 2 additions & 2 deletions dev/breeze/tests/test_packages.py
Original file line number Diff line number Diff line change
Expand Up @@ -119,12 +119,12 @@ def test_get_removed_providers():

def test_get_suspended_provider_ids():
# Modify it every time we suspend/resume provider
assert get_suspended_provider_ids() == ["apache.beam"]
assert get_suspended_provider_ids() == []


def test_get_suspended_provider_folders():
# Modify it every time we suspend/resume provider
assert get_suspended_provider_folders() == ["apache/beam"]
assert get_suspended_provider_folders() == []


@pytest.mark.parametrize(
Expand Down
52 changes: 28 additions & 24 deletions dev/breeze/tests/test_selective_checks.py
Original file line number Diff line number Diff line change
Expand Up @@ -536,7 +536,7 @@ def assert_outputs_are_printed(expected_outputs: dict[str, str], stderr: str):
pytest.param(
("providers/apache/beam/tests/unit/apache/beam/file.py",),
{
"selected-providers-list-as-string": "common.compat google",
"selected-providers-list-as-string": "apache.beam common.compat google",
"all-python-versions": f"['{DEFAULT_PYTHON_MAJOR_MINOR_VERSION}']",
"all-python-versions-list-as-string": DEFAULT_PYTHON_MAJOR_MINOR_VERSION,
"python-versions": f"['{DEFAULT_PYTHON_MAJOR_MINOR_VERSION}']",
Expand All @@ -556,16 +556,17 @@ def assert_outputs_are_printed(expected_outputs: dict[str, str], stderr: str):
"providers-test-types-list-as-strings-in-json": json.dumps(
[
{
"description": "common.compat...google",
"test_types": "Providers[common.compat] Providers[google]",
"description": "apache.beam,c...google",
"test_types": "Providers[apache.beam,common.compat] Providers[google]",
}
]
),
"individual-providers-test-types-list-as-strings-in-json": json.dumps(
[
{
"description": "common.compat...google",
"test_types": "Providers[common.compat] Providers[google]",
"description": "apache.beam...google",
"test_types": "Providers[apache.beam] "
"Providers[common.compat] Providers[google]",
}
]
),
Expand All @@ -578,7 +579,7 @@ def assert_outputs_are_printed(expected_outputs: dict[str, str], stderr: str):
pytest.param(
("providers/apache/beam/tests/unit/apache/beam/file.py",),
{
"selected-providers-list-as-string": "common.compat google",
"selected-providers-list-as-string": "apache.beam common.compat google",
"all-python-versions": f"['{DEFAULT_PYTHON_MAJOR_MINOR_VERSION}']",
"all-python-versions-list-as-string": DEFAULT_PYTHON_MAJOR_MINOR_VERSION,
"python-versions": f"['{DEFAULT_PYTHON_MAJOR_MINOR_VERSION}']",
Expand All @@ -598,16 +599,17 @@ def assert_outputs_are_printed(expected_outputs: dict[str, str], stderr: str):
"providers-test-types-list-as-strings-in-json": json.dumps(
[
{
"description": "common.compat...google",
"test_types": "Providers[common.compat] Providers[google]",
"description": "apache.beam,c...google",
"test_types": "Providers[apache.beam,common.compat] Providers[google]",
}
]
),
"individual-providers-test-types-list-as-strings-in-json": json.dumps(
[
{
"description": "common.compat...google",
"test_types": "Providers[common.compat] Providers[google]",
"description": "apache.beam...google",
"test_types": "Providers[apache.beam] "
"Providers[common.compat] Providers[google]",
}
]
),
Expand All @@ -624,7 +626,7 @@ def assert_outputs_are_printed(expected_outputs: dict[str, str], stderr: str):
"providers/apache/beam/tests/unit/apache/beam/file.py",
),
{
"selected-providers-list-as-string": "common.compat google",
"selected-providers-list-as-string": "apache.beam common.compat google",
"all-python-versions": f"['{DEFAULT_PYTHON_MAJOR_MINOR_VERSION}']",
"all-python-versions-list-as-string": DEFAULT_PYTHON_MAJOR_MINOR_VERSION,
"python-versions": f"['{DEFAULT_PYTHON_MAJOR_MINOR_VERSION}']",
Expand All @@ -644,16 +646,17 @@ def assert_outputs_are_printed(expected_outputs: dict[str, str], stderr: str):
"providers-test-types-list-as-strings-in-json": json.dumps(
[
{
"description": "common.compat...google",
"test_types": "Providers[common.compat] Providers[google]",
"description": "apache.beam,c...google",
"test_types": "Providers[apache.beam,common.compat] Providers[google]",
}
]
),
"individual-providers-test-types-list-as-strings-in-json": json.dumps(
[
{
"description": "common.compat...google",
"test_types": "Providers[common.compat] Providers[google]",
"description": "apache.beam...google",
"test_types": "Providers[apache.beam] "
"Providers[common.compat] Providers[google]",
}
]
),
Expand All @@ -670,7 +673,7 @@ def assert_outputs_are_printed(expected_outputs: dict[str, str], stderr: str):
"providers/apache/beam/tests/unit/apache/beam/file.py",
),
{
"selected-providers-list-as-string": "common.compat google",
"selected-providers-list-as-string": "apache.beam common.compat google",
"all-python-versions": f"['{DEFAULT_PYTHON_MAJOR_MINOR_VERSION}']",
"all-python-versions-list-as-string": DEFAULT_PYTHON_MAJOR_MINOR_VERSION,
"python-versions": f"['{DEFAULT_PYTHON_MAJOR_MINOR_VERSION}']",
Expand All @@ -690,16 +693,17 @@ def assert_outputs_are_printed(expected_outputs: dict[str, str], stderr: str):
"providers-test-types-list-as-strings-in-json": json.dumps(
[
{
"description": "common.compat...google",
"test_types": "Providers[common.compat] Providers[google]",
"description": "apache.beam,c...google",
"test_types": "Providers[apache.beam,common.compat] Providers[google]",
}
]
),
"individual-providers-test-types-list-as-strings-in-json": json.dumps(
[
{
"description": "common.compat...google",
"test_types": "Providers[common.compat] Providers[google]",
"description": "apache.beam...google",
"test_types": "Providers[apache.beam] "
"Providers[common.compat] Providers[google]",
}
]
),
Expand Down Expand Up @@ -2418,7 +2422,7 @@ def test_expected_output_push(
"providers/google/tests/unit/google/file.py",
),
{
"selected-providers-list-as-string": "amazon apache.cassandra apache.kafka "
"selected-providers-list-as-string": "amazon apache.beam apache.cassandra apache.kafka "
"cncf.kubernetes common.compat common.messaging common.sql databricks "
"facebook google hashicorp http microsoft.azure microsoft.mssql mongo mysql "
"openlineage oracle postgres presto salesforce samba sftp ssh standard trino",
Expand All @@ -2430,7 +2434,7 @@ def test_expected_output_push(
"run-unit-tests": "true",
"skip-providers-tests": "false",
"docs-build": "true",
"docs-list-as-string": "apache-airflow helm-chart amazon apache.cassandra "
"docs-list-as-string": "apache-airflow helm-chart amazon apache.beam apache.cassandra "
"apache.kafka cncf.kubernetes common.compat common.messaging common.sql databricks facebook google hashicorp http microsoft.azure "
"microsoft.mssql mongo mysql openlineage oracle postgres "
"presto salesforce samba sftp ssh standard trino",
Expand All @@ -2453,7 +2457,7 @@ def test_expected_output_push(
[
{
"description": "amazon...standard",
"test_types": "Providers[amazon] Providers[apache.cassandra,"
"test_types": "Providers[amazon] Providers[apache.beam,apache.cassandra,"
"apache.kafka,cncf.kubernetes,common.compat,common.messaging,common.sql,databricks,facebook,"
"hashicorp,http,microsoft.azure,microsoft.mssql,mongo,mysql,"
"openlineage,oracle,postgres,presto,salesforce,samba,sftp,ssh,trino] "
Expand Down Expand Up @@ -2724,7 +2728,7 @@ def test_upgrade_to_newer_dependencies(
pytest.param(
("providers/google/docs/some_file.rst",),
{
"docs-list-as-string": "amazon apache.cassandra apache.kafka "
"docs-list-as-string": "amazon apache.beam apache.cassandra apache.kafka "
"cncf.kubernetes common.compat common.messaging common.sql databricks facebook google hashicorp http "
"microsoft.azure microsoft.mssql mongo mysql openlineage oracle "
"postgres presto salesforce samba sftp ssh standard trino",
Expand Down
2 changes: 1 addition & 1 deletion devel-common/src/tests_common/test_utils/providers.py
Original file line number Diff line number Diff line change
Expand Up @@ -68,7 +68,7 @@ def get_provider_min_airflow_version(provider_name: str) -> tuple[int, ...]:


# Ignore module import errors for any suspended provider paths used in example dags
IGNORE_MODULE_IMPORT_ERRORS: list[str] = ["airflow.providers.apache.beam"]
IGNORE_MODULE_IMPORT_ERRORS: list[str] = []


def get_suspended_providers_folders() -> list[str]:
Expand Down
3 changes: 2 additions & 1 deletion providers/apache/beam/README.rst
Original file line number Diff line number Diff line change
Expand Up @@ -55,7 +55,8 @@ PIP package Version required
========================================== ==================================================================
``apache-airflow`` ``>=2.11.0``
``apache-airflow-providers-common-compat`` ``>=1.12.0``
``apache-beam`` ``>=2.69.0``
``apache-beam`` ``>=2.72.0``
``betterproto`` ``==2.0.0b6; python_version >= "3.11"``
``pyarrow`` ``>=16.1.0; python_version < "3.14"``
``pyarrow`` ``>=22.0.0; python_version >= "3.14"``
``numpy`` ``>=1.22.4; python_version < "3.11"``
Expand Down
23 changes: 12 additions & 11 deletions providers/apache/beam/docs/index.rst
Original file line number Diff line number Diff line change
Expand Up @@ -100,16 +100,18 @@ PIP package Version required
========================================== ==================================================================
``apache-airflow`` ``>=2.11.0``
``apache-airflow-providers-common-compat`` ``>=1.12.0``
``apache-beam`` ``>=2.69.0``
``pyarrow`` ``>=16.1.0``
``apache-beam`` ``>=2.72.0``
``betterproto`` ``==2.0.0b6; python_version >= "3.11"``
``pyarrow`` ``>=16.1.0; python_version < "3.14"``
``pyarrow`` ``>=22.0.0; python_version >= "3.14"``
``numpy`` ``>=1.22.4; python_version < "3.11"``
``numpy`` ``>=1.23.2; python_version < "3.12" and python_version >= "3.11"``
``numpy`` ``>=1.26.0; python_version >= "3.12" and python_version < "3.14"``
``numpy`` ``>=2.4.3; python_version >= "3.14"``
========================================== ==================================================================

Cross provider package dependencies
-----------------------------------
Optional cross provider package dependencies
--------------------------------------------

Those are dependencies that might be needed in order to use all the features of the package.
You need to install the specified provider distributions in order to use them.
Expand All @@ -118,15 +120,14 @@ You can install such cross-provider dependencies when installing from PyPI. For

.. code-block:: bash

pip install apache-airflow-providers-apache-beam[common.compat]
pip install apache-airflow-providers-apache-beam[google]


================================================================================================================== =================
Dependent package Extra
================================================================================================================== =================
`apache-airflow-providers-common-compat <https://airflow.apache.org/docs/apache-airflow-providers-common-compat>`_ ``common.compat``
`apache-airflow-providers-google <https://airflow.apache.org/docs/apache-airflow-providers-google>`_ ``google``
================================================================================================================== =================
==================================================================================================== ==========
Dependent package Extra
==================================================================================================== ==========
`apache-airflow-providers-google <https://airflow.apache.org/docs/apache-airflow-providers-google>`_ ``google``
==================================================================================================== ==========

Downloading official packages
-----------------------------
Expand Down
4 changes: 2 additions & 2 deletions providers/apache/beam/provider.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -21,9 +21,9 @@ name: Apache Beam
description: |
`Apache Beam <https://beam.apache.org/>`__.

state: suspended
source-date-epoch: 1768334134
state: ready
lifecycle: production
source-date-epoch: 1772063928
# Note that those versions are maintained by release manager - do not update them manually
# with the exception of case where other provider in sources has >= new provider version.
# In such case adding >= NEW_VERSION and bumping to NEW_VERSION in a provider have
Expand Down
8 changes: 6 additions & 2 deletions providers/apache/beam/pyproject.toml
Original file line number Diff line number Diff line change
Expand Up @@ -61,7 +61,11 @@ requires-python = ">=3.10"
dependencies = [
"apache-airflow>=2.11.0",
"apache-airflow-providers-common-compat>=1.12.0",
"apache-beam>=2.69.0",
"apache-beam>=2.72.0",
# For Python>=3.11, the apache-beam package depends on envoy-data-plane.
# Currently, there is only one compatible version(1.0.3) of envoy-data-plane available for these
# Python versions, and it explicitly requires a prerelease version of betterproto.
"betterproto==2.0.0b6; python_version >= '3.11'",
Comment on lines +65 to +68

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I am not a fan of this.
this is a beta version from 2023. This is not a case where in a few days/weeks things will change and a stable version would be released, is it?

I am not in the details of this so I will leave @potiuk the call here but from my point of view we should not deliever stable release with depenedency on beta packages.
Who gurentees that tommorow the author won't remove them from PyPi with a thought that there won't be any production impact on anyone?

@potiuk potiuk Jun 24, 2026

Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Not a big fan either.

But I think thee newer version of envoy-data-plane has been questioned by others for example here: apache/beam#37854

And I see that this is what latest beam has:

envoy-data-plane<2,>=1.0.3; python_version >= "3.11"

And the 2.1.0 version of envoy-data-plane uses already betterproto2 - which is a maintained replacement of betterproto.

So maybe we can ask apache-beam to upgrade to the envoy-data-plane>=2 ?

Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I commented in the issue.

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I am not a big fan of this idea either.

The reason why I decided to add it was in that this beta version exists from 2023 and I am not sure that it will be release in the future. One more point for me to added it was that I noticed that Apache Beam SDK use this beta version in there generated requirements for containers. And I had idea that if they already had hardcoded it in apache-beam requirements, from which our provider already depends, maybe it makes sense temporary hardcoded it in provider as well for fixing constrains problem and unsuspend provider.

@eladkal eladkal Jun 25, 2026

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

We can't tell beam what to do from thier side but Airflow should not do this.

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Created this PR in beam to try and tackle this issue:
apache/beam#39213

Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This exact-prerelease pin is a workaround for the envoy-data-plane 1.0.3 constraint, so per the project's workaround convention it needs a GitHub tracking issue — linked by full URL here at the pin site (not a bare #NNNNN) and referenced in the PR body — capturing the follow-up: getting apache-beam onto envoy-data-plane>=2 / betterproto2 so this ==2.0.0b6 pin can be removed. Without that, an exact prerelease pin tends to quietly become permanent.

"pyarrow>=16.1.0; python_version < '3.14'",
"pyarrow>=22.0.0; python_version >= '3.14'",
"numpy>=1.22.4; python_version<'3.11'",
Expand All @@ -74,7 +78,7 @@ dependencies = [
# Any change in the dependencies is preserved when the file is regenerated
[project.optional-dependencies]
"google" = [
"apache-beam[gcp]>=2.69.0",
"apache-beam[gcp]>=2.72.0",
]

[dependency-groups]
Expand Down
1 change: 1 addition & 0 deletions providers/google/docs/index.rst
Original file line number Diff line number Diff line change
Expand Up @@ -209,6 +209,7 @@ You can install such cross-provider dependencies when installing from PyPI. For
Dependent package Extra
======================================================================================================================== ====================
`apache-airflow-providers-amazon <https://airflow.apache.org/docs/apache-airflow-providers-amazon>`_ ``amazon``
`apache-airflow-providers-apache-beam <https://airflow.apache.org/docs/apache-airflow-providers-apache-beam>`_ ``apache.beam``
`apache-airflow-providers-apache-cassandra <https://airflow.apache.org/docs/apache-airflow-providers-apache-cassandra>`_ ``apache.cassandra``
`apache-airflow-providers-cncf-kubernetes <https://airflow.apache.org/docs/apache-airflow-providers-cncf-kubernetes>`_ ``cncf.kubernetes``
`apache-airflow-providers-common-messaging <https://airflow.apache.org/docs/apache-airflow-providers-common-messaging>`_ ``common.messaging``
Expand Down
Loading
Loading