diff --git a/.github/workflows/ci-cd.yml b/.github/workflows/ci-cd.yml index ed72f2f5bf9..bc66ed702e8 100644 --- a/.github/workflows/ci-cd.yml +++ b/.github/workflows/ci-cd.yml @@ -322,7 +322,7 @@ jobs: AIOHTTP_NO_EXTENSIONS: ${{ matrix.no-extensions }} PIP_USER: 1 run: >- - pytest --junitxml=junit.xml --numprocesses=auto --cov -m 'not dev_mode and not autobahn' + pytest --junitxml=junit.xml -m 'not dev_mode and not autobahn' shell: bash - name: Re-run the failing tests with maximum verbosity if: failure() @@ -330,7 +330,7 @@ jobs: COLOR: yes AIOHTTP_NO_EXTENSIONS: ${{ matrix.no-extensions }} run: >- # `exit 1` makes sure that the job remains red with flaky runs - pytest --no-cov -vvvvv --lf && exit 1 + pytest --no-cov --numprocesses=0 -vvvvv --lf && exit 1 shell: bash - name: Run dev_mode tests env: @@ -338,7 +338,7 @@ jobs: AIOHTTP_NO_EXTENSIONS: ${{ matrix.no-extensions }} PIP_USER: 1 PYTHONDEVMODE: 1 - run: pytest -m dev_mode --cov --cov-append --cov-report=xml + run: pytest -m dev_mode --cov-append --cov-report=xml --numprocesses=0 shell: bash - name: Upload coverage uses: codecov/codecov-action@v7 @@ -422,9 +422,9 @@ jobs: CIBW_PLATFORM: ${{ matrix.config.platform }} CIBW_ARCHS: ${{ matrix.config.archs }} CIBW_TEST_REQUIRES: -r requirements/test-mobile.txt - CIBW_TEST_SOURCES: setup.cfg README.rst tests + CIBW_TEST_SOURCES: pytest.ini README.rst tests # Currently only Android supports colored output. See https://github.com/python/cpython/issues/150932 for iOS. - CIBW_TEST_COMMAND: python -m pytest ${{ matrix.config.platform == 'android' && '--color=yes' || '' }} + CIBW_TEST_COMMAND: python -m pytest --no-cov --numprocesses=0 ${{ matrix.config.platform == 'android' && '--color=yes' || '' }} SDIST_PATH: >- dist/${{ needs.build-pure-python-dists.outputs.sdist-filename }} @@ -473,8 +473,8 @@ jobs: PIP_USER: 1 run: >- PATH="${HOME}/Library/Python/3.11/bin:${HOME}/.local/bin:${PATH}" - pytest --junitxml=junit.xml --cov --cov-report=xml - --timeout=0 -m autobahn + pytest --junitxml=junit.xml --cov-report=xml --numprocesses=0 + --timeout=0 -o faulthandler_timeout=0 -m autobahn shell: bash - name: Upload coverage uses: codecov/codecov-action@v7 @@ -561,7 +561,9 @@ jobs: uses: CodSpeedHQ/action@v5.2.1 with: mode: instrumentation - run: python -Im pytest --no-cov -vvvvv --codspeed --durations=30 --timeout=0 + run: >- + python -Im pytest --no-cov --numprocesses=0 -vvvvv --codspeed + --durations=30 --timeout=0 -o faulthandler_timeout=0 cython-coverage: @@ -609,9 +611,8 @@ jobs: PIP_USER: 1 run: >- pytest tests/test_client_functional.py tests/test_http_parser.py tests/test_http_writer.py tests/test_web_functional.py tests/test_web_response.py tests/test_websocket_parser.py - --cov-config=.coveragerc-cython.toml --cov + --cov-config=.coveragerc-cython.toml --cov-report=xml:cython-coverage.xml - --numprocesses=auto -m 'not dev_mode and not autobahn' shell: bash - name: Upload coverage diff --git a/CHANGES/12620.contrib.rst b/CHANGES/12620.contrib.rst new file mode 100644 index 00000000000..62d0029e212 --- /dev/null +++ b/CHANGES/12620.contrib.rst @@ -0,0 +1,15 @@ +Moved the pytest configuration from :file:`setup.cfg` to a dedicated +:file:`pytest.ini` that follows the layout shared with ``propcache`` and +other ``aio-libs`` projects. Compared to the old configuration, +``minversion`` is raised from ``3.8.2`` to ``8.4``; ``pytest-xdist`` +(``--numprocesses=auto``) and ``pytest-cov`` (``--cov``, +``--cov-context=test``, ``--no-cov-on-fail``) are enabled by default +again, so pass ``--numprocesses=0`` and/or ``--no-cov`` to opt out, as +the :file:`Makefile` targets and CI jobs now do where needed; +``--doctest-modules``, ``--strict-markers`` and +``faulthandler_timeout = 30`` are enabled; ``-v`` is no longer added; +empty parameter sets are marked ``xfail`` instead of skipped; +``--junitxml`` reports use ``xunit1`` with captured output and +call-only durations; and ``norecursedirs`` skips more directories, +including :file:`tests/isolated/` +-- by :user:`aiolibsbot`. diff --git a/CHANGES/12621.contrib.rst b/CHANGES/12621.contrib.rst new file mode 120000 index 00000000000..105d10b5c30 --- /dev/null +++ b/CHANGES/12621.contrib.rst @@ -0,0 +1 @@ +12620.contrib.rst \ No newline at end of file diff --git a/MANIFEST.in b/MANIFEST.in index f4dbcb70d9b..c1a4cc91533 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -4,6 +4,7 @@ include README.rst include CONTRIBUTORS.txt include Makefile include .mypy.ini +include pytest.ini include .coveragerc.toml include .coveragerc-cython.toml graft aiohttp diff --git a/Makefile b/Makefile index 2ac94cb78ad..8ba209715e8 100644 --- a/Makefile +++ b/Makefile @@ -107,17 +107,17 @@ mypy: .PHONY: test test: .develop - @pytest -q + @pytest -q --no-cov --numprocesses=0 .PHONY: vtest vtest: .develop - @pytest -s -v - @python -X dev -m pytest --cov-append -s -v -m dev_mode + @pytest -s -v --numprocesses=0 + @python -X dev -m pytest --cov-append -s -v --numprocesses=0 -m dev_mode .PHONY: vvtest vvtest: .develop - @pytest -vv - @python -X dev -m pytest --cov-append -s -vv -m dev_mode + @pytest -vv --numprocesses=0 + @python -X dev -m pytest --cov-append -s -vv --numprocesses=0 -m dev_mode .PHONY: cov-dev cov-dev: .develop diff --git a/docs/contributing.rst b/docs/contributing.rst index 052d124c18f..8dd196a15c4 100644 --- a/docs/contributing.rst +++ b/docs/contributing.rst @@ -102,11 +102,12 @@ Install pre-commit hooks: .. warning:: - If you plan to use temporary ``print()``, ``pdb`` or ``ipdb`` within the test suite, execute it with ``-s``: + If you plan to use temporary ``print()``, ``pdb`` or ``ipdb`` within the test suite, execute it with ``-s`` + and without the parallel ``pytest-xdist`` workers that are enabled by default: .. code-block:: shell - $ pytest tests -s + $ pytest tests -s --numprocesses=0 in order to run the tests without output capturing. diff --git a/pytest.ini b/pytest.ini new file mode 100644 index 00000000000..314d4636657 --- /dev/null +++ b/pytest.ini @@ -0,0 +1,132 @@ +[pytest] +addopts = + # `pytest-xdist`: + --numprocesses=auto + + # Show 10 slowest invocations: + --durations=10 + + # Report all the things == -rxXs: + -ra + + # Show values of the local vars in errors/tracebacks: + --showlocals + + # Autocollect and invoke the doctests from all modules: + # https://docs.pytest.org/en/stable/doctest.html + --doctest-modules + + # Pre-load the `pytest-cov` plugin early: + -p pytest_cov + + # `pytest-cov`: + --cov + --cov-config=.coveragerc.toml + --cov-context=test + --no-cov-on-fail + + # Fail on config parsing warnings: + # --strict-config + + # Fail on non-existing markers: + # * Deprecated since v6.2.0 but may be reintroduced later covering a + # broader scope: + # --strict + # * Exists since v4.5.0 (advised to be used instead of `--strict`): + --strict-markers + + # Skip dev_mode/autobahn/internal tests by default; CI opts in explicitly: + -m "not dev_mode and not autobahn and not internal" + + # Disable entry-point auto-load, otherwise we miss coverage. + -p no:aiohttp + +asyncio_mode = auto + +doctest_optionflags = ALLOW_UNICODE ELLIPSIS + +# Marks tests with an empty parameterset as xfail(run=False) +empty_parameter_set_mark = xfail + +faulthandler_timeout = 30 + +filterwarnings = + error + + ignore:module 'ssl' has no attribute 'OP_NO_COMPRESSION'. The Python interpreter is compiled against OpenSSL < 1.0.0. Ref. https.//docs.python.org/3/library/ssl.html#ssl.OP_NO_COMPRESSION:UserWarning + ignore:Unclosed client session 2022.06.15`. + ignore:path is deprecated. Use files.. instead. Refer to https.//importlib-resources.readthedocs.io/en/latest/using.html#migrating-from-legacy for migration advice.:DeprecationWarning:certifi.core + # Dateutil deprecation warning already fixed upstream. + # Can be dropped with the next release, `dateutil > 2.8.2` + # https://github.com/dateutil/dateutil/pull/1285 + ignore:datetime.*utcfromtimestamp\(\) is deprecated and scheduled for removal:DeprecationWarning:dateutil.tz.tz + # Tracked upstream and waiting for PR review + # https://github.com/spulec/freezegun/issues/508 + # https://github.com/spulec/freezegun/pull/511 + ignore:datetime.*utcnow\(\) is deprecated and scheduled for removal:DeprecationWarning:freezegun.api + # coverage's C tracer is not available on iOS/Android (pure-Python fallback is used instead) + ignore:Couldn't import C tracer:coverage.exceptions.CoverageWarning + # Weird issue in Python 3.13+ triggered in test_multipart.py + ignore:coroutine method 'aclose' of 'BodyPartReader._decode_content_async' was never awaited:RuntimeWarning + # uvloop 0.22+ accesses the deprecated asyncio.AbstractEventLoopPolicy + # alias, which Python 3.14 marks for removal in 3.16. Drop this when + # uvloop stops touching the deprecated alias. + ignore:'asyncio.AbstractEventLoopPolicy' is deprecated:DeprecationWarning:uvloop + +# https://docs.pytest.org/en/stable/usage.html#creating-junitxml-format-files +junit_duration_report = call +# xunit1 contains more metadata than xunit2 so it's better for CI UIs: +junit_family = xunit1 +junit_logging = all +junit_log_passing_tests = true +junit_suite_name = aiohttp_test_suite + +# A mapping of markers to their descriptions allowed in strict mode: +markers = + autobahn: Autobahn testsuite. Should be run as a separate job. + dev_mode: mark test to run in dev mode. + internal: tests which may cause issues for packagers, but should be run in aiohttp's CI. + skip_blockbuster: mark test to skip the blockbuster fixture. + +minversion = 8.4 + +# Optimize pytest's lookup by restricting potentially deep dir tree scan: +norecursedirs = + build + dist + docs + requirements + vendor + venv + virtualenv + *.egg-info + .cache + .eggs + .git + .github + .tox + *.egg + # Scripts that `tests/test_leaks.py` runs in a subprocess; importing + # them for `--doctest-modules` would execute them: + tests/isolated + +testpaths = tests/ + +# 2-minute per-test timeout so a hung test surfaces by name instead of taking +# down the whole job. Autobahn and benchmark jobs override with `--timeout=0`. +timeout = 120 + +xfail_strict = true diff --git a/requirements/test-mobile.in b/requirements/test-mobile.in index 72238fd941e..39e0c793ece 100644 --- a/requirements/test-mobile.in +++ b/requirements/test-mobile.in @@ -6,3 +6,7 @@ backports-asyncio-runner; python_version < "3.11" # transitive dependency of "pytest-asyncio"; not installable on Python >= 3.11 using pip (uv works though) cffi; sys_platform != 'android' and sys_platform != 'ios' # transitive dependency of "pycares" pycares; sys_platform != 'android' and sys_platform != 'ios' # transitive dependency of "aiodns" + +# `pytest.ini` passes `--numprocesses`, so the plugin must be importable even +# though mobile runs disable it with `--numprocesses=0`: +pytest-xdist diff --git a/requirements/test-mobile.txt b/requirements/test-mobile.txt index 1574b90c8d4..deeeb405df5 100644 --- a/requirements/test-mobile.txt +++ b/requirements/test-mobile.txt @@ -40,6 +40,8 @@ exceptiongroup==1.3.1 # via # aiofastnet # pytest +execnet==2.1.2 + # via pytest-xdist freezegun==1.5.5 # via -r requirements/test-common-base.in frozenlist==1.8.0 @@ -89,6 +91,7 @@ pytest==9.1.1 # pytest-cov # pytest-mock # pytest-timeout + # pytest-xdist pytest-aiohttp==1.1.1 # via -r requirements/test-common-base.in pytest-asyncio==1.4.0 @@ -99,6 +102,8 @@ pytest-mock==3.15.1 # via -r requirements/test-common-base.in pytest-timeout==2.4.0 # via -r requirements/test-common-base.in +pytest-xdist==3.8.0 + # via -r requirements/test-mobile.in python-dateutil==2.9.0.post0 # via freezegun setuptools-git==1.2 diff --git a/setup.cfg b/setup.cfg index 2a24abcb0cc..dbbb1a3051b 100644 --- a/setup.cfg +++ b/setup.cfg @@ -33,70 +33,3 @@ combine_as_imports=True known_third_party=jinja2,pytest,multidict,yarl,gunicorn,freezegun known_first_party=aiohttp,aiohttp_jinja2,aiopg - -[tool:pytest] -addopts = - # show 10 slowest invocations: - --durations=10 - - # a bit of verbosity doesn't hurt: - -v - - # report all the things == -rxXs: - -ra - - # show values of the local vars in errors: - --showlocals - - -m "not dev_mode and not autobahn and not internal" - - # Disable entry-point auto-load, otherwise we miss coverage. - -p no:aiohttp -asyncio_mode = auto -# 2-minute per-test timeout so a hung test surfaces by name instead of taking -# down the whole job. Autobahn and benchmark jobs override with `--timeout=0`. -timeout = 120 -filterwarnings = - error - ignore:module 'ssl' has no attribute 'OP_NO_COMPRESSION'. The Python interpreter is compiled against OpenSSL < 1.0.0. Ref. https.//docs.python.org/3/library/ssl.html#ssl.OP_NO_COMPRESSION:UserWarning - ignore:Unclosed client session 2022.06.15`. - ignore:path is deprecated. Use files.. instead. Refer to https.//importlib-resources.readthedocs.io/en/latest/using.html#migrating-from-legacy for migration advice.:DeprecationWarning:certifi.core - # Dateutil deprecation warning already fixed upstream. - # Can be dropped with the next release, `dateutil > 2.8.2` - # https://github.com/dateutil/dateutil/pull/1285 - ignore:datetime.*utcfromtimestamp\(\) is deprecated and scheduled for removal:DeprecationWarning:dateutil.tz.tz - # Tracked upstream and waiting for PR review - # https://github.com/spulec/freezegun/issues/508 - # https://github.com/spulec/freezegun/pull/511 - ignore:datetime.*utcnow\(\) is deprecated and scheduled for removal:DeprecationWarning:freezegun.api - # coverage's C tracer is not available on iOS/Android (pure-Python fallback is used instead) - ignore:Couldn't import C tracer:coverage.exceptions.CoverageWarning - # Weird issue in Python 3.13+ triggered in test_multipart.py - ignore:coroutine method 'aclose' of 'BodyPartReader._decode_content_async' was never awaited:RuntimeWarning - # uvloop 0.22+ accesses the deprecated asyncio.AbstractEventLoopPolicy - # alias, which Python 3.14 marks for removal in 3.16. Drop this when - # uvloop stops touching the deprecated alias. - ignore:'asyncio.AbstractEventLoopPolicy' is deprecated:DeprecationWarning:uvloop -junit_suite_name = aiohttp_test_suite -norecursedirs = dist docs build .tox .eggs -minversion = 3.8.2 -testpaths = tests/ -xfail_strict = true -markers = - autobahn: Autobahn testsuite. Should be run as a separate job. - dev_mode: mark test to run in dev mode. - internal: tests which may cause issues for packagers, but should be run in aiohttp's CI. - skip_blockbuster: mark test to skip the blockbuster fixture. diff --git a/tools/testing/entrypoint.sh b/tools/testing/entrypoint.sh index e2a763db2e1..0c68cfedc9a 100644 --- a/tools/testing/entrypoint.sh +++ b/tools/testing/entrypoint.sh @@ -2,4 +2,4 @@ [[ "$AIOHTTP_NO_EXTENSIONS" != "y" ]] && make cythonize -python -m pytest -qx --no-cov $1 +python -m pytest -qx --no-cov --numprocesses=0 $1