diff --git a/doc/tutorial.rst b/doc/tutorial.rst index 9b7128c..971798a 100644 --- a/doc/tutorial.rst +++ b/doc/tutorial.rst @@ -497,6 +497,54 @@ When the tests are run with ``-vv`` then it will show both errors:: > +Server-side errors in the test report +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Calling ``check()`` (or any other similar method) raises the server-side +errors in the test but this must be done explicitly. If the test fails before +these errors would be raised (for example an assertion about the response +fails), the errors collected by the server remain unseen, although they could +explain the failure of the test. + +To make the debugging of such tests easier, the plugin automatically attaches +these errors to the report of the failed test. When a test fails, the pending +server-side assertions (such as the one added when the server received a +request which was not matching any of the registered handlers) and the errors +raised in the request handlers are shown as an extra section of the failure +output:: + + --------------------------- pytest-httpserver report --------------------------- + Server-side errors collected by pytest-httpserver (not raised during the test): + --- + Pending server-side assertions: + + 1. No handler found for request with data b''.Ordered matchers: + none + + Oneshot matchers: + none + + Persistent matchers: + > + + Note: these errors were not raised because the test did not call the check methods of the server (for example HTTPServer.check()). + +Errors which were already raised during the test (by calling the ``check()`` +methods) are not added to the report again. If the test fails in multiple +phases (such as setup and teardown), each error is attached to one report +only - errors collected after a failed phase are added to the report of the +next failed phase. The collected errors are also not leaked into the report +of the other tests. + +This works for every test which uses the ``httpserver``, ``httpserver_ipv4`` +or ``httpserver_ipv6`` fixture, no configuration is needed. If you use your +own server object in a custom fixture (for example a +:py:class:`pytest_httpserver.BlockingHTTPServer`), you can call the +``register_server()`` function of the ``pytest_httpserver.pytest_plugin`` +module (with the ``request`` fixture and the server object as parameters) to +get the same behavior. + + Logs ~~~~ diff --git a/pyproject.toml b/pyproject.toml index 9fb32ba..f1508fe 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -101,7 +101,6 @@ implicit_reexport = false [tool.black] line-length = 120 -safe = true [tool.ruff] lint.select = ["ALL"] diff --git a/pytest_httpserver/pytest_plugin.py b/pytest_httpserver/pytest_plugin.py index f19ec58..50565c5 100644 --- a/pytest_httpserver/pytest_plugin.py +++ b/pytest_httpserver/pytest_plugin.py @@ -6,11 +6,129 @@ import pytest from .httpserver import HTTPServer +from .httpserver import HTTPServerBase if TYPE_CHECKING: from collections.abc import Generator from ssl import SSLContext +REPORT_SECTION_TITLE = "pytest-httpserver report" + + +class _ServerRegistry: + """ + Registry of the servers used by a test. + + It keeps track of the server findings (assertions and handler errors): + + * which findings were already attached to a test report, so a finding is + never added to more than one report, even if the test fails in multiple + phases (setup, call and teardown), + + * a snapshot of the findings taken when the fixtures of the test are torn + down (see :py:meth:`snapshot`), so the findings collected during the + teardown phase can be attached to the teardown report even if the + server's state is cleared by another (for example session scoped) + fixture finalizer before the report is made. + """ + + def __init__(self) -> None: + self._servers: list[HTTPServerBase] = [] + self._reported_findings: list[object] = [] + self._snapshotted_findings: list[tuple[HTTPServerBase, str, object]] = [] + + def register(self, server: HTTPServerBase) -> bool: + """ + Registers the server. + + :param server: the server object + :return: ``True`` if the server was not registered before, ``False`` otherwise + """ + if server in self._servers: + return False + self._servers.append(server) + return True + + def snapshot(self, server: HTTPServerBase) -> None: + """ + Copies the not yet reported findings of the server to the registry. + + This is called from a finalizer of the fixture which registered the + server, so the findings collected during the teardown phase of the test + are copied before any other (typically session scoped) fixture + finalizer clears the state of the server. + + :param server: the server object + """ + for kind, finding in self._iter_findings(server): + if not self._is_reported(finding): + self._snapshotted_findings.append((server, kind, finding)) + + def _is_reported(self, finding: object) -> bool: + return any(finding is reported for reported in self._reported_findings) + + def _iter_findings(self, server: HTTPServerBase) -> Generator[tuple[str, object], None, None]: + for assertion in server.assertions: + yield ("assertion", assertion) + for handler_error in server.handler_errors: + yield ("error", handler_error) + + def collect_findings(self) -> list[str]: + """ + Returns the report lines of the findings which were not reported yet. + + The returned findings are marked as reported so subsequent calls won't + include them again. + + :return: list of the report lines, empty list if there's nothing to report + """ + + section_lines: list[str] = [] + + for server in self._servers: + server_lines: list[str] = [] + new_assertions: list[object] = [] + new_handler_errors: list[object] = [] + + candidates: list[tuple[str, object]] = list(self._iter_findings(server)) + candidates.extend( + (kind, finding) + for snapshotted_server, kind, finding in self._snapshotted_findings + if snapshotted_server is server + ) + + for kind, finding in candidates: + if self._is_reported(finding): + continue + if kind == "assertion": + new_assertions.append(finding) + else: + new_handler_errors.append(finding) + self._reported_findings.append(finding) + + if new_assertions: + server_lines.append("Pending server-side assertions:") + server_lines.append("") + for index, assertion in enumerate(new_assertions, start=1): + server_lines.append("{}. {}".format(index, assertion)) + server_lines.append("") + + if new_handler_errors: + server_lines.append("Errors raised by request handlers:") + server_lines.append("") + for index, error in enumerate(new_handler_errors, start=1): + server_lines.append("{}. {!r}".format(index, error)) + server_lines.append("") + + if server_lines: + section_lines.append("--- {}".format(server)) + section_lines.extend(server_lines) + + return section_lines + + +SERVERS_STASH_KEY = pytest.StashKey[_ServerRegistry]() + class Plugin: SERVER: PluginHTTPServer | None = None @@ -26,6 +144,41 @@ def stop(self) -> None: Plugin.SERVER = None +def register_server(request: pytest.FixtureRequest, server: HTTPServerBase) -> None: + """ + Registers the server for the test currently being set up. + + Registered servers are inspected when the test fails: pending server-side + assertions (such as the one added when the server received a request which + was not matching any of the registered handlers) and the errors collected + from the request handlers are attached to the test report to make the + debugging of the failed test easier. + + The fixtures provided by this plugin call this method automatically but it + can be also called from a custom fixture (with the ``request`` fixture as + the first parameter) to get the same report behavior for other server + objects, such as a :py:class:`pytest_httpserver.BlockingHTTPServer`. + + :param request: the ``request`` fixture (or a :py:class:`pytest.FixtureRequest` object) + :param server: the server object to register + """ + node = request.node + registry = node.stash.get(SERVERS_STASH_KEY, None) + if registry is None: + registry = _ServerRegistry() + node.stash[SERVERS_STASH_KEY] = registry + + def snapshot() -> None: + registry.snapshot(server) + + if registry.register(server): + # copy the findings of the server before the state of the server is + # cleared (which may happen in another fixture finalizer), so the + # findings collected during the teardown phase of the test can be + # attached to the report of the teardown phase + request.addfinalizer(snapshot) + + def get_httpserver_listen_address() -> tuple[str | None, int | None]: listen_host = os.environ.get("PYTEST_HTTPSERVER_HOST") listen_port_str = os.environ.get("PYTEST_HTTPSERVER_PORT") @@ -71,9 +224,10 @@ def pytest_sessionfinish(session: pytest.Session, exitstatus: int) -> None: # n @pytest.fixture -def httpserver(make_httpserver: HTTPServer) -> HTTPServer: +def httpserver(make_httpserver: HTTPServer, request: pytest.FixtureRequest) -> HTTPServer: server = make_httpserver server.clear() + register_server(request, server) return server @@ -90,9 +244,10 @@ def make_httpserver_ipv4( @pytest.fixture -def httpserver_ipv4(make_httpserver_ipv4: HTTPServer) -> HTTPServer: +def httpserver_ipv4(make_httpserver_ipv4: HTTPServer, request: pytest.FixtureRequest) -> HTTPServer: server = make_httpserver_ipv4 server.clear() + register_server(request, server) return server @@ -109,7 +264,60 @@ def make_httpserver_ipv6( @pytest.fixture -def httpserver_ipv6(make_httpserver_ipv6: HTTPServer) -> HTTPServer: +def httpserver_ipv6(make_httpserver_ipv6: HTTPServer, request: pytest.FixtureRequest) -> HTTPServer: server = make_httpserver_ipv6 server.clear() + register_server(request, server) return server + + +def _format_report_section(section_lines: list[str]) -> str: + section_lines.insert(0, "Server-side errors collected by pytest-httpserver (not raised during the test):") + section_lines.append( + "Note: these errors were not raised because the test did not call the check " + "methods of the server (for example HTTPServer.check())." + ) + return "\n".join(section_lines) + + +@pytest.hookimpl(wrapper=True) +def pytest_runtest_makereport( + item: pytest.Item, + call: pytest.CallInfo[None], +) -> Generator[None, object, object]: + """ + Adds the pending server-side errors to the report of the failed test. + + When a test fails, there can be errors collected by the httpserver(s) used + by the test which were not raised during the test run. This typically + happens when the server receives a request which does not match any of the + registered handlers (so the server has no clue how to serve it), and the + test does not call :py:meth:`pytest_httpserver.HTTPServer.check` (or + similar methods) to raise them. + + To make the debugging easier in these cases, the errors are attached to + the report of the failed test (as a report section), as they may explain + why the test failed. + + Each error is attached to one report only, so if the test fails in + multiple phases (such as setup and teardown), the errors are not + duplicated - errors collected after a failed phase are added to the + report of the next failed phase. + + The registry of the servers is removed from the test item when the report + of the teardown phase is made, so neither the servers nor their state + (such as the log of the requests) outlives the test. + """ + report = yield + if isinstance(report, pytest.TestReport): + registry = item.stash.get(SERVERS_STASH_KEY, None) + if registry is not None: + if report.failed: + section_lines = registry.collect_findings() + if section_lines: + report.sections.append((REPORT_SECTION_TITLE, _format_report_section(section_lines))) + if call.when == "teardown": + # this is the last report of the test, drop the registry to + # not keep the server objects (and their state) alive + del item.stash[SERVERS_STASH_KEY] + return report diff --git a/releasenotes/notes/makereport-hook-07d7b858a7915de4.yaml b/releasenotes/notes/makereport-hook-07d7b858a7915de4.yaml new file mode 100644 index 0000000..60414f1 --- /dev/null +++ b/releasenotes/notes/makereport-hook-07d7b858a7915de4.yaml @@ -0,0 +1,9 @@ +--- +features: + - | + The plugin now implements the ``pytest_runtest_makereport`` hook: when a + test fails, the server-side errors which were collected but not raised + during the test (such as the assertion added when the server received a + request which was not matching any of the registered handlers) are + attached to the test report as a new report section, making the debugging + of the failed test easier. diff --git a/tests/conftest.py b/tests/conftest.py index 9f9dfe5..6f9b1ce 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1,5 +1,7 @@ import pytest +pytest_plugins = ["pytester"] + def pytest_addoption(parser): parser.addoption("--ssl", action="store_true", default=False, help="run ssl tests") diff --git a/tests/test_makereport.py b/tests/test_makereport.py new file mode 100644 index 0000000..f9e9d95 --- /dev/null +++ b/tests/test_makereport.py @@ -0,0 +1,272 @@ +from __future__ import annotations + +from typing import TYPE_CHECKING + +# requests is imported (although not used directly here) so the inner pytest +# runs performed by the pytester fixture use the same module instance. +# Otherwise the sys.modules cleanup done by the pytester between the runs +# results in mixed module identities (causing weird errors in the requests +# library) +import requests # noqa: F401 + +if TYPE_CHECKING: + import pytest + +pytester_args = ("-p", "pytest_httpserver") + + +def test_unmatched_request_shown_in_failed_report(pytester: pytest.Pytester): + pytester.makepyfile(""" + import requests + + def test_unmatched(httpserver): + httpserver.expect_request("/expected").respond_with_data("OK") + requests.get(httpserver.url_for("/unexpected")) + assert False, "test failed" + """) + result = pytester.runpytest(*pytester_args) + result.assert_outcomes(failed=1) + result.stdout.fnmatch_lines( + [ + "*pytest-httpserver report*", + "*No handler found for request*unexpected*", + "*Persistent matchers:*", + "*uri='/expected'*", + ] + ) + + +def test_unmatched_request_not_shown_in_passed_report(pytester: pytest.Pytester): + pytester.makepyfile(""" + import requests + + def test_unmatched(httpserver): + requests.get(httpserver.url_for("/unexpected")) + """) + result = pytester.runpytest(*pytester_args) + result.assert_outcomes(passed=1) + result.stdout.no_fnmatch_line("*pytest-httpserver report*") + + +def test_no_section_when_no_pending_errors(pytester: pytest.Pytester): + pytester.makepyfile(""" + import requests + + def test_ok_request_but_failing_assertion(httpserver): + httpserver.expect_request("/expected").respond_with_data("OK") + requests.get(httpserver.url_for("/expected")) + assert False, "test failed" + """) + result = pytester.runpytest(*pytester_args) + result.assert_outcomes(failed=1) + result.stdout.no_fnmatch_line("*pytest-httpserver report*") + + +def test_checked_assertion_not_reported_twice(pytester: pytest.Pytester): + pytester.makepyfile(""" + import pytest + import requests + + def test_checked(httpserver): + requests.get(httpserver.url_for("/unexpected")) + with pytest.raises(AssertionError): + httpserver.check_assertions() + assert False, "test failed" + """) + result = pytester.runpytest(*pytester_args) + result.assert_outcomes(failed=1) + # the assertion was already raised (and shown in the traceback) so the + # report section must not contain it again + result.stdout.no_fnmatch_line("*pytest-httpserver report*") + + +def test_multiple_unmatched_requests_all_reported(pytester: pytest.Pytester): + pytester.makepyfile(""" + import requests + + def test_multiple_unmatched(httpserver): + requests.get(httpserver.url_for("/unexpected1")) + requests.get(httpserver.url_for("/unexpected2")) + assert False, "test failed" + """) + result = pytester.runpytest(*pytester_args) + result.assert_outcomes(failed=1) + result.stdout.fnmatch_lines( + [ + "*pytest-httpserver report*", + "*No handler found for request*unexpected1*", + "*No handler found for request*unexpected2*", + ] + ) + + +def test_handler_error_shown_in_failed_report(pytester: pytest.Pytester): + pytester.makepyfile(""" + import requests + + def test_handler_error(httpserver): + def handler(_): + raise ValueError("handler is broken") + + httpserver.expect_request("/error").respond_with_handler(handler) + requests.get(httpserver.url_for("/error")) + assert False, "test failed" + """) + result = pytester.runpytest(*pytester_args) + result.assert_outcomes(failed=1) + result.stdout.fnmatch_lines( + [ + "*pytest-httpserver report*", + "*Errors raised by request handlers:*", + "*ValueError('handler is broken')*", + ] + ) + + +def test_error_in_setup_shows_report(pytester: pytest.Pytester): + pytester.makepyfile(""" + import pytest + import requests + + @pytest.fixture + def client(httpserver): + requests.get(httpserver.url_for("/unexpected")) + raise RuntimeError("error in the fixture") + + def test_setup_error(client): + assert client == "client" + """) + result = pytester.runpytest(*pytester_args) + result.assert_outcomes(errors=1) + result.stdout.fnmatch_lines( + [ + "*pytest-httpserver report*", + "*No handler found for request*unexpected*", + ] + ) + + +def test_server_state_not_leaked_between_tests(pytester: pytest.Pytester): + pytester.makepyfile(""" + import requests + + def test_unmatched_but_passing(httpserver): + requests.get(httpserver.url_for("/unexpected")) + + def test_failing_without_httpserver(): + assert False, "test failed" + """) + result = pytester.runpytest(*pytester_args) + result.assert_outcomes(passed=1, failed=1) + # the second test does not use the httpserver fixture so the pending + # assertion of the first test must not leak into its report + result.stdout.fnmatch_lines(["*test_failing_without_httpserver*"]) + result.stdout.no_fnmatch_line("*pytest-httpserver report*") + + +def test_ipv4_server_reported(pytester: pytest.Pytester): + pytester.makepyfile(""" + import requests + + def test_unmatched(httpserver_ipv4): + requests.get(httpserver_ipv4.url_for("/unexpected")) + assert False, "test failed" + """) + result = pytester.runpytest(*pytester_args) + result.assert_outcomes(failed=1) + result.stdout.fnmatch_lines( + [ + "*pytest-httpserver report*", + "*No handler found for request*unexpected*", + ] + ) + + +def test_no_duplication_when_setup_and_teardown_fails(pytester: pytest.Pytester): + pytester.makepyfile(""" + import pytest + import requests + + @pytest.fixture + def client(httpserver): + requests.get(httpserver.url_for("/unexpected_setup")) + raise RuntimeError("error in setup") + + @pytest.fixture + def teardown_error(): + yield + raise RuntimeError("error in teardown") + + def test_setup_and_teardown_error(teardown_error, client): + pass + """) + result = pytester.runpytest(*pytester_args) + result.assert_outcomes(errors=2) + # the same pending assertion must be attached to one report only + title_lines = [line for line in result.stdout.lines if "pytest-httpserver report" in line] + assert len(title_lines) == 1 + result.stdout.fnmatch_lines( + [ + "*pytest-httpserver report*", + "*No handler found for request*unexpected_setup*", + ] + ) + + +def test_errors_made_in_teardown_reported_once(pytester: pytest.Pytester): + pytester.makepyfile(""" + import pytest + import requests + + @pytest.fixture + def client(httpserver): + yield "client" + requests.get(httpserver.url_for("/unexpected_teardown")) + raise RuntimeError("error in teardown") + + def test_teardown_error(client, httpserver): + requests.get(httpserver.url_for("/unexpected_call")) + assert False, "test failed" + """) + result = pytester.runpytest(*pytester_args) + result.assert_outcomes(failed=1, errors=1) + # each error is reported in the report of the phase where it was collected + title_lines = [line for line in result.stdout.lines if "pytest-httpserver report" in line] + assert len(title_lines) == 2 + result.stdout.fnmatch_lines( + [ + "*No handler found for request*unexpected_teardown*", + "*No handler found for request*unexpected_call*", + ] + ) + + +def test_registered_custom_server_reported(pytester: pytest.Pytester): + pytester.makepyfile(""" + import pytest + import requests + from pytest_httpserver import BlockingHTTPServer + from pytest_httpserver.pytest_plugin import register_server + + @pytest.fixture + def blocking_httpserver(request): + server = BlockingHTTPServer(timeout=1) + server.start() + register_server(request, server) + yield server + server.clear() + if server.is_running(): + server.stop() + + def test_registered(blocking_httpserver): + requests.get(blocking_httpserver.url_for("/unexpected"), timeout=5) + assert False, "test failed" + """) + result = pytester.runpytest(*pytester_args) + result.assert_outcomes(failed=1) + result.stdout.fnmatch_lines( + [ + "*pytest-httpserver report*", + "*No handler found for request*unexpected*", + ] + ) diff --git a/tests/test_release.py b/tests/test_release.py index 8643d11..aa56ddc 100644 --- a/tests/test_release.py +++ b/tests/test_release.py @@ -226,6 +226,7 @@ def test_sdist_contents(build: Build, version: str): "test_json_matcher.py", "test_log_leak.py", "test_log_querying.py", + "test_makereport.py", "test_mixed.py", "test_oneshot.py", "test_ordered.py",