Skip to content

Commit a68b704

Browse files
committed
Add server-side errors to pytest report
1 parent 6b3cef7 commit a68b704

6 files changed

Lines changed: 378 additions & 3 deletions

File tree

‎doc/tutorial.rst‎

Lines changed: 45 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -497,6 +497,51 @@ When the tests are run with ``-vv`` then it will show both errors::
497497
<RequestMatcher uri='/bar' method='__ALL' query_string=None headers={} data=None json=<UNDEFINED>>
498498

499499

500+
Server-side errors in the test report
501+
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
502+
503+
Calling ``check()`` (or any other similar method) raises the server-side
504+
errors in the test but this must be done explicitly. If the test fails before
505+
these errors would be raised (for example an assertion about the response
506+
fails), the errors collected by the server remain unseen, although they could
507+
explain the failure of the test.
508+
509+
To make the debugging of such tests easier, the plugin automatically attaches
510+
these errors to the report of the failed test. When a test fails, the pending
511+
server-side assertions (such as the one added when the server received a
512+
request which was not matching any of the registered handlers) and the errors
513+
raised in the request handlers are shown as an extra section of the failure
514+
output::
515+
516+
--------------------------- pytest-httpserver report ---------------------------
517+
Server-side errors collected by pytest-httpserver (not raised during the test):
518+
--- <HTTPServer host=localhost port=37425>
519+
Pending server-side assertions:
520+
521+
1. No handler found for request <Request 'http://localhost:37425/foobar' [GET]> with data b''.Ordered matchers:
522+
none
523+
524+
Oneshot matchers:
525+
none
526+
527+
Persistent matchers:
528+
<RequestMatcher uri='/foo' method='__ALL' query_string=None headers={} data=None json=<UNDEFINED>>
529+
530+
Note: these errors were not raised because the test did not call the check methods of the server (for example HTTPServer.check()).
531+
532+
Errors which were already raised during the test (by calling the ``check()``
533+
methods) are not added to the report again, and the errors collected during a
534+
test are not leaked into the report of the other tests.
535+
536+
This works for every test which uses the ``httpserver``, ``httpserver_ipv4``
537+
or ``httpserver_ipv6`` fixture, no configuration is needed. If you use your
538+
own server object in a custom fixture (for example a
539+
:py:class:`pytest_httpserver.BlockingHTTPServer`), you can call the
540+
``register_server()`` function of the ``pytest_httpserver.pytest_plugin``
541+
module (with the ``request`` fixture and the server object as parameters) to
542+
get the same behavior.
543+
544+
500545
Logs
501546
~~~~
502547

‎pytest_httpserver/pytest_plugin.py‎

Lines changed: 108 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -6,11 +6,16 @@
66
import pytest
77

88
from .httpserver import HTTPServer
9+
from .httpserver import HTTPServerBase
910

1011
if TYPE_CHECKING:
1112
from collections.abc import Generator
1213
from ssl import SSLContext
1314

15+
REPORT_SECTION_TITLE = "pytest-httpserver report"
16+
17+
SERVERS_STASH_KEY = pytest.StashKey[list[HTTPServerBase]]()
18+
1419

1520
class Plugin:
1621
SERVER: PluginHTTPServer | None = None
@@ -26,6 +31,33 @@ def stop(self) -> None:
2631
Plugin.SERVER = None
2732

2833

34+
def register_server(request: pytest.FixtureRequest, server: HTTPServerBase) -> None:
35+
"""
36+
Registers the server for the test currently being set up.
37+
38+
Registered servers are inspected when the test fails: pending server-side
39+
assertions (such as the one added when the server received a request which
40+
was not matching any of the registered handlers) and the errors collected
41+
from the request handlers are attached to the test report to make the
42+
debugging of the failed test easier.
43+
44+
The fixtures provided by this plugin call this method automatically but it
45+
can be also called from a custom fixture (with the ``request`` fixture as
46+
the first parameter) to get the same report behavior for other server
47+
objects, such as a :py:class:`pytest_httpserver.BlockingHTTPServer`.
48+
49+
:param request: the ``request`` fixture (or a :py:class:`pytest.FixtureRequest` object)
50+
:param server: the server object to register
51+
"""
52+
node = request.node
53+
servers = node.stash.get(SERVERS_STASH_KEY, None)
54+
if servers is None:
55+
servers = []
56+
node.stash[SERVERS_STASH_KEY] = servers
57+
if server not in servers:
58+
servers.append(server)
59+
60+
2961
def get_httpserver_listen_address() -> tuple[str | None, int | None]:
3062
listen_host = os.environ.get("PYTEST_HTTPSERVER_HOST")
3163
listen_port_str = os.environ.get("PYTEST_HTTPSERVER_PORT")
@@ -71,9 +103,10 @@ def pytest_sessionfinish(session: pytest.Session, exitstatus: int) -> None: # n
71103

72104

73105
@pytest.fixture
74-
def httpserver(make_httpserver: HTTPServer) -> HTTPServer:
106+
def httpserver(make_httpserver: HTTPServer, request: pytest.FixtureRequest) -> HTTPServer:
75107
server = make_httpserver
76108
server.clear()
109+
register_server(request, server)
77110
return server
78111

79112

@@ -90,9 +123,10 @@ def make_httpserver_ipv4(
90123

91124

92125
@pytest.fixture
93-
def httpserver_ipv4(make_httpserver_ipv4: HTTPServer) -> HTTPServer:
126+
def httpserver_ipv4(make_httpserver_ipv4: HTTPServer, request: pytest.FixtureRequest) -> HTTPServer:
94127
server = make_httpserver_ipv4
95128
server.clear()
129+
register_server(request, server)
96130
return server
97131

98132

@@ -109,7 +143,78 @@ def make_httpserver_ipv6(
109143

110144

111145
@pytest.fixture
112-
def httpserver_ipv6(make_httpserver_ipv6: HTTPServer) -> HTTPServer:
146+
def httpserver_ipv6(make_httpserver_ipv6: HTTPServer, request: pytest.FixtureRequest) -> HTTPServer:
113147
server = make_httpserver_ipv6
114148
server.clear()
149+
register_server(request, server)
115150
return server
151+
152+
153+
def _format_server_findings(server: HTTPServerBase) -> list[str]:
154+
"""
155+
Formats the pending (not yet raised) errors of the specified server.
156+
157+
:param server: the server object
158+
:return: list of the report lines, empty list if there's nothing to report
159+
"""
160+
161+
lines: list[str] = []
162+
163+
assertions = list(server.assertions)
164+
if assertions:
165+
lines.append("Pending server-side assertions:")
166+
lines.append("")
167+
for index, assertion in enumerate(assertions, start=1):
168+
lines.append("{}. {}".format(index, assertion))
169+
lines.append("")
170+
171+
handler_errors = list(server.handler_errors)
172+
if handler_errors:
173+
lines.append("Errors raised by request handlers:")
174+
lines.append("")
175+
for index, error in enumerate(handler_errors, start=1):
176+
lines.append("{}. {!r}".format(index, error))
177+
lines.append("")
178+
179+
return lines
180+
181+
182+
@pytest.hookimpl(wrapper=True)
183+
def pytest_runtest_makereport(
184+
item: pytest.Item,
185+
call: pytest.CallInfo[None], # noqa: ARG001
186+
) -> Generator[None, object, object]:
187+
"""
188+
Adds the pending server-side errors to the report of the failed test.
189+
190+
When a test fails, there can be errors collected by the httpserver(s) used
191+
by the test which were not raised during the test run. This typically
192+
happens when the server receives a request which does not match any of the
193+
registered handlers (so the server has no clue how to serve it), and the
194+
test does not call :py:meth:`pytest_httpserver.HTTPServer.check` (or
195+
similar methods) to raise them.
196+
197+
To make the debugging easier in these cases, the errors are attached to
198+
the report of the failed test (as a report section), as they may explain
199+
why the test failed.
200+
"""
201+
report = yield
202+
if isinstance(report, pytest.TestReport) and report.failed:
203+
servers = item.stash.get(SERVERS_STASH_KEY, None)
204+
if servers:
205+
section_lines: list[str] = []
206+
for server in servers:
207+
server_lines = _format_server_findings(server)
208+
if server_lines:
209+
section_lines.append("--- {}".format(server))
210+
section_lines.extend(server_lines)
211+
if section_lines:
212+
section_lines.insert(
213+
0, "Server-side errors collected by pytest-httpserver (not raised during the test):"
214+
)
215+
section_lines.append(
216+
"Note: these errors were not raised because the test did not call the check "
217+
"methods of the server (for example HTTPServer.check())."
218+
)
219+
report.sections.append((REPORT_SECTION_TITLE, "\n".join(section_lines)))
220+
return report
Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,9 @@
1+
---
2+
features:
3+
- |
4+
The plugin now implements the ``pytest_runtest_makereport`` hook: when a
5+
test fails, the server-side errors which were collected but not raised
6+
during the test (such as the assertion added when the server received a
7+
request which was not matching any of the registered handlers) are
8+
attached to the test report as a new report section, making the debugging
9+
of the failed test easier.

‎tests/conftest.py‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,7 @@
11
import pytest
22

3+
pytest_plugins = ["pytester"]
4+
35

46
def pytest_addoption(parser):
57
parser.addoption("--ssl", action="store_true", default=False, help="run ssl tests")

0 commit comments

Comments
 (0)