From 22e885f109a85b865a798c13c7bd0c7310b5bac6 Mon Sep 17 00:00:00 2001 From: Bruno Oliveira Date: Sun, 19 May 2024 18:17:40 -0300 Subject: [PATCH 01/10] Fix link in changelog (#12343) (#12344) Co-authored-by: Hynek Schlawack --- doc/en/changelog.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/en/changelog.rst b/doc/en/changelog.rst index f69b9782bbc..ca7f5b999bd 100644 --- a/doc/en/changelog.rst +++ b/doc/en/changelog.rst @@ -57,7 +57,7 @@ Bug Fixes Trivial/Internal Changes ------------------------ -- `#12333 `_: pytest releases are now attested using the recent `Artifact Attestation ` support from GitHub, allowing users to verify the provenance of pytest's sdist and wheel artifacts. +- `#12333 `_: pytest releases are now attested using the recent `Artifact Attestation `_ support from GitHub, allowing users to verify the provenance of pytest's sdist and wheel artifacts. pytest 8.2.0 (2024-04-27) From f0e7a07667430ebd7fb4fb46cd4d7c3bb9cf18ce Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Tue, 21 May 2024 16:11:45 +0000 Subject: [PATCH 02/10] [8.2.x] Issue #12290 - Docs using Furo Theme W/ Dark Mode (#12348) Co-authored-by: Sam Jirovec <47160720+samjirovec@users.noreply.github.com> --- AUTHORS | 1 + CONTRIBUTING.rst | 4 ---- changelog/12290.doc.rst | 1 + doc/en/{img => _static}/pytest1.png | Bin doc/en/_templates/sidebar/brand.html | 7 ++++++ doc/en/_templates/slim_searchbox.html | 14 ------------ doc/en/_templates/style.html | 7 ++++++ doc/en/conf.py | 30 ++++++++++---------------- doc/en/deprecations.rst | 4 ---- doc/en/index.rst | 17 +++++++-------- doc/en/reference/reference.rst | 3 --- doc/en/requirements.txt | 4 ++-- 12 files changed, 37 insertions(+), 55 deletions(-) create mode 100644 changelog/12290.doc.rst rename doc/en/{img => _static}/pytest1.png (100%) create mode 100644 doc/en/_templates/sidebar/brand.html delete mode 100644 doc/en/_templates/slim_searchbox.html create mode 100644 doc/en/_templates/style.html diff --git a/AUTHORS b/AUTHORS index 54ed85fc732..cc53ce10d4f 100644 --- a/AUTHORS +++ b/AUTHORS @@ -358,6 +358,7 @@ Sadra Barikbin Saiprasad Kale Samuel Colvin Samuel Dion-Girardeau +Samuel Jirovec Samuel Searles-Bryant Samuel Therrien (Avasam) Samuele Pedroni diff --git a/CONTRIBUTING.rst b/CONTRIBUTING.rst index d7da59c812d..0bf440da261 100644 --- a/CONTRIBUTING.rst +++ b/CONTRIBUTING.rst @@ -5,10 +5,6 @@ Contribution getting started Contributions are highly welcomed and appreciated. Every little bit of help counts, so do not hesitate! -.. contents:: - :depth: 2 - :backlinks: none - .. _submitfeedback: diff --git a/changelog/12290.doc.rst b/changelog/12290.doc.rst new file mode 100644 index 00000000000..07fe3babc66 --- /dev/null +++ b/changelog/12290.doc.rst @@ -0,0 +1 @@ +Updated Sphinx theme to use Furo instead of Flask, enabling Dark mode theme. diff --git a/doc/en/img/pytest1.png b/doc/en/_static/pytest1.png similarity index 100% rename from doc/en/img/pytest1.png rename to doc/en/_static/pytest1.png diff --git a/doc/en/_templates/sidebar/brand.html b/doc/en/_templates/sidebar/brand.html new file mode 100644 index 00000000000..f997c4cca5f --- /dev/null +++ b/doc/en/_templates/sidebar/brand.html @@ -0,0 +1,7 @@ + + + diff --git a/doc/en/_templates/slim_searchbox.html b/doc/en/_templates/slim_searchbox.html deleted file mode 100644 index f088ff8d312..00000000000 --- a/doc/en/_templates/slim_searchbox.html +++ /dev/null @@ -1,14 +0,0 @@ -{# - basic/searchbox.html with heading removed. -#} -{%- if pagename != "search" and builder != "singlehtml" %} - - -{%- endif %} diff --git a/doc/en/_templates/style.html b/doc/en/_templates/style.html new file mode 100644 index 00000000000..400cb75ff97 --- /dev/null +++ b/doc/en/_templates/style.html @@ -0,0 +1,7 @@ + diff --git a/doc/en/conf.py b/doc/en/conf.py index 32ecaa17435..6438167dc7f 100644 --- a/doc/en/conf.py +++ b/doc/en/conf.py @@ -15,9 +15,7 @@ # # The full version, including alpha/beta/rc tags. # The short X.Y version. -import os import shutil -import sys from textwrap import dedent from typing import TYPE_CHECKING @@ -65,7 +63,6 @@ # Add any Sphinx extension module names here, as strings. They can be extensions # coming with Sphinx (named 'sphinx.ext.*') or your custom ones. extensions = [ - "pallets_sphinx_themes", "pygments_pytest", "sphinx.ext.autodoc", "sphinx.ext.autosummary", @@ -140,10 +137,6 @@ # output. They are ignored by default. # show_authors = False -# The name of the Pygments (syntax highlighting) style to use. -pygments_style = "sphinx" - - # A list of ignored prefixes for module index sorting. # modindex_common_prefix = [] @@ -216,12 +209,9 @@ # -- Options for HTML output --------------------------------------------------- -sys.path.append(os.path.abspath("_themes")) -html_theme_path = ["_themes"] - # The theme to use for HTML and HTML Help pages. See the documentation for # a list of builtin themes. -html_theme = "flask" +html_theme = "furo" # Theme options are theme-specific and customize the look and feel of a theme # further. For a list of options available for each theme, see the @@ -266,18 +256,24 @@ html_sidebars = { "index": [ - "slim_searchbox.html", + "sidebar/brand.html", + "sidebar/search.html", + "sidebar/scroll-start.html", "sidebarintro.html", "globaltoc.html", "links.html", - "sourcelink.html", + "sidebar/scroll-end.html", + "style.html", ], "**": [ - "slim_searchbox.html", + "sidebar/brand.html", + "sidebar/search.html", + "sidebar/scroll-start.html", "globaltoc.html", "relations.html", "links.html", - "sourcelink.html", + "sidebar/scroll-end.html", + "style.html", ], } @@ -337,10 +333,6 @@ ) ] -# The name of an image file (relative to this directory) to place at the top of -# the title page. -latex_logo = "img/pytest1.png" - # For "manual" documents, if this is true, then toplevel headings are parts, # not chapters. # latex_use_parts = False diff --git a/doc/en/deprecations.rst b/doc/en/deprecations.rst index a65ea331663..bf6268a4980 100644 --- a/doc/en/deprecations.rst +++ b/doc/en/deprecations.rst @@ -7,10 +7,6 @@ This page lists all pytest features that are currently deprecated or have been r The objective is to give users a clear rationale why a certain feature has been removed, and what alternatives should be used instead. -.. contents:: - :depth: 3 - :local: - Deprecated Features ------------------- diff --git a/doc/en/index.rst b/doc/en/index.rst index 83eb27b0a53..2aebb0d6b9c 100644 --- a/doc/en/index.rst +++ b/doc/en/index.rst @@ -1,16 +1,16 @@ :orphan: -.. sidebar:: Next Open Trainings and Events +.. _features: - - `Professional Testing with Python `_, via `Python Academy `_ (3 day in-depth training): - * **June 11th to 13th 2024**, Remote - * **March 4th to 6th 2025**, Leipzig, Germany / Remote - - `pytest development sprint `_, **June 17th -- 22nd 2024** - - pytest tips and tricks for a better testsuite, `Europython 2024 `_, **July 8th -- 14th 2024** (3h), Prague +.. sidebar:: **Next Open Trainings and Events** - Also see :doc:`previous talks and blogposts `. + - `Professional Testing with Python `_, via `Python Academy `_ (3 day in-depth training): + * **June 11th to 13th 2024**, Remote + * **March 4th to 6th 2025**, Leipzig, Germany / Remote + - `pytest development sprint `_, **June 17th -- 22nd 2024** + - pytest tips and tricks for a better testsuite, `Europython 2024 `_, **July 8th -- 14th 2024** (3h), Prague -.. _features: + Also see :doc:`previous talks and blogposts ` pytest: helps you write better programs ======================================= @@ -25,7 +25,6 @@ scale to support complex functional testing for applications and libraries. **PyPI package name**: :pypi:`pytest` - A quick example --------------- diff --git a/doc/en/reference/reference.rst b/doc/en/reference/reference.rst index 4036b7d9912..3675c7cb2ee 100644 --- a/doc/en/reference/reference.rst +++ b/doc/en/reference/reference.rst @@ -7,9 +7,6 @@ API Reference This page contains the full reference to pytest's API. -.. contents:: - :depth: 3 - :local: Constants --------- diff --git a/doc/en/requirements.txt b/doc/en/requirements.txt index 974988c8cf4..6e7221d645a 100644 --- a/doc/en/requirements.txt +++ b/doc/en/requirements.txt @@ -1,4 +1,3 @@ -pallets-sphinx-themes pluggy>=1.5.0 pygments-pytest>=2.3.0 sphinx-removed-in>=0.2.0 @@ -8,4 +7,5 @@ sphinxcontrib-svg2pdfconverter # Pin packaging because it no longer handles 'latest' version, which # is the version that is assigned to the docs. # See https://github.com/pytest-dev/pytest/pull/10578#issuecomment-1348249045. -packaging <22 +packaging +furo From 558e4fa71a28f55481f6857c30dd836cf6f6fa09 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Fri, 24 May 2024 08:22:20 -0300 Subject: [PATCH 03/10] [8.2.x] Add thread safety section to flaky test docs (#12362) Co-authored-by: Nathan Goldbaum --- AUTHORS | 1 + changelog/12356.doc.rst | 2 ++ doc/en/explanation/flaky.rst | 22 +++++++++++++++++++--- 3 files changed, 22 insertions(+), 3 deletions(-) create mode 100644 changelog/12356.doc.rst diff --git a/AUTHORS b/AUTHORS index cc53ce10d4f..18c60750e30 100644 --- a/AUTHORS +++ b/AUTHORS @@ -289,6 +289,7 @@ Mike Lundy Milan Lesnek Miro Hrončok mrbean-bremen +Nathan Goldbaum Nathaniel Compton Nathaniel Waisbrot Ned Batchelder diff --git a/changelog/12356.doc.rst b/changelog/12356.doc.rst new file mode 100644 index 00000000000..312c26d3298 --- /dev/null +++ b/changelog/12356.doc.rst @@ -0,0 +1,2 @@ +Added a subsection to the documentation for debugging flaky tests to mention +lack of thread safety in pytest as a possible source of flakyness. diff --git a/doc/en/explanation/flaky.rst b/doc/en/explanation/flaky.rst index 41cbe847989..cb6c3983424 100644 --- a/doc/en/explanation/flaky.rst +++ b/doc/en/explanation/flaky.rst @@ -18,7 +18,7 @@ System state Broadly speaking, a flaky test indicates that the test relies on some system state that is not being appropriately controlled - the test environment is not sufficiently isolated. Higher level tests are more likely to be flaky as they rely on more state. -Flaky tests sometimes appear when a test suite is run in parallel (such as use of pytest-xdist). This can indicate a test is reliant on test ordering. +Flaky tests sometimes appear when a test suite is run in parallel (such as use of `pytest-xdist`_). This can indicate a test is reliant on test ordering. - Perhaps a different test is failing to clean up after itself and leaving behind data which causes the flaky test to fail. - The flaky test is reliant on data from a previous test that doesn't clean up after itself, and in parallel runs that previous test is not always present @@ -30,9 +30,22 @@ Overly strict assertion Overly strict assertions can cause problems with floating point comparison as well as timing issues. :func:`pytest.approx` is useful here. +Thread safety +~~~~~~~~~~~~~ -Pytest features -^^^^^^^^^^^^^^^ +pytest is single-threaded, executing its tests always in the same thread, sequentially, never spawning any threads itself. + +Even in case of plugins which run tests in parallel, for example `pytest-xdist`_, usually work by spawning multiple *processes* and running tests in batches, without using multiple threads. + +It is of course possible (and common) for tests and fixtures to spawn threads themselves as part of their testing workflow (for example, a fixture that starts a server thread in the background, or a test which executes production code that spawns threads), but some care must be taken: + +* Make sure to eventually wait on any spawned threads -- for example at the end of a test, or during the teardown of a fixture. +* Avoid using primitives provided by pytest (:func:`pytest.warns`, :func:`pytest.raises`, etc) from multiple threads, as they are not thread-safe. + +If your test suite uses threads and your are seeing flaky test results, do not discount the possibility that the test is implicitly using global state in pytest itself. + +Related features +^^^^^^^^^^^^^^^^ Xfail strict ~~~~~~~~~~~~ @@ -123,3 +136,6 @@ Resources * `Flaky Tests at Google and How We Mitigate Them `_ by John Micco, 2016 * `Where do Google's flaky tests come from? `_ by Jeff Listfield, 2017 + + +.. _pytest-xdist: https://github.com/pytest-dev/pytest-xdist From f7358aec2884720b4de4594ffd0811b46316514c Mon Sep 17 00:00:00 2001 From: Ran Benita Date: Sun, 26 May 2024 10:34:11 +0300 Subject: [PATCH 04/10] [8.2.x] unittest: fix class instances no longer released on test teardown since pytest 8.2.0 --- changelog/12367.bugfix.rst | 1 + src/_pytest/unittest.py | 3 ++- testing/test_unittest.py | 36 ++++++++++++++++++++---------------- 3 files changed, 23 insertions(+), 17 deletions(-) create mode 100644 changelog/12367.bugfix.rst diff --git a/changelog/12367.bugfix.rst b/changelog/12367.bugfix.rst new file mode 100644 index 00000000000..e8bf2e4f155 --- /dev/null +++ b/changelog/12367.bugfix.rst @@ -0,0 +1 @@ +Fix a regression in pytest 8.2.0 where unittest class instances (a fresh one is created for each test) were not released promptly on test teardown but only on session teardown. diff --git a/src/_pytest/unittest.py b/src/_pytest/unittest.py index 8f1791bf744..baca58bd9b2 100644 --- a/src/_pytest/unittest.py +++ b/src/_pytest/unittest.py @@ -212,11 +212,12 @@ def setup(self) -> None: super().setup() def teardown(self) -> None: - super().teardown() if self._explicit_tearDown is not None: self._explicit_tearDown() self._explicit_tearDown = None self._obj = None + self._instance = None + super().teardown() def startTest(self, testcase: "unittest.TestCase") -> None: pass diff --git a/testing/test_unittest.py b/testing/test_unittest.py index 96223b22a2e..d238ffbfa74 100644 --- a/testing/test_unittest.py +++ b/testing/test_unittest.py @@ -1,5 +1,4 @@ # mypy: allow-untyped-defs -import gc import sys from typing import List @@ -192,30 +191,35 @@ def test_check(self): def test_teardown_issue1649(pytester: Pytester) -> None: """ Are TestCase objects cleaned up? Often unittest TestCase objects set - attributes that are large and expensive during setUp. + attributes that are large and expensive during test run or setUp. The TestCase will not be cleaned up if the test fails, because it would then exist in the stackframe. + + Regression test for #1649 (see also #12367). """ - testpath = pytester.makepyfile( + pytester.makepyfile( """ import unittest - class TestCaseObjectsShouldBeCleanedUp(unittest.TestCase): - def setUp(self): - self.an_expensive_object = 1 - def test_demo(self): - pass + import gc - """ + class TestCaseObjectsShouldBeCleanedUp(unittest.TestCase): + def test_expensive(self): + self.an_expensive_obj = object() + + def test_is_it_still_alive(self): + gc.collect() + for obj in gc.get_objects(): + if type(obj).__name__ == "TestCaseObjectsShouldBeCleanedUp": + assert not hasattr(obj, "an_expensive_obj") + break + else: + assert False, "Could not find TestCaseObjectsShouldBeCleanedUp instance" + """ ) - pytester.inline_run("-s", testpath) - gc.collect() - - # Either already destroyed, or didn't run setUp. - for obj in gc.get_objects(): - if type(obj).__name__ == "TestCaseObjectsShouldBeCleanedUp": - assert not hasattr(obj, "an_expensive_obj") + result = pytester.runpytest() + assert result.ret == ExitCode.OK def test_unittest_skip_issue148(pytester: Pytester) -> None: From 240a252d34fff26efad5b3a92e62be4c9af94b70 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Sun, 26 May 2024 10:38:25 +0000 Subject: [PATCH 05/10] [8.2.x] Add html_baseurl to sphinx conf.py (#12372) Co-authored-by: James Frost --- AUTHORS | 1 + changelog/12363.doc.rst | 1 + doc/en/conf.py | 3 +++ 3 files changed, 5 insertions(+) create mode 100644 changelog/12363.doc.rst diff --git a/AUTHORS b/AUTHORS index 18c60750e30..748b9bae2c8 100644 --- a/AUTHORS +++ b/AUTHORS @@ -192,6 +192,7 @@ Jake VanderPlas Jakob van Santen Jakub Mitoraj James Bourbeau +James Frost Jan Balster Janne Vanhala Jason R. Coombs diff --git a/changelog/12363.doc.rst b/changelog/12363.doc.rst new file mode 100644 index 00000000000..c657281babf --- /dev/null +++ b/changelog/12363.doc.rst @@ -0,0 +1 @@ +The documentation webpages now links to a canonical version to reduce outdated documentation in search engine results. diff --git a/doc/en/conf.py b/doc/en/conf.py index 6438167dc7f..48b40ef9079 100644 --- a/doc/en/conf.py +++ b/doc/en/conf.py @@ -312,6 +312,9 @@ # Output file base name for HTML help builder. htmlhelp_basename = "pytestdoc" +# The base URL which points to the root of the HTML documentation. It is used +# to indicate the location of document using the canonical link relation (#12363). +html_baseurl = "https://docs.pytest.org/en/stable/" # -- Options for LaTeX output -------------------------------------------------- From 1d103e5cdc1cb08f332e61a5b20fb205fa5228e7 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Tue, 28 May 2024 17:41:35 +0000 Subject: [PATCH 06/10] [8.2.x] Clarify pytest_ignore_collect docs (#12386) Co-authored-by: Bruno Oliveira --- src/_pytest/hookspec.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/src/_pytest/hookspec.py b/src/_pytest/hookspec.py index 9ec9b3b5e10..b19cd3272a9 100644 --- a/src/_pytest/hookspec.py +++ b/src/_pytest/hookspec.py @@ -310,7 +310,12 @@ def pytest_collection_finish(session: "Session") -> None: def pytest_ignore_collect( collection_path: Path, path: "LEGACY_PATH", config: "Config" ) -> Optional[bool]: - """Return True to prevent considering this path for collection. + """Return ``True`` to ignore this path for collection. + + Return ``None`` to let other plugins ignore the path for collection. + + Returning ``False`` will forcefully *not* ignore this path for collection, + without giving a chance for other plugins to ignore this path. This hook is consulted for all files and directories prior to calling more specific hooks. From 4569a01e3d20d64811d48b0b09539596520ea5a6 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Thu, 30 May 2024 17:05:32 +0200 Subject: [PATCH 07/10] [8.2.x] doc: Update trainings/events (#12402) Co-authored-by: Florian Bruhin --- doc/en/index.rst | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/doc/en/index.rst b/doc/en/index.rst index 2aebb0d6b9c..58527ea7331 100644 --- a/doc/en/index.rst +++ b/doc/en/index.rst @@ -4,11 +4,10 @@ .. sidebar:: **Next Open Trainings and Events** - - `Professional Testing with Python `_, via `Python Academy `_ (3 day in-depth training): - * **June 11th to 13th 2024**, Remote - * **March 4th to 6th 2025**, Leipzig, Germany / Remote - - `pytest development sprint `_, **June 17th -- 22nd 2024** - - pytest tips and tricks for a better testsuite, `Europython 2024 `_, **July 8th -- 14th 2024** (3h), Prague + - `pytest development sprint `_, **June 17th -- 22nd 2024**, Klaus (AT) / Remote + - `pytest tips and tricks for a better testsuite `_, at `Europython 2024 `_, **July 8th -- 14th 2024** (3h), Prague (CZ) + - `pytest: Professionelles Testen (nicht nur) für Python `_, at `CH Open Workshoptage `_, **September 2nd 2024**, HSLU Rotkreuz (CH) + - `Professional Testing with Python `_, via `Python Academy `_ (3 day in-depth training), **March 4th -- 6th 2025**, Leipzig (DE) / Remote Also see :doc:`previous talks and blogposts ` From 9bb73d734ff40f52d7bbebd708b5e3ab1ba20798 Mon Sep 17 00:00:00 2001 From: Ran Benita Date: Mon, 3 Jun 2024 12:43:59 +0300 Subject: [PATCH 08/10] [8.2.x] cacheprovider: fix "Directory not empty" crash from cache directory creation --- changelog/12381.bugfix.rst | 1 + src/_pytest/cacheprovider.py | 27 +++++++++++++++++++-------- 2 files changed, 20 insertions(+), 8 deletions(-) create mode 100644 changelog/12381.bugfix.rst diff --git a/changelog/12381.bugfix.rst b/changelog/12381.bugfix.rst new file mode 100644 index 00000000000..02233cd4a84 --- /dev/null +++ b/changelog/12381.bugfix.rst @@ -0,0 +1 @@ +Fix possible "Directory not empty" crashes arising from concurent cache dir (``.pytest_cache``) creation. Regressed in pytest 8.2.0. diff --git a/src/_pytest/cacheprovider.py b/src/_pytest/cacheprovider.py index a9cbb77cbbb..30a71aaeb67 100755 --- a/src/_pytest/cacheprovider.py +++ b/src/_pytest/cacheprovider.py @@ -4,6 +4,7 @@ # This plugin was not named "cache" to avoid conflicts with the external # pytest-cache version. import dataclasses +import errno import json import os from pathlib import Path @@ -227,14 +228,24 @@ def _ensure_cache_dir_and_supporting_files(self) -> None: with open(path.joinpath("CACHEDIR.TAG"), "xb") as f: f.write(CACHEDIR_TAG_CONTENT) - path.rename(self._cachedir) - # Create a directory in place of the one we just moved so that `TemporaryDirectory`'s - # cleanup doesn't complain. - # - # TODO: pass ignore_cleanup_errors=True when we no longer support python < 3.10. See - # https://github.com/python/cpython/issues/74168. Note that passing delete=False would - # do the wrong thing in case of errors and isn't supported until python 3.12. - path.mkdir() + try: + path.rename(self._cachedir) + except OSError as e: + # If 2 concurrent pytests both race to the rename, the loser + # gets "Directory not empty" from the rename. In this case, + # everything is handled so just continue (while letting the + # temporary directory be cleaned up). + if e.errno != errno.ENOTEMPTY: + raise + else: + # Create a directory in place of the one we just moved so that + # `TemporaryDirectory`'s cleanup doesn't complain. + # + # TODO: pass ignore_cleanup_errors=True when we no longer support python < 3.10. + # See https://github.com/python/cpython/issues/74168. Note that passing + # delete=False would do the wrong thing in case of errors and isn't supported + # until python 3.12. + path.mkdir() class LFPluginCollWrapper: From 153a436bc40c9e89d90d62255ef5a89e9a762dca Mon Sep 17 00:00:00 2001 From: Ran Benita Date: Sun, 2 Jun 2024 17:57:38 +0300 Subject: [PATCH 09/10] [8.2.x] fixtures: fix catastrophic performance problem in `reorder_items` Manual minimal backport from commit e89d23b24741c001e8651a77303992cfa41c1664. Fix #12355. In the issue, it was reported that the `reorder_items` has quadratic (or worse...) behavior with certain simple parametrizations. After some debugging I found that the problem happens because the "Fix items_by_argkey order" loop keeps adding the same item to the deque, and it reaches epic sizes which causes the slowdown. I don't claim to understand how the `reorder_items` algorithm works, but if as far as I understand, if an item already exists in the deque, the correct thing to do is to move it to the front. Since a deque doesn't have such an (efficient) operation, this switches to `OrderedDict` which can efficiently append from both sides, deduplicate and move to front. --- changelog/12355.bugfix.rst | 1 + src/_pytest/fixtures.py | 19 +++++++++++-------- testing/python/fixtures.py | 19 +++++++++++++++++++ 3 files changed, 31 insertions(+), 8 deletions(-) create mode 100644 changelog/12355.bugfix.rst diff --git a/changelog/12355.bugfix.rst b/changelog/12355.bugfix.rst new file mode 100644 index 00000000000..1ce43e60ebd --- /dev/null +++ b/changelog/12355.bugfix.rst @@ -0,0 +1 @@ +Fix possible catastrophic performance slowdown on a certain parametrization pattern involving many higher-scoped parameters. diff --git a/src/_pytest/fixtures.py b/src/_pytest/fixtures.py index 7fd63f937c1..647328e3bea 100644 --- a/src/_pytest/fixtures.py +++ b/src/_pytest/fixtures.py @@ -23,6 +23,7 @@ from typing import MutableMapping from typing import NoReturn from typing import Optional +from typing import OrderedDict from typing import overload from typing import Sequence from typing import Set @@ -75,8 +76,6 @@ if TYPE_CHECKING: - from typing import Deque - from _pytest.main import Session from _pytest.python import CallSpec2 from _pytest.python import Function @@ -207,16 +206,18 @@ def get_parametrized_fixture_keys( def reorder_items(items: Sequence[nodes.Item]) -> List[nodes.Item]: argkeys_cache: Dict[Scope, Dict[nodes.Item, Dict[FixtureArgKey, None]]] = {} - items_by_argkey: Dict[Scope, Dict[FixtureArgKey, Deque[nodes.Item]]] = {} + items_by_argkey: Dict[ + Scope, Dict[FixtureArgKey, OrderedDict[nodes.Item, None]] + ] = {} for scope in HIGH_SCOPES: scoped_argkeys_cache = argkeys_cache[scope] = {} - scoped_items_by_argkey = items_by_argkey[scope] = defaultdict(deque) + scoped_items_by_argkey = items_by_argkey[scope] = defaultdict(OrderedDict) for item in items: keys = dict.fromkeys(get_parametrized_fixture_keys(item, scope), None) if keys: scoped_argkeys_cache[item] = keys for key in keys: - scoped_items_by_argkey[key].append(item) + scoped_items_by_argkey[key][item] = None items_dict = dict.fromkeys(items, None) return list( reorder_items_atscope(items_dict, argkeys_cache, items_by_argkey, Scope.Session) @@ -226,17 +227,19 @@ def reorder_items(items: Sequence[nodes.Item]) -> List[nodes.Item]: def fix_cache_order( item: nodes.Item, argkeys_cache: Dict[Scope, Dict[nodes.Item, Dict[FixtureArgKey, None]]], - items_by_argkey: Dict[Scope, Dict[FixtureArgKey, "Deque[nodes.Item]"]], + items_by_argkey: Dict[Scope, Dict[FixtureArgKey, OrderedDict[nodes.Item, None]]], ) -> None: for scope in HIGH_SCOPES: + scoped_items_by_argkey = items_by_argkey[scope] for key in argkeys_cache[scope].get(item, []): - items_by_argkey[scope][key].appendleft(item) + scoped_items_by_argkey[key][item] = None + scoped_items_by_argkey[key].move_to_end(item, last=False) def reorder_items_atscope( items: Dict[nodes.Item, None], argkeys_cache: Dict[Scope, Dict[nodes.Item, Dict[FixtureArgKey, None]]], - items_by_argkey: Dict[Scope, Dict[FixtureArgKey, "Deque[nodes.Item]"]], + items_by_argkey: Dict[Scope, Dict[FixtureArgKey, OrderedDict[nodes.Item, None]]], scope: Scope, ) -> Dict[nodes.Item, None]: if scope is Scope.Function or len(items) < 3: diff --git a/testing/python/fixtures.py b/testing/python/fixtures.py index 741cf7dcf42..ebdf451e40b 100644 --- a/testing/python/fixtures.py +++ b/testing/python/fixtures.py @@ -2219,6 +2219,25 @@ def test_check(): reprec = pytester.inline_run("-s") reprec.assertoutcome(passed=2) + def test_reordering_catastrophic_performance(self, pytester: Pytester) -> None: + """Check that a certain high-scope parametrization pattern doesn't cause + a catasrophic slowdown. + + Regression test for #12355. + """ + pytester.makepyfile(""" + import pytest + + params = tuple("abcdefghijklmnopqrstuvwxyz") + @pytest.mark.parametrize(params, [range(len(params))] * 3, scope="module") + def test_parametrize(a, b, c, d, e, f, g, h, i, j, k, l, m, n, o, p, q, r, s, t, u, v, w, x, y, z): + pass + """) + + result = pytester.runpytest() + + result.assert_outcomes(passed=3) + class TestFixtureMarker: def test_parametrize(self, pytester: Pytester) -> None: From 329d3712146e69c471be3e30883d54bdde2f76cb Mon Sep 17 00:00:00 2001 From: pytest bot Date: Tue, 4 Jun 2024 12:54:55 +0000 Subject: [PATCH 10/10] Prepare release version 8.2.2 --- changelog/12290.doc.rst | 1 - changelog/12355.bugfix.rst | 1 - changelog/12356.doc.rst | 2 -- changelog/12363.doc.rst | 1 - changelog/12367.bugfix.rst | 1 - changelog/12381.bugfix.rst | 1 - doc/en/announce/index.rst | 1 + doc/en/announce/release-8.2.2.rst | 19 +++++++++++++++++++ doc/en/builtin.rst | 4 ++-- doc/en/changelog.rst | 29 +++++++++++++++++++++++++++++ doc/en/example/parametrize.rst | 6 +++--- doc/en/example/pythoncollection.rst | 4 ++-- doc/en/getting-started.rst | 2 +- doc/en/how-to/fixtures.rst | 2 +- 14 files changed, 58 insertions(+), 16 deletions(-) delete mode 100644 changelog/12290.doc.rst delete mode 100644 changelog/12355.bugfix.rst delete mode 100644 changelog/12356.doc.rst delete mode 100644 changelog/12363.doc.rst delete mode 100644 changelog/12367.bugfix.rst delete mode 100644 changelog/12381.bugfix.rst create mode 100644 doc/en/announce/release-8.2.2.rst diff --git a/changelog/12290.doc.rst b/changelog/12290.doc.rst deleted file mode 100644 index 07fe3babc66..00000000000 --- a/changelog/12290.doc.rst +++ /dev/null @@ -1 +0,0 @@ -Updated Sphinx theme to use Furo instead of Flask, enabling Dark mode theme. diff --git a/changelog/12355.bugfix.rst b/changelog/12355.bugfix.rst deleted file mode 100644 index 1ce43e60ebd..00000000000 --- a/changelog/12355.bugfix.rst +++ /dev/null @@ -1 +0,0 @@ -Fix possible catastrophic performance slowdown on a certain parametrization pattern involving many higher-scoped parameters. diff --git a/changelog/12356.doc.rst b/changelog/12356.doc.rst deleted file mode 100644 index 312c26d3298..00000000000 --- a/changelog/12356.doc.rst +++ /dev/null @@ -1,2 +0,0 @@ -Added a subsection to the documentation for debugging flaky tests to mention -lack of thread safety in pytest as a possible source of flakyness. diff --git a/changelog/12363.doc.rst b/changelog/12363.doc.rst deleted file mode 100644 index c657281babf..00000000000 --- a/changelog/12363.doc.rst +++ /dev/null @@ -1 +0,0 @@ -The documentation webpages now links to a canonical version to reduce outdated documentation in search engine results. diff --git a/changelog/12367.bugfix.rst b/changelog/12367.bugfix.rst deleted file mode 100644 index e8bf2e4f155..00000000000 --- a/changelog/12367.bugfix.rst +++ /dev/null @@ -1 +0,0 @@ -Fix a regression in pytest 8.2.0 where unittest class instances (a fresh one is created for each test) were not released promptly on test teardown but only on session teardown. diff --git a/changelog/12381.bugfix.rst b/changelog/12381.bugfix.rst deleted file mode 100644 index 02233cd4a84..00000000000 --- a/changelog/12381.bugfix.rst +++ /dev/null @@ -1 +0,0 @@ -Fix possible "Directory not empty" crashes arising from concurent cache dir (``.pytest_cache``) creation. Regressed in pytest 8.2.0. diff --git a/doc/en/announce/index.rst b/doc/en/announce/index.rst index 8a33f7fb57d..c65eb5f3613 100644 --- a/doc/en/announce/index.rst +++ b/doc/en/announce/index.rst @@ -6,6 +6,7 @@ Release announcements :maxdepth: 2 + release-8.2.2 release-8.2.1 release-8.2.0 release-8.1.2 diff --git a/doc/en/announce/release-8.2.2.rst b/doc/en/announce/release-8.2.2.rst new file mode 100644 index 00000000000..3b1d93bd08b --- /dev/null +++ b/doc/en/announce/release-8.2.2.rst @@ -0,0 +1,19 @@ +pytest-8.2.2 +======================================= + +pytest 8.2.2 has just been released to PyPI. + +This is a bug-fix release, being a drop-in replacement. To upgrade:: + + pip install --upgrade pytest + +The full changelog is available at https://docs.pytest.org/en/stable/changelog.html. + +Thanks to all of the contributors to this release: + +* Bruno Oliveira +* Ran Benita + + +Happy testing, +The pytest Development Team diff --git a/doc/en/builtin.rst b/doc/en/builtin.rst index 458253fabbb..8dfffb0828a 100644 --- a/doc/en/builtin.rst +++ b/doc/en/builtin.rst @@ -22,7 +22,7 @@ For information about fixtures, see :ref:`fixtures`. To see a complete list of a cachedir: .pytest_cache rootdir: /home/sweet/project collected 0 items - cache -- .../_pytest/cacheprovider.py:549 + cache -- .../_pytest/cacheprovider.py:560 Return a cache object that can persist state between testing sessions. cache.get(key, default) @@ -115,7 +115,7 @@ For information about fixtures, see :ref:`fixtures`. To see a complete list of a For more details: :ref:`doctest_namespace`. - pytestconfig [session scope] -- .../_pytest/fixtures.py:1335 + pytestconfig [session scope] -- .../_pytest/fixtures.py:1338 Session-scoped fixture that returns the session's :class:`pytest.Config` object. diff --git a/doc/en/changelog.rst b/doc/en/changelog.rst index ca7f5b999bd..3ab2307ee5d 100644 --- a/doc/en/changelog.rst +++ b/doc/en/changelog.rst @@ -28,6 +28,35 @@ with advance notice in the **Deprecations** section of releases. .. towncrier release notes start +pytest 8.2.2 (2024-06-04) +========================= + +Bug Fixes +--------- + +- `#12355 `_: Fix possible catastrophic performance slowdown on a certain parametrization pattern involving many higher-scoped parameters. + + +- `#12367 `_: Fix a regression in pytest 8.2.0 where unittest class instances (a fresh one is created for each test) were not released promptly on test teardown but only on session teardown. + + +- `#12381 `_: Fix possible "Directory not empty" crashes arising from concurent cache dir (``.pytest_cache``) creation. Regressed in pytest 8.2.0. + + + +Improved Documentation +---------------------- + +- `#12290 `_: Updated Sphinx theme to use Furo instead of Flask, enabling Dark mode theme. + + +- `#12356 `_: Added a subsection to the documentation for debugging flaky tests to mention + lack of thread safety in pytest as a possible source of flakyness. + + +- `#12363 `_: The documentation webpages now links to a canonical version to reduce outdated documentation in search engine results. + + pytest 8.2.1 (2024-05-19) ========================= diff --git a/doc/en/example/parametrize.rst b/doc/en/example/parametrize.rst index 03f6852e5c0..d540bf08337 100644 --- a/doc/en/example/parametrize.rst +++ b/doc/en/example/parametrize.rst @@ -162,7 +162,7 @@ objects, they are still using the default pytest representation: rootdir: /home/sweet/project collected 8 items - + @@ -239,7 +239,7 @@ If you just collect tests you'll also nicely see 'advanced' and 'basic' as varia rootdir: /home/sweet/project collected 4 items - + @@ -318,7 +318,7 @@ Let's first see how it looks like at collection time: rootdir: /home/sweet/project collected 2 items - + diff --git a/doc/en/example/pythoncollection.rst b/doc/en/example/pythoncollection.rst index aa9d05d7227..39b799ed934 100644 --- a/doc/en/example/pythoncollection.rst +++ b/doc/en/example/pythoncollection.rst @@ -152,7 +152,7 @@ The test collection would look like this: configfile: pytest.ini collected 2 items - + @@ -215,7 +215,7 @@ You can always peek at the collection tree without running tests like this: configfile: pytest.ini collected 3 items - + diff --git a/doc/en/getting-started.rst b/doc/en/getting-started.rst index 94e0d80e656..85bee729ba1 100644 --- a/doc/en/getting-started.rst +++ b/doc/en/getting-started.rst @@ -22,7 +22,7 @@ Install ``pytest`` .. code-block:: bash $ pytest --version - pytest 8.2.1 + pytest 8.2.2 .. _`simpletest`: diff --git a/doc/en/how-to/fixtures.rst b/doc/en/how-to/fixtures.rst index 6cc20c8c3e4..35c3238dea7 100644 --- a/doc/en/how-to/fixtures.rst +++ b/doc/en/how-to/fixtures.rst @@ -1418,7 +1418,7 @@ Running the above tests results in the following test IDs being used: rootdir: /home/sweet/project collected 12 items - +