From 2b5cc117c616d81590a060fcbffe314e00604835 Mon Sep 17 00:00:00 2001 From: James Cave Date: Sat, 22 Jul 2023 21:58:09 -0400 Subject: [PATCH 1/4] gh-107089: Improve Shelf.clear method performance The clear method used to be implemented by inheriting a mix-in from the MutableMapping ABC. It was a poor fit for shelves, and a better implementation is now in place --- Lib/shelve.py | 19 +++++++++++++++++++ Misc/ACKS | 1 + ...-07-22-21-57-34.gh-issue-107089.Dnget2.rst | 2 ++ 3 files changed, 22 insertions(+) create mode 100644 Misc/NEWS.d/next/Library/2023-07-22-21-57-34.gh-issue-107089.Dnget2.rst diff --git a/Lib/shelve.py b/Lib/shelve.py index e053c397345a07e..fd908bc23fe0ade 100644 --- a/Lib/shelve.py +++ b/Lib/shelve.py @@ -171,6 +171,25 @@ def sync(self): if hasattr(self.dict, 'sync'): self.dict.sync() + def clear(self): + """Remove all items from the shelf.""" + self.cache.clear() + try: + self.dict.clear() + except AttributeError: + # dbm objects don't have a clear method, so we need to do + # the clearing here. + keys = self.dict.keys() + if not isinstance(keys, list): + # The keys method on dbm objects returns a list. + # Typically, the keys method on a mapping returns a + # lazy iterator, so we need to watch out for that in + # case someone passes in a backing object that behaves + # that way. + keys = list(keys) + for k in keys: + del self.dict[k] + class BsdDbShelf(Shelf): """Shelf implementation using the "BSD" db interface. diff --git a/Misc/ACKS b/Misc/ACKS index fadf488888aa8ba..8b8c5ad8434bd72 100644 --- a/Misc/ACKS +++ b/Misc/ACKS @@ -289,6 +289,7 @@ Edward Catmur Lorenzo M. Catucci Bruno Cauet Donn Cave +James Cave Charles Cazabon Jesús Cea Avión Per Cederqvist diff --git a/Misc/NEWS.d/next/Library/2023-07-22-21-57-34.gh-issue-107089.Dnget2.rst b/Misc/NEWS.d/next/Library/2023-07-22-21-57-34.gh-issue-107089.Dnget2.rst new file mode 100644 index 000000000000000..c67301a918ae600 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2023-07-22-21-57-34.gh-issue-107089.Dnget2.rst @@ -0,0 +1,2 @@ +The :meth:`Shelf.clear` method in the :mod:`shelve` module is much faster. +Patch by James Cave. From 0e3f387ca1f5c25b0b59dbb81064bd1e4b57df55 Mon Sep 17 00:00:00 2001 From: James Cave Date: Mon, 24 Jul 2023 10:41:29 -0400 Subject: [PATCH 2/4] gh-107089: call dbm.clear method Because gh-107089 is peculiar to implementation details of dbm objects, it would be less disruptive to implement it in the DbfilenameShelf class, which is used for calls to shelve.open. Since it is known that the backing object is specifically one of the dbm objects, its clear method (see gh-107122) can be used with no fallback code. --- Lib/shelve.py | 27 +++++-------------- ...-07-22-21-57-34.gh-issue-107089.Dnget2.rst | 4 +-- 2 files changed, 9 insertions(+), 22 deletions(-) diff --git a/Lib/shelve.py b/Lib/shelve.py index fd908bc23fe0ade..d71b243dd359e2e 100644 --- a/Lib/shelve.py +++ b/Lib/shelve.py @@ -171,26 +171,6 @@ def sync(self): if hasattr(self.dict, 'sync'): self.dict.sync() - def clear(self): - """Remove all items from the shelf.""" - self.cache.clear() - try: - self.dict.clear() - except AttributeError: - # dbm objects don't have a clear method, so we need to do - # the clearing here. - keys = self.dict.keys() - if not isinstance(keys, list): - # The keys method on dbm objects returns a list. - # Typically, the keys method on a mapping returns a - # lazy iterator, so we need to watch out for that in - # case someone passes in a backing object that behaves - # that way. - keys = list(keys) - for k in keys: - del self.dict[k] - - class BsdDbShelf(Shelf): """Shelf implementation using the "BSD" db interface. @@ -244,6 +224,13 @@ class DbfilenameShelf(Shelf): def __init__(self, filename, flag='c', protocol=None, writeback=False): import dbm Shelf.__init__(self, dbm.open(filename, flag), protocol, writeback) + + def clear(self): + """Remove all items from the shelf.""" + # Call through to the clear method on dbm-backed shelves. + # see https://github.com/python/cpython/issues/107089 + self.cache.clear() + self.dict.clear() def open(filename, flag='c', protocol=None, writeback=False): diff --git a/Misc/NEWS.d/next/Library/2023-07-22-21-57-34.gh-issue-107089.Dnget2.rst b/Misc/NEWS.d/next/Library/2023-07-22-21-57-34.gh-issue-107089.Dnget2.rst index c67301a918ae600..9d5ba1a2d7ccba8 100644 --- a/Misc/NEWS.d/next/Library/2023-07-22-21-57-34.gh-issue-107089.Dnget2.rst +++ b/Misc/NEWS.d/next/Library/2023-07-22-21-57-34.gh-issue-107089.Dnget2.rst @@ -1,2 +1,2 @@ -The :meth:`Shelf.clear` method in the :mod:`shelve` module is much faster. -Patch by James Cave. +Shelves opened with :func:`shelve.open` have a much faster :meth:`clear` +method. Patch by James Cave. \ No newline at end of file From 0f413694df2291ba7a7a769713753c72828c8113 Mon Sep 17 00:00:00 2001 From: James Cave Date: Mon, 24 Jul 2023 10:56:09 -0400 Subject: [PATCH 3/4] fix whitespace --- Lib/shelve.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Lib/shelve.py b/Lib/shelve.py index d71b243dd359e2e..ec6cf3073c09559 100644 --- a/Lib/shelve.py +++ b/Lib/shelve.py @@ -224,7 +224,7 @@ class DbfilenameShelf(Shelf): def __init__(self, filename, flag='c', protocol=None, writeback=False): import dbm Shelf.__init__(self, dbm.open(filename, flag), protocol, writeback) - + def clear(self): """Remove all items from the shelf.""" # Call through to the clear method on dbm-backed shelves. From d90b207029f30c4be77f4d31cb0ca116fe99bd84 Mon Sep 17 00:00:00 2001 From: James Cave Date: Thu, 27 Jul 2023 16:48:26 -0400 Subject: [PATCH 4/4] fix whitespace --- Lib/shelve.py | 1 + 1 file changed, 1 insertion(+) diff --git a/Lib/shelve.py b/Lib/shelve.py index ec6cf3073c09559..50584716e9ea641 100644 --- a/Lib/shelve.py +++ b/Lib/shelve.py @@ -171,6 +171,7 @@ def sync(self): if hasattr(self.dict, 'sync'): self.dict.sync() + class BsdDbShelf(Shelf): """Shelf implementation using the "BSD" db interface.