Avii/Radicale
1
0
Fork 0
Code Issues Pull Requests Actions 2 Packages Projects Releases Wiki Activity

multifilesystem: Improve performance of PROPFIND

Browse Source 
The PROPFIND does not need to check the filesystem for the file existence,
because the list comes from the directory listing, similar as the other
parts already do it. That was highly visible with large collections (10k+ items),
causing massive delays.

Also avoid repeated stat() calls on the same path.
This commit is contained in:
Milan Crha
2026-08-07 12:22:22 +02:00
parent 86fd5e8aa5
commit dc2a6ac1d7
5 changed files with 52 additions and 5 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
radicale/storage/multifilesystem/discover.py
View File

@@ -88,7 +88,9 @@ class StoragePartDiscover(StorageBase):
for href in collection._list():
with child_context_manager(sane_path, href):
item = collection._get(href)
# We don't need to check for collisions, because the file
# names are from _list() (os.scandir).
item = collection._get(href, verify_href=False)
if item is not None:
yield item

10
radicale/storage/multifilesystem/get.py
View File

@@ -68,6 +68,7 @@ class CollectionPartGet(CollectionPartCache, CollectionPartLock,
return None
else:
path = os.path.join(self._filesystem_path, href)
item_stat: Optional[os.stat_result] = None
try:
if self._storage._use_mtime_and_size_for_item_cache is True:
# try to avoid "open"
@@ -78,6 +79,9 @@ class CollectionPartGet(CollectionPartCache, CollectionPartLock,
raise IsADirectoryError(path)
if not os.access(path, os.R_OK):
raise PermissionError(path)
# Single stat() reused below for both the cache hash and
# last_modified, instead of stat-ing the same path 3 times.
item_stat = os.stat(path)
else:
with open(path, "rb") as f:
# early read of the content
@@ -95,7 +99,8 @@ class CollectionPartGet(CollectionPartCache, CollectionPartLock,
# The hash of the component in the file system. This is used to check,
# if the entry in the cache is still valid.
if self._storage._use_mtime_and_size_for_item_cache is True:
cache_hash = self._item_cache_mtime_and_size(os.stat(path).st_size, os.stat(path).st_mtime_ns)
assert item_stat is not None
cache_hash = self._item_cache_mtime_and_size(item_stat.st_size, item_stat.st_mtime_ns)
if self._storage._debug_cache_actions is True:
logger.debug("Item cache check for: %r with mtime and size %r", path, cache_hash)
else:
@@ -149,7 +154,8 @@ class CollectionPartGet(CollectionPartCache, CollectionPartLock,
logger.debug("Item cache hit for: %r", path)
last_modified = time.strftime(
"%a, %d %b %Y %H:%M:%S GMT",
time.gmtime(os.path.getmtime(path)))
time.gmtime(item_stat.st_mtime if item_stat is not None
else os.path.getmtime(path)))
# Don't keep reference to ``vobject_item``, because it requires a lot
# of memory.
return radicale_item.Item(

6
radicale/storage/multifilesystem/upload.py
View File

@@ -50,7 +50,8 @@ class CollectionPartUpload(CollectionPartGet, CollectionPartCache,
(href, self.path, e)) from e
# store cache file
if self._storage._use_mtime_and_size_for_item_cache is True:
cache_hash = self._item_cache_mtime_and_size(os.stat(path).st_size, os.stat(path).st_mtime_ns)
path_stat = os.stat(path)
cache_hash = self._item_cache_mtime_and_size(path_stat.st_size, path_stat.st_mtime_ns)
if self._storage._debug_cache_actions is True:
logger.debug("Item cache store for: %r with mtime and size %r", path, cache_hash)
else:
@@ -121,7 +122,8 @@ class CollectionPartUpload(CollectionPartGet, CollectionPartCache,
# store cache file
if self._storage._use_mtime_and_size_for_item_cache is True:
cache_hash = self._item_cache_mtime_and_size(os.stat(path).st_size, os.stat(path).st_mtime_ns)
path_stat = os.stat(path)
cache_hash = self._item_cache_mtime_and_size(path_stat.st_size, path_stat.st_mtime_ns)
if self._storage._debug_cache_actions is True:
logger.debug("Item cache store for: %r with mtime and size %r", path, cache_hash)
else: