From 80dc4995cf60e375e00969d41e073dbfb59cbdf9 Mon Sep 17 00:00:00 2001 From: Nate Harris Date: Mon, 14 Jul 2025 00:16:19 -0600 Subject: [PATCH 001/290] - Capture previous version of event pre-overwrite for use in notification hooks - Use previous version of event in email hooks to determine added/deleted/updated email type --- radicale/app/__init__.py | 2 +- radicale/app/delete.py | 22 +- radicale/app/proppatch.py | 6 +- radicale/app/put.py | 50 +++-- radicale/config.py | 21 +- radicale/hook/__init__.py | 21 +- radicale/hook/email/__init__.py | 198 ++++++++++++------ radicale/storage/__init__.py | 12 +- .../multifilesystem/create_collection.py | 45 +++- radicale/storage/multifilesystem/upload.py | 7 +- 10 files changed, 274 insertions(+), 110 deletions(-) diff --git a/radicale/app/__init__.py b/radicale/app/__init__.py index b69950b9..6764b7c6 100644 --- a/radicale/app/__init__.py +++ b/radicale/app/__init__.py @@ -323,7 +323,7 @@ class Application(ApplicationPartDelete, ApplicationPartHead, if "W" in self._rights.authorization(user, principal_path): with self._storage.acquire_lock("w", user): try: - new_coll = self._storage.create_collection(principal_path) + new_coll, _, _ = self._storage.create_collection(principal_path) if new_coll: jsn_coll = self.configuration.get("storage", "predefined_collections") for (name_coll, props) in jsn_coll.items(): diff --git a/radicale/app/delete.py b/radicale/app/delete.py index a111df00..d6cdbfcb 100644 --- a/radicale/app/delete.py +++ b/radicale/app/delete.py @@ -24,7 +24,7 @@ from typing import Optional from radicale import httputils, storage, types, xmlutils from radicale.app.base import Access, ApplicationBase -from radicale.hook import DeleteHookNotificationItem +from radicale.hook import HookNotificationItem, HookNotificationItemTypes from radicale.log import logger @@ -82,10 +82,12 @@ class ApplicationPartDelete(ApplicationBase): return httputils.NOT_ALLOWED for i in item.get_all(): hook_notification_item_list.append( - DeleteHookNotificationItem( - access.path, - i.uid, - old_content=item.serialize() # type: ignore + HookNotificationItem( + notification_item_type=HookNotificationItemTypes.DELETE, + path=access.path, + uid=i.uid, + old_content=item.serialize(), # type: ignore + new_content=None ) ) xml_answer = xml_delete(base_prefix, path, item) @@ -93,10 +95,12 @@ class ApplicationPartDelete(ApplicationBase): assert item.collection is not None assert item.href is not None hook_notification_item_list.append( - DeleteHookNotificationItem( - access.path, - item.uid, - old_content=item.serialize() # type: ignore + HookNotificationItem( + notification_item_type=HookNotificationItemTypes.DELETE, + path=access.path, + uid=item.uid, + old_content=item.serialize(), # type: ignore + new_content=None, ) ) xml_answer = xml_delete( diff --git a/radicale/app/proppatch.py b/radicale/app/proppatch.py index d2c32811..99ec6ae0 100644 --- a/radicale/app/proppatch.py +++ b/radicale/app/proppatch.py @@ -102,9 +102,9 @@ class ApplicationPartProppatch(ApplicationBase): item) if xml_content is not None: hook_notification_item = HookNotificationItem( - HookNotificationItemTypes.CPATCH, - access.path, - DefusedET.tostring( + notification_item_type=HookNotificationItemTypes.CPATCH, + path=access.path, + new_content=DefusedET.tostring( xml_content, encoding=self._encoding ).decode(encoding=self._encoding) diff --git a/radicale/app/put.py b/radicale/app/put.py index 343f3324..575134c6 100644 --- a/radicale/app/put.py +++ b/radicale/app/put.py @@ -243,14 +243,27 @@ class ApplicationPartPut(ApplicationBase): if write_whole_collection: try: - etag = self._storage.create_collection( - path, prepared_items, props).etag + col, replaced_items, new_item_hrefs = self._storage.create_collection( + href=path, + items=prepared_items, + props=props) for item in prepared_items: - hook_notification_item = HookNotificationItem( - HookNotificationItemTypes.UPSERT, - access.path, - item.serialize() - ) + # Try to grab the previously-existing item by href + existing_item = replaced_items.get(item.href, None) + if existing_item: + hook_notification_item = HookNotificationItem( + notification_item_type=HookNotificationItemTypes.UPSERT, + path=access.path, + old_content=existing_item.serialize(), + new_content=item.serialize() + ) + else: # We assume the item is new because it was not in the replaced_items + hook_notification_item = HookNotificationItem( + notification_item_type=HookNotificationItemTypes.UPSERT, + path=access.path, + old_content=None, + new_content=item.serialize() + ) self._hook.notify(hook_notification_item) except ValueError as e: logger.warning( @@ -267,12 +280,23 @@ class ApplicationPartPut(ApplicationBase): href = posixpath.basename(pathutils.strip_path(path)) try: - etag = parent_item.upload(href, prepared_item).etag - hook_notification_item = HookNotificationItem( - HookNotificationItemTypes.UPSERT, - access.path, - prepared_item.serialize() - ) + uploaded_item, replaced_item = parent_item.upload(href, prepared_item) + etag = uploaded_item.etag + if replaced_item: + # If the item was replaced, we notify with the old content + hook_notification_item = HookNotificationItem( + notification_item_type=HookNotificationItemTypes.UPSERT, + path=access.path, + old_content=replaced_item.serialize(), + new_content=prepared_item.serialize() + ) + else: # If it was a new item, we notify with no old content + hook_notification_item = HookNotificationItem( + notification_item_type=HookNotificationItemTypes.UPSERT, + path=access.path, + old_content=None, + new_content=prepared_item.serialize() + ) self._hook.notify(hook_notification_item) except ValueError as e: # return better matching HTTP result in case errno is provided and catched diff --git a/radicale/config.py b/radicale/config.py index 63f627b8..62a3c5ce 100644 --- a/radicale/config.py +++ b/radicale/config.py @@ -477,7 +477,7 @@ DEFAULT_CONFIG_SCHEMA: types.CONFIG_SCHEMA = OrderedDict([ "value": "False", "help": "Send one email to all attendees, versus one email per attendee", "type": bool}), - ("added_template", { + ("new_or_added_to_event_template", { "value": """Hello $attendee_name, You have been added as an attendee to the following calendar event. @@ -487,20 +487,31 @@ You have been added as an attendee to the following calendar event. $event_location This is an automated message. Please do not reply.""", - "help": "Template for the email sent when an event is added or updated. Select placeholder words prefixed with $ will be replaced", + "help": "Template for the email sent when an event is created or attendee is added. Select placeholder words prefixed with $ will be replaced", "type": str}), - ("removed_template", { + ("deleted_or_removed_from_event_template", { "value": """Hello $attendee_name, -You have been removed as an attendee from the following calendar event. +The following event has been deleted. $event_title $event_start_time - $event_end_time $event_location This is an automated message. Please do not reply.""", - "help": "Template for the email sent when an event is deleted. Select placeholder words prefixed with $ will be replaced", + "help": "Template for the email sent when an event is deleted or attendee is removed. Select placeholder words prefixed with $ will be replaced", "type": str}), + ("updated_event_template", { + "value": """Hello $attendee_name, +The following event has been updated. + $event_title + $event_start_time - $event_end_time + $event_location + +This is an automated message. Please do not reply.""", + "help": "Template for the email sent when an event is updated. Select placeholder words prefixed with $ will be replaced", + "type": str + }) ])), ("web", OrderedDict([ ("type", { diff --git a/radicale/hook/__init__.py b/radicale/hook/__init__.py index 835cbe01..04378a2e 100644 --- a/radicale/hook/__init__.py +++ b/radicale/hook/__init__.py @@ -55,10 +55,21 @@ def _cleanup(path): class HookNotificationItem: - def __init__(self, notification_item_type, path, content): + def __init__(self, notification_item_type, path, uid=None, new_content=None, old_content=None): self.type = notification_item_type.value self.point = _cleanup(path) - self.content = content + self.uid = uid + self.new_content = new_content + self.old_content = old_content + + @property + def content(self): # For backward compatibility + return self.uid or self.new_content or self.old_content + + @property + def replaces_existing_item(self) -> bool: + """Check if this notification item replaces/deletes an existing item.""" + return self.old_content is not None def to_json(self): return json.dumps( @@ -67,9 +78,3 @@ class HookNotificationItem: sort_keys=True, indent=4 ) - - -class DeleteHookNotificationItem(HookNotificationItem): - def __init__(self, path, uid, old_content=None): - super().__init__(notification_item_type=HookNotificationItemTypes.DELETE, path=path, content=uid) - self.old_content = old_content diff --git a/radicale/hook/email/__init__.py b/radicale/hook/email/__init__.py index 75d043aa..9d80fbd8 100644 --- a/radicale/hook/email/__init__.py +++ b/radicale/hook/email/__init__.py @@ -29,8 +29,7 @@ from typing import Any, Dict, List, Optional, Sequence, Tuple import vobject -from radicale.hook import (BaseHook, DeleteHookNotificationItem, - HookNotificationItem, HookNotificationItemTypes) +from radicale.hook import (BaseHook, HookNotificationItem, HookNotificationItemTypes) from radicale.log import logger PLUGIN_CONFIG_SCHEMA = { @@ -63,7 +62,7 @@ PLUGIN_CONFIG_SCHEMA = { "value": "", "type": str }, - "added_template": { + "new_or_added_to_event_template": { "value": """Hello $attendee_name, You have been added as an attendee to the following calendar event. @@ -75,15 +74,25 @@ You have been added as an attendee to the following calendar event. This is an automated message. Please do not reply.""", "type": str }, - "removed_template": { + "deleted_or_removed_from_event_template": { "value": """Hello $attendee_name, -You have been removed as an attendee from the following calendar event. +The following event has been deleted. $event_title $event_start_time - $event_end_time $event_location +This is an automated message. Please do not reply.""", + "type": str + }, + "updated_event_template": { + "value": """Hello $attendee_name, +The following event has been updated. + $event_title + $event_start_time - $event_end_time + $event_location + This is an automated message. Please do not reply.""", "type": str }, @@ -143,14 +152,22 @@ SMTP_SSL_VERIFY_MODES: Sequence[str] = (SMTP_SSL_VERIFY_MODE_ENUM.NONE.value, SMTP_SSL_VERIFY_MODE_ENUM.REQUIRED.value) -def ics_contents_contains_invited_event(contents: str): +def read_ics_event(contents: str) -> Optional['Event']: """ - Check if the ICS contents contain an event (versus a VTODO or VJOURNAL). + Read the vobject item from the provided string and create an Event. + """ + v_cal: vobject.base.Component = vobject.readOne(contents) + cal: Calendar = Calendar(vobject_item=v_cal) + return cal.event if cal.event else None + + +def ics_contents_contains_event(contents: str): + """ + Check if the ICS contents contain an event (versus a VADDRESSBOOK, VTODO or VJOURNAL). :param contents: The contents of the ICS file. :return: True if the ICS file contains an event, False otherwise. """ - cal = vobject.readOne(contents) - return cal.vevent is not None + return read_ics_event(contents) is not None def extract_email(value: str) -> Optional[str]: @@ -165,6 +182,27 @@ def extract_email(value: str) -> Optional[str]: return value if "@" in value else None +def determine_added_removed_and_unaltered_attendees(original_event: 'Event', + new_event: 'Event') -> ( + Tuple)[List['Attendee'], List['Attendee'], List['Attendee']]: + """ + Determine the added, removed and unaltered attendees between two events. + """ + original_event_attendees = {attendee.email: attendee for attendee in original_event.attendees} + new_event_attendees = {attendee.email: attendee for attendee in new_event.attendees} + # Added attendees are those who are in the new event but not in the original event + added_attendees = [new_event_attendees[email] for email in new_event_attendees if + email not in original_event_attendees] + # Removed attendees are those who are in the original event but not in the new event + removed_attendees = [original_event_attendees[email] for email in original_event_attendees if + email not in new_event_attendees] + # Unaltered attendees are those who are in both events + unaltered_attendees = [original_event_attendees[email] for email in original_event_attendees if + email in new_event_attendees] + + return added_attendees, removed_attendees, unaltered_attendees + + class ContentLine: _key: str value: Any @@ -611,8 +649,9 @@ class EmailConfig: from_email: str, send_mass_emails: bool, dryrun: bool, - added_template: MessageTemplate, - removed_template: MessageTemplate): + new_or_added_to_event_template: MessageTemplate, + deleted_or_removed_from_event_template: MessageTemplate, + updated_event_template: MessageTemplate): self.host = host self.port = port self.security = SMTP_SECURITY_TYPE_ENUM.from_string(value=security) @@ -622,10 +661,9 @@ class EmailConfig: self.from_email = from_email self.send_mass_emails = send_mass_emails self.dryrun = dryrun - self.added_template = added_template - self.removed_template = removed_template - self.updated_template = added_template # Reuse added template for updated events - self.deleted_template = removed_template # Reuse removed template for deleted events + self.new_or_added_to_event_template = new_or_added_to_event_template + self.deleted_or_removed_from_event_template = deleted_or_removed_from_event_template + self.updated_event_template = updated_event_template def __str__(self) -> str: """ @@ -639,26 +677,16 @@ class EmailConfig: def send_added_email(self, attendees: List[Attendee], event: EmailEvent) -> bool: """ - Send a notification for added attendees. + Send a notification for created events (and/or adding attendees). :param attendees: The attendees to inform. - :param event: The event the attendee is being added to. + :param event: The event being created (or the event the attendee is being added to). :return: True if the email was sent successfully, False otherwise. """ ics_attachment = ICSEmailAttachment(file_content=event.ics_content, file_name=f"{event.file_name}") - return self._prepare_and_send_email(template=self.added_template, attendees=attendees, event=event, + return self._prepare_and_send_email(template=self.new_or_added_to_event_template, attendees=attendees, event=event, ics_attachment=ics_attachment) - def send_removed_email(self, attendees: List[Attendee], event: EmailEvent) -> bool: - """ - Send a notification for removed attendees. - :param attendees: The attendees to inform. - :param event: The event the attendee is being removed from. - :return: True if the email was sent successfully, False otherwise. - """ - return self._prepare_and_send_email(template=self.removed_template, attendees=attendees, event=event, - ics_attachment=None) - def send_updated_email(self, attendees: List[Attendee], event: EmailEvent) -> bool: """ Send a notification for updated events. @@ -668,17 +696,17 @@ class EmailConfig: """ ics_attachment = ICSEmailAttachment(file_content=event.ics_content, file_name=f"{event.file_name}") - return self._prepare_and_send_email(template=self.updated_template, attendees=attendees, event=event, + return self._prepare_and_send_email(template=self.updated_event_template, attendees=attendees, event=event, ics_attachment=ics_attachment) def send_deleted_email(self, attendees: List[Attendee], event: EmailEvent) -> bool: """ - Send a notification for deleted events. + Send a notification for deleted events (and/or removing attendees). :param attendees: The attendees to inform. - :param event: The event being deleted. + :param event: The event being deleted (or the event the attendee is being removed from). :return: True if the email was sent successfully, False otherwise. """ - return self._prepare_and_send_email(template=self.deleted_template, attendees=attendees, event=event, + return self._prepare_and_send_email(template=self.deleted_or_removed_from_event_template, attendees=attendees, event=event, ics_attachment=None) def _prepare_and_send_email(self, template: MessageTemplate, attendees: List[Attendee], @@ -825,7 +853,6 @@ def _read_event(vobject_data: str) -> EmailEvent: class Hook(BaseHook): def __init__(self, configuration): super().__init__(configuration) - self.dryrun = self.configuration.get("hook", "dryrun") self.email_config = EmailConfig( host=self.configuration.get("hook", "smtp_server"), port=self.configuration.get("hook", "smtp_port"), @@ -836,14 +863,18 @@ class Hook(BaseHook): from_email=self.configuration.get("hook", "from_email"), send_mass_emails=self.configuration.get("hook", "mass_email"), dryrun=self.configuration.get("hook", "dryrun"), - added_template=MessageTemplate( + new_or_added_to_event_template=MessageTemplate( subject="You have been added to an event", - body=self.configuration.get("hook", "added_template") + body=self.configuration.get("hook", "new_or_added_to_event_template") ), - removed_template=MessageTemplate( - subject="You have been removed from an event", - body=self.configuration.get("hook", "removed_template") + deleted_or_removed_from_event_template=MessageTemplate( + subject="An event you were invited to has been deleted", + body=self.configuration.get("hook", "deleted_or_removed_from_event_template") ), + updated_event_template=MessageTemplate( + subject="An event you are invited to has been updated", + body=self.configuration.get("hook", "updated_event_template") + ) ) logger.info( "Email hook initialized with configuration: %s", @@ -881,50 +912,97 @@ class Hook(BaseHook): return elif notification_type == HookNotificationItemTypes.UPSERT: - # Handle upsert notifications (POST request for new item and PUT for updating existing item) + # Handle upsert notifications - # We don't have access to the original content for a PUT request, just the incoming data + new_item_str: str = notification_item.new_content # type: ignore # A serialized vobject.base.Component + previous_item_str: Optional[str] = notification_item.old_content - item_str: str = notification_item.content # type: ignore # A serialized vobject.base.Component - - if not ics_contents_contains_invited_event(contents=item_str): - # If the ICS file does not contain an event, we do not send any notifications. + if not ics_contents_contains_event(contents=new_item_str): + # If ICS file does not contain an event, do not send any notifications (regardless of previous content). logger.debug("No event found in the ICS file, skipping notification.") return - email_event: EmailEvent = _read_event(vobject_data=item_str) # type: ignore + email_event: EmailEvent = _read_event(vobject_data=new_item_str) # type: ignore - email_success: bool = self.email_config.send_updated_email( # type: ignore - attendees=email_event.event.attendees, - event=email_event - ) - if not email_success: - logger.error("Failed to send some or all email notifications for event: %s", email_event.event.uid) + if not previous_item_str: + # Dealing with a completely new event, no previous content to compare against. + # Email every attendee about the new event. + logger.debug("New event detected, sending notifications to all attendees.") + email_success: bool = self.email_config.send_added_email( # type: ignore + attendees=email_event.event.attendees, + event=email_event + ) + if not email_success: + logger.error("Failed to send some or all added email notifications for event: %s", email_event.event.uid) + return + + # Dealing with an update to an existing event, compare new and previous content. + new_event: Event = read_ics_event(contents=new_item_str) + previous_event: Optional[Event] = read_ics_event(contents=previous_item_str) + if not previous_event: + # If we cannot parse the previous event for some reason, simply treat it as a new event. + logger.warning("Previous event content could not be parsed, treating as a new event.") + email_success: bool = self.email_config.send_added_email( # type: ignore + attendees=email_event.event.attendees, + event=email_event + ) + if not email_success: + logger.error("Failed to send some or all added email notifications for event: %s", email_event.event.uid) + return + + # Determine added, removed, and unaltered attendees + added_attendees, removed_attendees, unaltered_attendees = determine_added_removed_and_unaltered_attendees( + original_event=previous_event, new_event=new_event) + + # Notify added attendees as "event created" + if added_attendees: + email_success: bool = self.email_config.send_added_email( # type: ignore + attendees=added_attendees, + event=email_event + ) + if not email_success: + logger.error("Failed to send some or all added email notifications for event: %s", email_event.event.uid) + + # Notify removed attendees as "event deleted" + if removed_attendees: + email_success: bool = self.email_config.send_deleted_email( # type: ignore + attendees=removed_attendees, + event=email_event + ) + if not email_success: + logger.error("Failed to send some or all removed email notifications for event: %s", email_event.event.uid) + + # Notify unaltered attendees as "event updated" + if unaltered_attendees: + # TODO: Determine WHAT was updated in the event and send a more specific message if needed + # TODO: Don't send an email to unaltered attendees if only change was adding/removing other attendees + email_success: bool = self.email_config.send_updated_email( # type: ignore + attendees=unaltered_attendees, + event=email_event + ) + if not email_success: + logger.error("Failed to send some or all updated email notifications for event: %s", email_event.event.uid) return elif notification_type == HookNotificationItemTypes.DELETE: - # Handle delete notifications (DELETE requests) + # Handle delete notifications - # Ensure it's a delete notification, as we need the old content - if not isinstance(notification_item, DeleteHookNotificationItem): - return + deleted_item_str: str = notification_item.old_content # type: ignore # A serialized vobject.base.Component - item_str: str = notification_item.old_content # type: ignore # A serialized vobject.base.Component - - if not ics_contents_contains_invited_event(contents=item_str): + if not ics_contents_contains_event(contents=deleted_item_str): # If the ICS file does not contain an event, we do not send any notifications. logger.debug("No event found in the ICS file, skipping notification.") return - email_event: EmailEvent = _read_event(vobject_data=item_str) # type: ignore + email_event: EmailEvent = _read_event(vobject_data=deleted_item_str) # type: ignore email_success: bool = self.email_config.send_deleted_email( # type: ignore attendees=email_event.event.attendees, event=email_event ) if not email_success: - logger.error("Failed to send some or all email notifications for event: %s", email_event.event.uid) + logger.error("Failed to send some or all deleted email notifications for event: %s", email_event.event.uid) return diff --git a/radicale/storage/__init__.py b/radicale/storage/__init__.py index b9a6864e..4f7f1be4 100644 --- a/radicale/storage/__init__.py +++ b/radicale/storage/__init__.py @@ -28,7 +28,7 @@ import json import xml.etree.ElementTree as ET from hashlib import sha256 from typing import (Callable, ContextManager, Iterable, Iterator, Mapping, - Optional, Sequence, Set, Tuple, Union, overload) + Optional, Sequence, Set, Tuple, Union, overload, Dict, List) import vobject @@ -175,8 +175,11 @@ class BaseCollection: return False def upload(self, href: str, item: "radicale_item.Item") -> ( - "radicale_item.Item"): - """Upload a new or replace an existing item.""" + "radicale_item.Item", Optional["radicale_item.Item"]): + """Upload a new or replace an existing item. + + Return the uploaded item and the old item if it was replaced. + """ raise NotImplementedError def delete(self, href: Optional[str] = None) -> None: @@ -328,7 +331,8 @@ class BaseStorage: def create_collection( self, href: str, items: Optional[Iterable["radicale_item.Item"]] = None, - props: Optional[Mapping[str, str]] = None) -> BaseCollection: + props: Optional[Mapping[str, str]] = None) -> ( + Tuple)[BaseCollection, Dict[str, "radicale_item.Item"], List[str]]: """Create a collection. ``href`` is the sanitized path. diff --git a/radicale/storage/multifilesystem/create_collection.py b/radicale/storage/multifilesystem/create_collection.py index cbbdee53..6bbb4062 100644 --- a/radicale/storage/multifilesystem/create_collection.py +++ b/radicale/storage/multifilesystem/create_collection.py @@ -19,7 +19,7 @@ import os from tempfile import TemporaryDirectory -from typing import Iterable, Optional, cast +from typing import Iterable, Optional, cast, List, Tuple, Dict import radicale.item as radicale_item from radicale import pathutils @@ -30,9 +30,37 @@ from radicale.storage.multifilesystem.base import StorageBase class StoragePartCreateCollection(StorageBase): + def _discover_existing_items_pre_overwrite(self, + tmp_collection: "multifilesystem.Collection", + dst_path: str) -> Tuple[Dict[str, radicale_item.Item], List[str]]: + """Discover existing items in the collection before overwriting them.""" + existing_items = {} + new_item_hrefs = [] + + existing_collection = self._collection_class( + cast(multifilesystem.Storage, self), + pathutils.unstrip_path(dst_path, True)) + existing_item_hrefs = set(existing_collection._list()) + tmp_collection_hrefs = set(tmp_collection._list()) + for item_href in tmp_collection_hrefs: + if item_href not in existing_item_hrefs: + # Item in temporary collection does not exist in the existing collection (is new) + new_item_hrefs.append(item_href) + continue + # Item exists in both collections, grab the existing item for reference + try: + item = existing_collection._get(item_href, verify_href=False) + if item is not None: + existing_items[item_href] = item + except Exception: + # TODO: Log exception? + continue + + return existing_items, new_item_hrefs + def create_collection(self, href: str, items: Optional[Iterable[radicale_item.Item]] = None, - props=None) -> "multifilesystem.Collection": + props=None) -> Tuple["multifilesystem.Collection", Dict[str, radicale_item.Item], List[str]]: folder = self._get_collection_root_folder() # Path should already be sanitized @@ -44,11 +72,14 @@ class StoragePartCreateCollection(StorageBase): self._makedirs_synced(filesystem_path) return self._collection_class( cast(multifilesystem.Storage, self), - pathutils.unstrip_path(sane_path, True)) + pathutils.unstrip_path(sane_path, True)), {}, [] parent_dir = os.path.dirname(filesystem_path) self._makedirs_synced(parent_dir) + replaced_items: Dict[str, radicale_item.Item] = {} + new_item_hrefs: List[str] = [] + # Create a temporary directory with an unsafe name try: with TemporaryDirectory(prefix=".Radicale.tmp-", dir=parent_dir @@ -68,14 +99,20 @@ class StoragePartCreateCollection(StorageBase): col._upload_all_nonatomic(items, suffix=".vcf") if os.path.lexists(filesystem_path): + replaced_items, new_item_hrefs = self._discover_existing_items_pre_overwrite( + tmp_collection=col, + dst_path=sane_path) pathutils.rename_exchange(tmp_filesystem_path, filesystem_path) else: + # If the destination path does not exist, obviously all items are new + new_item_hrefs = list(col._list()) os.rename(tmp_filesystem_path, filesystem_path) self._sync_directory(parent_dir) except Exception as e: raise ValueError("Failed to create collection %r as %r %s" % (href, filesystem_path, e)) from e + # TODO: Return new-old pairs and just-new items (new vs updated) return self._collection_class( cast(multifilesystem.Storage, self), - pathutils.unstrip_path(sane_path, True)) + pathutils.unstrip_path(sane_path, True)), replaced_items, new_item_hrefs diff --git a/radicale/storage/multifilesystem/upload.py b/radicale/storage/multifilesystem/upload.py index 3814f428..6f163e8b 100644 --- a/radicale/storage/multifilesystem/upload.py +++ b/radicale/storage/multifilesystem/upload.py @@ -21,7 +21,7 @@ import errno import os import pickle import sys -from typing import Iterable, Iterator, TextIO, cast +from typing import Iterable, Iterator, TextIO, cast, Optional, Tuple import radicale.item as radicale_item from radicale import pathutils @@ -36,10 +36,11 @@ class CollectionPartUpload(CollectionPartGet, CollectionPartCache, CollectionPartHistory, CollectionBase): def upload(self, href: str, item: radicale_item.Item - ) -> radicale_item.Item: + ) -> Tuple[radicale_item.Item, Optional[radicale_item.Item]]: if not pathutils.is_safe_filesystem_path_component(href): raise pathutils.UnsafePathError(href) path = pathutils.path_to_filesystem(self._filesystem_path, href) + old_item = self._get(href, verify_href=False) try: with self._atomic_write(path, newline="") as fo: # type: ignore f = cast(TextIO, fo) @@ -67,7 +68,7 @@ class CollectionPartUpload(CollectionPartGet, CollectionPartCache, uploaded_item = self._get(href, verify_href=False) if uploaded_item is None: raise RuntimeError("Storage modified externally") - return uploaded_item + return uploaded_item, old_item def _upload_all_nonatomic(self, items: Iterable[radicale_item.Item], suffix: str = "") -> None: From e391c3aa92c410d6aea72862ea69c142d8ae118c Mon Sep 17 00:00:00 2001 From: Nate Harris Date: Sat, 19 Jul 2025 23:00:25 -0600 Subject: [PATCH 002/290] - Compare previous-current events to determine if non-invitee changes were made (notify non-added/removed attendees of event update) --- radicale/hook/__init__.py | 8 ++-- radicale/hook/email/__init__.py | 79 ++++++++++++++++++++++++++++----- 2 files changed, 71 insertions(+), 16 deletions(-) diff --git a/radicale/hook/__init__.py b/radicale/hook/__init__.py index 04378a2e..009f1b52 100644 --- a/radicale/hook/__init__.py +++ b/radicale/hook/__init__.py @@ -55,16 +55,17 @@ def _cleanup(path): class HookNotificationItem: - def __init__(self, notification_item_type, path, uid=None, new_content=None, old_content=None): + def __init__(self, notification_item_type, path, content=None, uid=None, new_content=None, old_content=None): self.type = notification_item_type.value self.point = _cleanup(path) + self._content_legacy = content self.uid = uid self.new_content = new_content self.old_content = old_content @property def content(self): # For backward compatibility - return self.uid or self.new_content or self.old_content + return self._content_legacy or self.uid or self.new_content or self.old_content @property def replaces_existing_item(self) -> bool: @@ -73,8 +74,7 @@ class HookNotificationItem: def to_json(self): return json.dumps( - self, - default=lambda o: o.__dict__, + {**self.__dict__, "content": self.content}, sort_keys=True, indent=4 ) diff --git a/radicale/hook/email/__init__.py b/radicale/hook/email/__init__.py index 9d80fbd8..0190744d 100644 --- a/radicale/hook/email/__init__.py +++ b/radicale/hook/email/__init__.py @@ -16,6 +16,8 @@ # along with Radicale. If not, see . import enum +import hashlib +import json import re import smtplib import ssl @@ -88,7 +90,9 @@ This is an automated message. Please do not reply.""", }, "updated_event_template": { "value": """Hello $attendee_name, + The following event has been updated. + $event_title $event_start_time - $event_end_time $event_location @@ -203,6 +207,42 @@ def determine_added_removed_and_unaltered_attendees(original_event: 'Event', return added_attendees, removed_attendees, unaltered_attendees +def event_details_other_than_attendees_changed(original_event: 'Event', + new_event: 'Event') -> bool: + """ + Check if any details other than attendees and IDs have changed between two events. + """ + def hash_dict(d: Dict[str, Any]) -> str: + """ + Create a hash of the dictionary to compare contents. + This will ignore None values and empty strings. + """ + return hashlib.sha1(json.dumps(d).encode("utf8")).hexdigest() + + original_event_details = { + "summary": original_event.summary, + "description": original_event.description, + "location": original_event.location, + "datetime_start": original_event.datetime_start.time_string() if original_event.datetime_start else None, + "datetime_end": original_event.datetime_end.time_string() if original_event.datetime_end else None, + "duration": original_event.duration, + "status": original_event.status, + "organizer": original_event.organizer + } + new_event_details = { + "summary": new_event.summary, + "description": new_event.description, + "location": new_event.location, + "datetime_start": new_event.datetime_start.time_string() if new_event.datetime_start else None, + "datetime_end": new_event.datetime_end.time_string() if new_event.datetime_end else None, + "duration": new_event.duration, + "status": new_event.status, + "organizer": new_event.organizer + } + + return hash_dict(original_event_details) != hash_dict(new_event_details) + + class ContentLine: _key: str value: Any @@ -453,6 +493,11 @@ class Event(VComponent): """Return the summary of the event.""" return self._get_content_lines("SUMMARY")[0].value + @property + def description(self) -> Optional[str]: + """Return the description of the event.""" + return self._get_content_lines("DESCRIPTION")[0].value + @property def location(self) -> Optional[str]: """Return the location of the event.""" @@ -684,7 +729,8 @@ class EmailConfig: """ ics_attachment = ICSEmailAttachment(file_content=event.ics_content, file_name=f"{event.file_name}") - return self._prepare_and_send_email(template=self.new_or_added_to_event_template, attendees=attendees, event=event, + return self._prepare_and_send_email(template=self.new_or_added_to_event_template, attendees=attendees, + event=event, ics_attachment=ics_attachment) def send_updated_email(self, attendees: List[Attendee], event: EmailEvent) -> bool: @@ -706,7 +752,8 @@ class EmailConfig: :param event: The event being deleted (or the event the attendee is being removed from). :return: True if the email was sent successfully, False otherwise. """ - return self._prepare_and_send_email(template=self.deleted_or_removed_from_event_template, attendees=attendees, event=event, + return self._prepare_and_send_email(template=self.deleted_or_removed_from_event_template, attendees=attendees, + event=event, ics_attachment=None) def _prepare_and_send_email(self, template: MessageTemplate, attendees: List[Attendee], @@ -933,7 +980,8 @@ class Hook(BaseHook): event=email_event ) if not email_success: - logger.error("Failed to send some or all added email notifications for event: %s", email_event.event.uid) + logger.error("Failed to send some or all added email notifications for event: %s", + email_event.event.uid) return # Dealing with an update to an existing event, compare new and previous content. @@ -947,7 +995,8 @@ class Hook(BaseHook): event=email_event ) if not email_success: - logger.error("Failed to send some or all added email notifications for event: %s", email_event.event.uid) + logger.error("Failed to send some or all added email notifications for event: %s", + email_event.event.uid) return # Determine added, removed, and unaltered attendees @@ -961,7 +1010,8 @@ class Hook(BaseHook): event=email_event ) if not email_success: - logger.error("Failed to send some or all added email notifications for event: %s", email_event.event.uid) + logger.error("Failed to send some or all added email notifications for event: %s", + email_event.event.uid) # Notify removed attendees as "event deleted" if removed_attendees: @@ -970,18 +1020,22 @@ class Hook(BaseHook): event=email_event ) if not email_success: - logger.error("Failed to send some or all removed email notifications for event: %s", email_event.event.uid) + logger.error("Failed to send some or all removed email notifications for event: %s", + email_event.event.uid) - # Notify unaltered attendees as "event updated" - if unaltered_attendees: - # TODO: Determine WHAT was updated in the event and send a more specific message if needed - # TODO: Don't send an email to unaltered attendees if only change was adding/removing other attendees + # Notify unaltered attendees as "event updated" if details other than attendees have changed + if unaltered_attendees and event_details_other_than_attendees_changed(original_event=previous_event, + new_event=new_event): email_success: bool = self.email_config.send_updated_email( # type: ignore attendees=unaltered_attendees, event=email_event ) if not email_success: - logger.error("Failed to send some or all updated email notifications for event: %s", email_event.event.uid) + logger.error("Failed to send some or all updated email notifications for event: %s", + email_event.event.uid) + + # Skip sending notifications to existing attendees if the only changes made to the event + # were the addition/removal of other attendees. return @@ -1002,7 +1056,8 @@ class Hook(BaseHook): event=email_event ) if not email_success: - logger.error("Failed to send some or all deleted email notifications for event: %s", email_event.event.uid) + logger.error("Failed to send some or all deleted email notifications for event: %s", + email_event.event.uid) return From dd5bbfb9e35f6cdca13336e106a993f477002615 Mon Sep 17 00:00:00 2001 From: Nate Harris Date: Sat, 19 Jul 2025 23:12:54 -0600 Subject: [PATCH 003/290] - Update documentation --- DOCUMENTATION.md | 42 +++++++++++++++++++++++++++++++++++++----- config | 3 +++ 2 files changed, 40 insertions(+), 5 deletions(-) diff --git a/DOCUMENTATION.md b/DOCUMENTATION.md index 6527fb14..256a5bfa 100644 --- a/DOCUMENTATION.md +++ b/DOCUMENTATION.md @@ -1630,11 +1630,11 @@ When enabled, send one email to all attendee email addresses. When disabled, sen Default: `False` -##### added_template +##### new_or_added_to_event_template _(>= 3.5.5)_ -Template to use for added/updated event email body. +Template to use for added/updated event email body (sent to an attendee when the event is created or they are added to a pre-existing event). The following placeholders will be replaced: - `$organizer_name`: Name of the organizer, or "Unknown Organizer" if not set in event @@ -1660,11 +1660,11 @@ You have been added as an attendee to the following calendar event. This is an automated message. Please do not reply. ``` -##### removed_template +##### deleted_or_removed_from_event_template _(>= 3.5.5)_ -Template to use for deleted event email body. +Template to use for deleted/removed event email body (sent to an attendee when the event is deleted or they are removed from the event). The following placeholders will be replaced: - `$organizer_name`: Name of the organizer, or "Unknown Organizer" if not set in event @@ -1681,7 +1681,7 @@ Default: ``` Hello $attendee_name, -You have been removed as an attendee from the following calendar event. +The following event has been deleted. $event_title $event_start_time - $event_end_time @@ -1690,6 +1690,38 @@ You have been removed as an attendee from the following calendar event. This is an automated message. Please do not reply. ``` +#### updated_event_template + +_(>= 3.5.5)_ + +Template to use for updated event email body (sent to an attendee when non-attendee-related details of the event are updated). + +Existing attendees will NOT be notified of a modified event if the only changes are adding/removing other attendees. + +The following placeholders will be replaced: +- `$organizer_name`: Name of the organizer, or "Unknown Organizer" if not set in event +- `$from_email`: Email address the email is sent from +- `$attendee_name`: Name of the attendee (email recipient), or "everyone" if mass email enabled. +- `$event_name`: Name/summary of the event, or "No Title" if not set in event +- `$event_start_time`: Start time of the event in ISO 8601 format +- `$event_end_time`: End time of the event in ISO 8601 format, or "No End Time" if the event has no end time +- `$event_location`: Location of the event, or "No Location Specified" if not set in event + +Providing any words prefixed with $ not included in the list above will result in an error. + +Default: +``` +Hello $attendee_name, + +The following event has been updated. + + $event_title + $event_start_time - $event_end_time + $event_location + +This is an automated message. Please do not reply. +``` + #### reporting ##### max_freebusy_occurrence diff --git a/config b/config index d3e2283f..c38457df 100644 --- a/config +++ b/config @@ -334,6 +334,9 @@ #smtp_password = #from_email = #mass_email = False +#new_or_added_to_event_template = +#deleted_or_removed_from_event_template = +#updated_event_template = [reporting] From 16b7311229e06544ec9ec4354e5bd5f570fb9014 Mon Sep 17 00:00:00 2001 From: Nate Harris Date: Sat, 19 Jul 2025 23:29:18 -0600 Subject: [PATCH 004/290] - Include legacy "content" parameter in HookNotificationItem usage --- radicale/app/delete.py | 2 ++ radicale/app/proppatch.py | 12 ++++++++---- radicale/app/put.py | 27 ++++++++++++--------------- 3 files changed, 22 insertions(+), 19 deletions(-) diff --git a/radicale/app/delete.py b/radicale/app/delete.py index d6cdbfcb..060abb18 100644 --- a/radicale/app/delete.py +++ b/radicale/app/delete.py @@ -85,6 +85,7 @@ class ApplicationPartDelete(ApplicationBase): HookNotificationItem( notification_item_type=HookNotificationItemTypes.DELETE, path=access.path, + content=i.uid, uid=i.uid, old_content=item.serialize(), # type: ignore new_content=None @@ -98,6 +99,7 @@ class ApplicationPartDelete(ApplicationBase): HookNotificationItem( notification_item_type=HookNotificationItemTypes.DELETE, path=access.path, + content=item.uid, uid=item.uid, old_content=item.serialize(), # type: ignore new_content=None, diff --git a/radicale/app/proppatch.py b/radicale/app/proppatch.py index 99ec6ae0..2e8eed47 100644 --- a/radicale/app/proppatch.py +++ b/radicale/app/proppatch.py @@ -101,13 +101,17 @@ class ApplicationPartProppatch(ApplicationBase): xml_answer = xml_proppatch(base_prefix, path, xml_content, item) if xml_content is not None: + content = DefusedET.tostring( + xml_content, + encoding=self._encoding + ).decode(encoding=self._encoding) hook_notification_item = HookNotificationItem( notification_item_type=HookNotificationItemTypes.CPATCH, path=access.path, - new_content=DefusedET.tostring( - xml_content, - encoding=self._encoding - ).decode(encoding=self._encoding) + content=content, + uid=None, + old_content=None, + new_content=content ) self._hook.notify(hook_notification_item) except ValueError as e: diff --git a/radicale/app/put.py b/radicale/app/put.py index 575134c6..46e957c0 100644 --- a/radicale/app/put.py +++ b/radicale/app/put.py @@ -254,6 +254,8 @@ class ApplicationPartPut(ApplicationBase): hook_notification_item = HookNotificationItem( notification_item_type=HookNotificationItemTypes.UPSERT, path=access.path, + content=existing_item.serialize(), + uid=None, old_content=existing_item.serialize(), new_content=item.serialize() ) @@ -261,6 +263,8 @@ class ApplicationPartPut(ApplicationBase): hook_notification_item = HookNotificationItem( notification_item_type=HookNotificationItemTypes.UPSERT, path=access.path, + content=item.serialize(), + uid=None, old_content=None, new_content=item.serialize() ) @@ -282,21 +286,14 @@ class ApplicationPartPut(ApplicationBase): try: uploaded_item, replaced_item = parent_item.upload(href, prepared_item) etag = uploaded_item.etag - if replaced_item: - # If the item was replaced, we notify with the old content - hook_notification_item = HookNotificationItem( - notification_item_type=HookNotificationItemTypes.UPSERT, - path=access.path, - old_content=replaced_item.serialize(), - new_content=prepared_item.serialize() - ) - else: # If it was a new item, we notify with no old content - hook_notification_item = HookNotificationItem( - notification_item_type=HookNotificationItemTypes.UPSERT, - path=access.path, - old_content=None, - new_content=prepared_item.serialize() - ) + hook_notification_item = HookNotificationItem( + notification_item_type=HookNotificationItemTypes.UPSERT, + path=access.path, + content=prepared_item.serialize(), + uid=None, + old_content=replaced_item.serialize() if replaced_item else None, + new_content=prepared_item.serialize() + ) self._hook.notify(hook_notification_item) except ValueError as e: # return better matching HTTP result in case errno is provided and catched From 5c9c5b1572216ebae7c84b3eb76d861cd454eb7d Mon Sep 17 00:00:00 2001 From: Nate Harris Date: Sat, 19 Jul 2025 23:38:37 -0600 Subject: [PATCH 005/290] - Linting --- radicale/app/put.py | 2 +- radicale/config.py | 2 +- radicale/hook/email/__init__.py | 5 +++-- radicale/storage/__init__.py | 22 +++++++++++-------- .../multifilesystem/create_collection.py | 2 +- radicale/storage/multifilesystem/upload.py | 2 +- 6 files changed, 20 insertions(+), 15 deletions(-) diff --git a/radicale/app/put.py b/radicale/app/put.py index 46e957c0..d7818eaa 100644 --- a/radicale/app/put.py +++ b/radicale/app/put.py @@ -249,7 +249,7 @@ class ApplicationPartPut(ApplicationBase): props=props) for item in prepared_items: # Try to grab the previously-existing item by href - existing_item = replaced_items.get(item.href, None) + existing_item = replaced_items.get(item.href, None) # type: ignore if existing_item: hook_notification_item = HookNotificationItem( notification_item_type=HookNotificationItemTypes.UPSERT, diff --git a/radicale/config.py b/radicale/config.py index 62a3c5ce..c6d93f41 100644 --- a/radicale/config.py +++ b/radicale/config.py @@ -507,7 +507,7 @@ The following event has been updated. $event_title $event_start_time - $event_end_time $event_location - + This is an automated message. Please do not reply.""", "help": "Template for the email sent when an event is updated. Select placeholder words prefixed with $ will be replaced", "type": str diff --git a/radicale/hook/email/__init__.py b/radicale/hook/email/__init__.py index 0190744d..27c1de24 100644 --- a/radicale/hook/email/__init__.py +++ b/radicale/hook/email/__init__.py @@ -31,7 +31,8 @@ from typing import Any, Dict, List, Optional, Sequence, Tuple import vobject -from radicale.hook import (BaseHook, HookNotificationItem, HookNotificationItemTypes) +from radicale.hook import (BaseHook, HookNotificationItem, + HookNotificationItemTypes) from radicale.log import logger PLUGIN_CONFIG_SCHEMA = { @@ -985,7 +986,7 @@ class Hook(BaseHook): return # Dealing with an update to an existing event, compare new and previous content. - new_event: Event = read_ics_event(contents=new_item_str) + new_event: Event = read_ics_event(contents=new_item_str) # type: ignore previous_event: Optional[Event] = read_ics_event(contents=previous_item_str) if not previous_event: # If we cannot parse the previous event for some reason, simply treat it as a new event. diff --git a/radicale/storage/__init__.py b/radicale/storage/__init__.py index 4f7f1be4..78f9a7d4 100644 --- a/radicale/storage/__init__.py +++ b/radicale/storage/__init__.py @@ -27,8 +27,8 @@ Take a look at the class ``BaseCollection`` if you want to implement your own. import json import xml.etree.ElementTree as ET from hashlib import sha256 -from typing import (Callable, ContextManager, Iterable, Iterator, Mapping, - Optional, Sequence, Set, Tuple, Union, overload, Dict, List) +from typing import (Callable, ContextManager, Dict, Iterable, Iterator, List, + Mapping, Optional, Sequence, Set, Tuple, Union, overload) import vobject @@ -44,7 +44,8 @@ INTERNAL_TYPES: Sequence[str] = ("multifilesystem", "multifilesystem_nolock",) # NOTE: change only if cache structure is modified to avoid cache invalidation on update CACHE_VERSION_RADICALE = "3.3.1" -CACHE_VERSION: bytes = ("%s=%s;%s=%s;" % ("radicale", CACHE_VERSION_RADICALE, "vobject", utils.package_version("vobject"))).encode() +CACHE_VERSION: bytes = ( + "%s=%s;%s=%s;" % ("radicale", CACHE_VERSION_RADICALE, "vobject", utils.package_version("vobject"))).encode() def load(configuration: "config.Configuration") -> "BaseStorage": @@ -112,17 +113,18 @@ class BaseCollection: invalid. """ + def hrefs_iter() -> Iterator[str]: for item in self.get_all(): assert item.href yield item.href + token = "http://radicale.org/ns/sync/%s" % self.etag.strip("\"") if old_token: raise ValueError("Sync token are not supported") return token, hrefs_iter() - def get_multi(self, hrefs: Iterable[str] - ) -> Iterable[Tuple[str, Optional["radicale_item.Item"]]]: + def get_multi(self, hrefs: Iterable[str]) -> Iterable[Tuple[str, Optional["radicale_item.Item"]]]: """Fetch multiple items. It's not required to return the requested items in the correct order. @@ -175,7 +177,7 @@ class BaseCollection: return False def upload(self, href: str, item: "radicale_item.Item") -> ( - "radicale_item.Item", Optional["radicale_item.Item"]): + Tuple)["radicale_item.Item", Optional["radicale_item.Item"]]: """Upload a new or replace an existing item. Return the uploaded item and the old item if it was replaced. @@ -191,10 +193,12 @@ class BaseCollection: raise NotImplementedError @overload - def get_meta(self, key: None = None) -> Mapping[str, str]: ... + def get_meta(self, key: None = None) -> Mapping[str, str]: + ... @overload - def get_meta(self, key: str) -> Optional[str]: ... + def get_meta(self, key: str) -> Optional[str]: + ... def get_meta(self, key: Optional[str] = None ) -> Union[Mapping[str, str], Optional[str]]: @@ -297,7 +301,7 @@ class BaseStorage: def discover( self, path: str, depth: str = "0", child_context_manager: Optional[ - Callable[[str, Optional[str]], ContextManager[None]]] = None, + Callable[[str, Optional[str]], ContextManager[None]]] = None, user_groups: Set[str] = set([])) -> Iterable["types.CollectionOrItem"]: """Discover a list of collections under the given ``path``. diff --git a/radicale/storage/multifilesystem/create_collection.py b/radicale/storage/multifilesystem/create_collection.py index 6bbb4062..71aca377 100644 --- a/radicale/storage/multifilesystem/create_collection.py +++ b/radicale/storage/multifilesystem/create_collection.py @@ -19,7 +19,7 @@ import os from tempfile import TemporaryDirectory -from typing import Iterable, Optional, cast, List, Tuple, Dict +from typing import Dict, Iterable, List, Optional, Tuple, cast import radicale.item as radicale_item from radicale import pathutils diff --git a/radicale/storage/multifilesystem/upload.py b/radicale/storage/multifilesystem/upload.py index 6f163e8b..674477c7 100644 --- a/radicale/storage/multifilesystem/upload.py +++ b/radicale/storage/multifilesystem/upload.py @@ -21,7 +21,7 @@ import errno import os import pickle import sys -from typing import Iterable, Iterator, TextIO, cast, Optional, Tuple +from typing import Iterable, Iterator, Optional, TextIO, Tuple, cast import radicale.item as radicale_item from radicale import pathutils From 74bc78aac434858a7f63cef1a9f35288ada042d6 Mon Sep 17 00:00:00 2001 From: Nate Harris Date: Sat, 19 Jul 2025 23:38:37 -0600 Subject: [PATCH 006/290] - Linting --- radicale/hook/email/__init__.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/radicale/hook/email/__init__.py b/radicale/hook/email/__init__.py index 27c1de24..90fb03a9 100644 --- a/radicale/hook/email/__init__.py +++ b/radicale/hook/email/__init__.py @@ -91,13 +91,13 @@ This is an automated message. Please do not reply.""", }, "updated_event_template": { "value": """Hello $attendee_name, - + The following event has been updated. $event_title $event_start_time - $event_end_time $event_location - + This is an automated message. Please do not reply.""", "type": str }, From 208dd22a421621aa53cf21eaaa6c2d58f4cbf276 Mon Sep 17 00:00:00 2001 From: Nate Harris Date: Mon, 28 Jul 2025 01:19:05 -0600 Subject: [PATCH 007/290] - Do not send notifications if end time is more than 1 minute in the past (buffer) --- radicale/hook/email/__init__.py | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/radicale/hook/email/__init__.py b/radicale/hook/email/__init__.py index 90fb03a9..cbf1523b 100644 --- a/radicale/hook/email/__init__.py +++ b/radicale/hook/email/__init__.py @@ -213,6 +213,7 @@ def event_details_other_than_attendees_changed(original_event: 'Event', """ Check if any details other than attendees and IDs have changed between two events. """ + def hash_dict(d: Dict[str, Any]) -> str: """ Create a hash of the dictionary to compare contents. @@ -971,6 +972,20 @@ class Hook(BaseHook): return email_event: EmailEvent = _read_event(vobject_data=new_item_str) # type: ignore + if not email_event: + logger.error("Failed to read event from new content: %s", new_item_str) + return + email_event_event = email_event.event # type: ignore + if not email_event_event: + logger.error("Event could not be parsed from the new content: %s", new_item_str) + return + email_event_end_time = email_event_event.datetime_end # type: ignore + # Skip notification if the event end time is more than 1 minute in the past. + if email_event_end_time and email_event_end_time.time and email_event_end_time.time < ( + datetime.now() - timedelta(minutes=1)): + logger.warning("Event end time is in the past, skipping notification for event: %s", + email_event_event.uid) + return if not previous_item_str: # Dealing with a completely new event, no previous content to compare against. From f32e50bc9dae630ffac3dfd21c39a5f98fc5f915 Mon Sep 17 00:00:00 2001 From: Nate Harris Date: Thu, 14 Aug 2025 00:06:55 -0600 Subject: [PATCH 008/290] - Add unit tests to confirm emails not triggered when adding/deleting event with past end date --- radicale/storage/__init__.py | 3 +- radicale/tests/test_hook_email.py | 56 +++++++++++++++++++++++++++++-- 2 files changed, 55 insertions(+), 4 deletions(-) diff --git a/radicale/storage/__init__.py b/radicale/storage/__init__.py index 78f9a7d4..fb17f834 100644 --- a/radicale/storage/__init__.py +++ b/radicale/storage/__init__.py @@ -300,8 +300,7 @@ class BaseStorage: def discover( self, path: str, depth: str = "0", - child_context_manager: Optional[ - Callable[[str, Optional[str]], ContextManager[None]]] = None, + child_context_manager: Optional[Callable[[str, Optional[str]], ContextManager[None]]] = None, user_groups: Set[str] = set([])) -> Iterable["types.CollectionOrItem"]: """Discover a list of collections under the given ``path``. diff --git a/radicale/tests/test_hook_email.py b/radicale/tests/test_hook_email.py index 74674589..af012a6c 100644 --- a/radicale/tests/test_hook_email.py +++ b/radicale/tests/test_hook_email.py @@ -21,6 +21,8 @@ Radicale tests related to hook 'email' import logging import os +import re +from datetime import datetime, timedelta from radicale.tests import BaseTest from radicale.tests.helpers import get_file_content @@ -63,11 +65,26 @@ permissions: RrWw""") self.configure({"hook": {"type": "email", "dryrun": "True"}}) - def test_add_event(self, caplog) -> None: + def _future_date_timestamp(self) -> str: + """Return a date timestamp for a future date.""" + future_date = datetime.now() + timedelta(days=1) + return future_date.strftime("%Y%m%dT%H%M%S") + + def _past_date_timestamp(self) -> str: + past_date = datetime.now() - timedelta(days=1) + return past_date.strftime("%Y%m%dT%H%M%S") + + def _replace_end_date_in_event(self, event: str, new_date: str) -> str: + """Replace the end date in an event string.""" + return re.sub(r"DTEND;TZID=Europe/Paris:\d{8}T\d{6}", + f"DTEND;TZID=Europe/Paris:{new_date}", event) + + def test_add_event_with_future_end_date(self, caplog) -> None: caplog.set_level(logging.WARNING) """Add an event.""" self.mkcalendar("/calendar.ics/") event = get_file_content("event1.ics") + event = self._replace_end_date_in_event(event, self._future_date_timestamp()) path = "/calendar.ics/event1.ics" self.put(path, event) _, headers, answer = self.request("GET", path, check=200) @@ -87,11 +104,30 @@ permissions: RrWw""") if (found != 7): raise ValueError("Logging misses expected log lines, found=%d", found) - def test_delete_event(self, caplog) -> None: + def test_add_event_with_past_end_date(self, caplog) -> None: + caplog.set_level(logging.WARNING) + """Add an event.""" + self.mkcalendar("/calendar.ics/") + event = get_file_content("event1.ics") + event = self._replace_end_date_in_event(event, self._past_date_timestamp()) + path = "/calendar.ics/event1.ics" + self.put(path, event) + _, headers, answer = self.request("GET", path, check=200) + assert "ETag" in headers + assert headers["Content-Type"] == "text/calendar; charset=utf-8" + assert "VEVENT" in answer + assert "Event" in answer + assert "UID:event" in answer + + # Should not trigger an email + assert len(caplog.messages) == 0 + + def test_delete_event_with_future_end_date(self, caplog) -> None: caplog.set_level(logging.WARNING) """Delete an event.""" self.mkcalendar("/calendar.ics/") event = get_file_content("event1.ics") + event = self._replace_end_date_in_event(event, self._future_date_timestamp()) path = "/calendar.ics/event1.ics" self.put(path, event) _, responses = self.delete(path) @@ -108,3 +144,19 @@ permissions: RrWw""") found = found | 4 if (found != 7): raise ValueError("Logging misses expected log lines, found=%d", found) + + def test_delete_event_with_past_end_date(self, caplog) -> None: + caplog.set_level(logging.WARNING) + """Delete an event.""" + self.mkcalendar("/calendar.ics/") + event = get_file_content("event1.ics") + event = self._replace_end_date_in_event(event, self._past_date_timestamp()) + path = "/calendar.ics/event1.ics" + self.put(path, event) + _, responses = self.delete(path) + assert responses[path] == 200 + _, answer = self.get("/calendar.ics/") + assert "VEVENT" not in answer + + # Should not trigger an email + assert len(caplog.messages) == 0 From 9b6ba72fa023e38c1f5e7ad1596d193f19bad86b Mon Sep 17 00:00:00 2001 From: Nate Harris Date: Thu, 14 Aug 2025 00:10:16 -0600 Subject: [PATCH 009/290] - Fix dryrun property --- radicale/hook/email/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/radicale/hook/email/__init__.py b/radicale/hook/email/__init__.py index cbf1523b..62bf3195 100644 --- a/radicale/hook/email/__init__.py +++ b/radicale/hook/email/__init__.py @@ -946,7 +946,7 @@ class Hook(BaseHook): :type notification_item: HookNotificationItem :return: None """ - if self.dryrun: + if self.email_config.dryrun: logger.warning("Hook 'email': DRY-RUN received notification_item: %r", vars(notification_item)) else: logger.debug("Received notification_item: %r", vars(notification_item)) From 7ce41aee37ea82639795af41e3f7d51b0daaff70 Mon Sep 17 00:00:00 2001 From: Georgiy Date: Sun, 17 Aug 2025 20:06:37 +0300 Subject: [PATCH 010/290] (#1845) Fix expanded item copying --- radicale/app/report.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/radicale/app/report.py b/radicale/app/report.py index 752d04a7..555154c3 100644 --- a/radicale/app/report.py +++ b/radicale/app/report.py @@ -475,7 +475,7 @@ def _expand( if not vevent: # Create new instance from recurrence - vevent = copy.deepcopy(base_vevent) + vevent = base_vevent.duplicate(base_vevent) # For all day events, the system timezone may influence the # results, so use recurrence_dt From 998b2e2121480b2c9cebedc0cb9901cfe83307f6 Mon Sep 17 00:00:00 2001 From: Nate Harris Date: Thu, 21 Aug 2025 00:21:11 -0600 Subject: [PATCH 011/290] - Fix unit tests for hook email trigger conditional based on end date --- radicale/hook/email/__init__.py | 11 +++++--- radicale/tests/test_hook_email.py | 46 +++++++++++++++---------------- 2 files changed, 29 insertions(+), 28 deletions(-) diff --git a/radicale/hook/email/__init__.py b/radicale/hook/email/__init__.py index 62bf3195..2defaa95 100644 --- a/radicale/hook/email/__init__.py +++ b/radicale/hook/email/__init__.py @@ -981,10 +981,13 @@ class Hook(BaseHook): return email_event_end_time = email_event_event.datetime_end # type: ignore # Skip notification if the event end time is more than 1 minute in the past. - if email_event_end_time and email_event_end_time.time and email_event_end_time.time < ( - datetime.now() - timedelta(minutes=1)): - logger.warning("Event end time is in the past, skipping notification for event: %s", - email_event_event.uid) + if email_event_end_time and email_event_end_time.time: + event_end = email_event_end_time.time # type: ignore + now = datetime.now( + event_end.tzinfo) if event_end.tzinfo else datetime.now() # Handle timezone-aware datetime + if event_end < (now - timedelta(minutes=1)): + logger.warning("Event end time is in the past, skipping notification for event: %s", + email_event_event.uid) return if not previous_item_str: diff --git a/radicale/tests/test_hook_email.py b/radicale/tests/test_hook_email.py index af012a6c..b7fef935 100644 --- a/radicale/tests/test_hook_email.py +++ b/radicale/tests/test_hook_email.py @@ -93,16 +93,12 @@ permissions: RrWw""") assert "VEVENT" in answer assert "Event" in answer assert "UID:event" in answer - found = 0 - for line in caplog.messages: - if line.find("notification_item: {'type': 'upsert'") != -1: - found = found | 1 - if line.find("to_addresses=['janedoe@example.com']") != -1: - found = found | 2 - if line.find("to_addresses=['johndoe@example.com']") != -1: - found = found | 4 - if (found != 7): - raise ValueError("Logging misses expected log lines, found=%d", found) + + logs = caplog.messages + # Should have a log saying the notification item was received + assert len([log for log in logs if "received notification_item: {'type': 'upsert'," in log]) == 1 + # Should NOT have a log saying that no email is sent (email won't actually be sent due to dryrun) + assert len([log for log in logs if "skipping notification for event: event1" in log]) == 0 def test_add_event_with_past_end_date(self, caplog) -> None: caplog.set_level(logging.WARNING) @@ -119,8 +115,11 @@ permissions: RrWw""") assert "Event" in answer assert "UID:event" in answer - # Should not trigger an email - assert len(caplog.messages) == 0 + logs = caplog.messages + # Should have a log saying the notification item was received + assert len([log for log in logs if "received notification_item: {'type': 'upsert'," in log]) == 1 + # Should have a log saying that no email is sent due to past end date + assert len([log for log in logs if "Event end time is in the past, skipping notification for event: event1" in log]) == 1 def test_delete_event_with_future_end_date(self, caplog) -> None: caplog.set_level(logging.WARNING) @@ -134,16 +133,12 @@ permissions: RrWw""") assert responses[path] == 200 _, answer = self.get("/calendar.ics/") assert "VEVENT" not in answer - found = 0 - for line in caplog.messages: - if line.find("notification_item: {'type': 'delete'") != -1: - found = found | 1 - if line.find("to_addresses=['janedoe@example.com']") != -1: - found = found | 2 - if line.find("to_addresses=['johndoe@example.com']") != -1: - found = found | 4 - if (found != 7): - raise ValueError("Logging misses expected log lines, found=%d", found) + + logs = caplog.messages + # Should have a log saying the notification item was received + assert len([log for log in logs if "received notification_item: {'type': 'delete'," in log]) == 1 + # Should NOT have a log saying that no email is sent (email won't actually be sent due to dryrun) + assert len([log for log in logs if "skipping notification for event: event1" in log]) == 0 def test_delete_event_with_past_end_date(self, caplog) -> None: caplog.set_level(logging.WARNING) @@ -158,5 +153,8 @@ permissions: RrWw""") _, answer = self.get("/calendar.ics/") assert "VEVENT" not in answer - # Should not trigger an email - assert len(caplog.messages) == 0 + logs = caplog.messages + # Should have a log saying the notification item was received + assert len([log for log in logs if "received notification_item: {'type': 'delete'," in log]) == 1 + # Should have a log saying that no email is sent due to past end date + assert len([log for log in logs if "Event end time is in the past, skipping notification for event: event1" in log]) == 1 From 74d21f011c89398fc6f2b976a764ff86e3f4e61c Mon Sep 17 00:00:00 2001 From: Peter Bieringer Date: Fri, 22 Aug 2025 07:49:09 +0200 Subject: [PATCH 012/290] enrich for optional tzinfo --- radicale/item/filter.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/radicale/item/filter.py b/radicale/item/filter.py index b846023a..ef43dbcc 100644 --- a/radicale/item/filter.py +++ b/radicale/item/filter.py @@ -47,7 +47,7 @@ else: TRIGGER = datetime | None -def date_to_datetime(d: date) -> datetime: +def date_to_datetime(d: date, tzinfo=vobject.icalendar.utc) -> datetime: """Transform any date to a UTC datetime. If ``d`` is a datetime without timezone, return as UTC datetime. If ``d`` @@ -58,7 +58,7 @@ def date_to_datetime(d: date) -> datetime: d = datetime.combine(d, datetime.min.time()) if not d.tzinfo: # NOTE: using vobject's UTC as it wasn't playing well with datetime's. - d = d.replace(tzinfo=vobject.icalendar.utc) + d = d.replace(tzinfo=tzinfo) return d From e1b19f1a2227b5713e8437b0e7760d4ffa023c2b Mon Sep 17 00:00:00 2001 From: Peter Bieringer Date: Fri, 22 Aug 2025 07:49:54 +0200 Subject: [PATCH 013/290] catch items having tzinfo only on dtstart or dtend set for whatever reason, overtake tzinfo from the other one --- radicale/item/filter.py | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/radicale/item/filter.py b/radicale/item/filter.py index ef43dbcc..94cdc015 100644 --- a/radicale/item/filter.py +++ b/radicale/item/filter.py @@ -366,6 +366,21 @@ def visit_time_ranges(vobject_item: vobject.base.Component, child_name: str, dtend = getattr(child, "dtend", None) if dtend is not None: dtend = dtend.value + + # Ensure that both datetime.datetime objects have a timezone or + # both do not have one before doing calculations. This is required + # as the library does not support performing mathematical operations + # on timezone-aware and timezone-naive objects. See #1847 + if hasattr(dtstart, 'tzinfo') and hasattr(dtend, 'tzinfo'): + if dtstart.tzinfo is None and dtend.tzinfo is not None: + dtstart_orig = dtstart + dtstart = date_to_datetime(dtstart, dtend.astimezone().tzinfo) + logger.debug("TRACE/ITEM/FILTER/get_children: overtake missing tzinfo on dtstart from dtend: '%s' -> '%s'", dtstart_orig, dtstart) + elif dtstart.tzinfo is not None and dtend.tzinfo is None: + dtend_orig = dtend + dtend = date_to_datetime(dtend, dtstart.astimezone().tzinfo) + logger.debug("TRACE/ITEM/FILTER/get_children: overtake missing tzinfo on dtend from dtstart: '%s' -> '%s'", dtend_orig, dtend) + original_duration = (dtend - dtstart).total_seconds() dtend = date_to_datetime(dtend) From 2a808fd37391de4e4092444f0e22db11b5a00225 Mon Sep 17 00:00:00 2001 From: Peter Bieringer Date: Fri, 22 Aug 2025 07:50:47 +0200 Subject: [PATCH 014/290] test items having tzinfo only on dtstart or dtend set for whatever reason --- radicale/tests/static/event_issue1847_1.ics | 14 ++++++++++++++ radicale/tests/static/event_issue1847_2.ics | 14 ++++++++++++++ 2 files changed, 28 insertions(+) create mode 100644 radicale/tests/static/event_issue1847_1.ics create mode 100644 radicale/tests/static/event_issue1847_2.ics diff --git a/radicale/tests/static/event_issue1847_1.ics b/radicale/tests/static/event_issue1847_1.ics new file mode 100644 index 00000000..121c0c6c --- /dev/null +++ b/radicale/tests/static/event_issue1847_1.ics @@ -0,0 +1,14 @@ +BEGIN:VCALENDAR +VERSION:2.0 +PRODID:-//algoo.fr//NONSGML Open Calendar v0.9//EN +BEGIN:VEVENT +CREATED:20250814T153429Z +LAST-MODIFIED:20250814T153503Z +DTSTAMP:20250814T153503Z +UID:f91964cb-53ca-4942-8811-c38f076f4328 +SUMMARY:error +DTSTART:20250814T180000 +DTEND;TZID=Europe/Brussels:20250814T190000 +TRANSP:OPAQUE +END:VEVENT +END:VCALENDAR diff --git a/radicale/tests/static/event_issue1847_2.ics b/radicale/tests/static/event_issue1847_2.ics new file mode 100644 index 00000000..03d09b49 --- /dev/null +++ b/radicale/tests/static/event_issue1847_2.ics @@ -0,0 +1,14 @@ +BEGIN:VCALENDAR +VERSION:2.0 +PRODID:-//algoo.fr//NONSGML Open Calendar v0.9//EN +BEGIN:VEVENT +CREATED:20250814T153429Z +LAST-MODIFIED:20250814T153503Z +DTSTAMP:20250814T153503Z +UID:f91964cb-53ca-4942-8811-c38f076f4328 +SUMMARY:error +DTSTART;TZID=Europe/Brussels:20250814T180000 +DTEND:20250814T190000 +TRANSP:OPAQUE +END:VEVENT +END:VCALENDAR From 7f28f69452d3c6250d7efdad85eb38464016b7a8 Mon Sep 17 00:00:00 2001 From: Peter Bieringer Date: Fri, 22 Aug 2025 07:51:15 +0200 Subject: [PATCH 015/290] extend test for items having tzinfo only on dtstart or dtend set for whatever reason, overtake tzinfo from the other one --- radicale/tests/test_base.py | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/radicale/tests/test_base.py b/radicale/tests/test_base.py index eb25bd1f..a9d0acc7 100644 --- a/radicale/tests/test_base.py +++ b/radicale/tests/test_base.py @@ -306,6 +306,22 @@ permissions: RrWw""") for uid2 in uids[i + 1:]: assert uid1 != uid2 + def test_add_event_tz_dtend_only(self) -> None: + """Add an event having TZ only on DTEND.""" + self.mkcalendar("/calendar.ics/") + event = get_file_content("event_issue1847_1.ics") + path = "/calendar.ics/event_issue1847_1.ics" + self.put(path, event) + _, headers, answer = self.request("GET", path, check=200) + + def test_add_event_tz_dtstart_only(self) -> None: + """Add an event having TZ only on DTSTART.""" + self.mkcalendar("/calendar.ics/") + event = get_file_content("event_issue1847_2.ics") + path = "/calendar.ics/event_issue1847_2.ics" + self.put(path, event) + _, headers, answer = self.request("GET", path, check=200) + def test_verify(self) -> None: """Verify the storage.""" contacts = get_file_content("contact_multiple.vcf") From 699a996be4a77240422421c2d53e5f5afd7f9e26 Mon Sep 17 00:00:00 2001 From: Peter Bieringer Date: Fri, 22 Aug 2025 07:51:54 +0200 Subject: [PATCH 016/290] changelog for items having tzinfo only on dtstart or dtend set for whatever reason, overtake tzinfo from the other one --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 7eafbee6..a144f1db 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -15,6 +15,7 @@ * Add: [hook] dryrun: option to disable real hook action for testing, add tests for email+rabbitmq * Fix: storage hook path now added to DELETE, MKCOL, MKCALENDAR, MOVE, and PROPPATCH * Add: storage hook placeholder now supports "request" and "to_path" (MOVE only) +* Improve: catch items having tzinfo only on dtstart or dtend set for whatever reason, overtake tzinfo from the other one ## 3.5.4 * Improve: item filter enhanced for 3rd level supporting VALARM and honoring TRIGGER (offset or absolute) From f268cd11e477324e2fcdc4d3d45ae4028553e086 Mon Sep 17 00:00:00 2001 From: Peter Bieringer Date: Fri, 22 Aug 2025 08:04:57 +0200 Subject: [PATCH 017/290] fix typo --- CHANGELOG.md | 2 +- config | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index a144f1db..b9753f25 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -128,7 +128,7 @@ * Fix: Using icalendar's tzinfo on created datetime to fix issue with icalendar * Fix: typos in code * Enhancement: Added free-busy report -* Enhancement: Added 'max_freebusy_occurrences` setting to avoid potential DOS on reports +* Enhancement: Added 'max_freebusy_occurrences` setting to avoid potential DoS on reports * Enhancement: remove unexpected control codes from uploaded items * Enhancement: add 'strip_domain' setting for username handling * Enhancement: add option to toggle debug log of rights rule with doesn't match diff --git a/config b/config index c38457df..0e08659b 100644 --- a/config +++ b/config @@ -342,5 +342,5 @@ [reporting] # When returning a free-busy report, limit the number of returned -# occurences per event to prevent DOS attacks. +# occurences per event to prevent DoS attacks. #max_freebusy_occurrence = 10000 From 8e4447e95bd1ffab9cf9aff5f7708e68e64d9056 Mon Sep 17 00:00:00 2001 From: Peter Bieringer Date: Fri, 22 Aug 2025 08:46:25 +0200 Subject: [PATCH 018/290] conditional log level for base_prefix strip action --- radicale/app/__init__.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/radicale/app/__init__.py b/radicale/app/__init__.py index 6764b7c6..9a06d34e 100644 --- a/radicale/app/__init__.py +++ b/radicale/app/__init__.py @@ -68,6 +68,8 @@ class Application(ApplicationPartDelete, ApplicationPartHead, _internal_server: bool _max_content_length: int _auth_realm: str + _auth_type: str + _web_type: str _script_name: str _extra_headers: Mapping[str, str] _permit_delete_collection: bool @@ -87,6 +89,8 @@ class Application(ApplicationPartDelete, ApplicationPartHead, self._request_header_on_debug = configuration.get("logging", "request_header_on_debug") self._response_content_on_debug = configuration.get("logging", "response_content_on_debug") self._auth_delay = configuration.get("auth", "delay") + self._auth_type = configuration.get("auth", "type") + self._web_type = configuration.get("web", "type") self._internal_server = configuration.get("server", "_internal_server") self._script_name = configuration.get("server", "script_name") if self._script_name: @@ -257,7 +261,10 @@ class Application(ApplicationPartDelete, ApplicationPartHead, logger.debug("Called by reverse proxy, remove base prefix %r from path: %r => %r", base_prefix, path, path_new) path = path_new else: - logger.warning("Called by reverse proxy, cannot remove base prefix %r from path: %r as not matching", base_prefix, path) + if self._auth_type in ['remote_user', 'http_x_remote_user'] and self._web_type == 'internal': + logger.warning("Called by reverse proxy, cannot remove base prefix %r from path: %r as not matching (may cause authentication issues using internal WebUI)", base_prefix, path) + else: + logger.debug("Called by reverse proxy, cannot remove base prefix %r from path: %r as not matching", base_prefix, path) # Get function corresponding to method function = getattr(self, "do_%s" % request_method, None) From f583adf6a45d786471ef6494568062b97b5672ff Mon Sep 17 00:00:00 2001 From: Peter Bieringer Date: Fri, 22 Aug 2025 08:50:51 +0200 Subject: [PATCH 019/290] changelog for conditional log level for base_prefix strip action --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index b9753f25..6440d9bc 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -16,6 +16,7 @@ * Fix: storage hook path now added to DELETE, MKCOL, MKCALENDAR, MOVE, and PROPPATCH * Add: storage hook placeholder now supports "request" and "to_path" (MOVE only) * Improve: catch items having tzinfo only on dtstart or dtend set for whatever reason, overtake tzinfo from the other one +* Improve: conditional log level for base_prefix strip action depending on auth and web type ## 3.5.4 * Improve: item filter enhanced for 3rd level supporting VALARM and honoring TRIGGER (offset or absolute) From 1d747fb407c32653e8433e74b5d0e399adcb5edf Mon Sep 17 00:00:00 2001 From: Peter Bieringer Date: Fri, 22 Aug 2025 08:53:38 +0200 Subject: [PATCH 020/290] release 3.5.5 --- CHANGELOG.md | 2 +- pyproject.toml | 2 +- setup.py.legacy | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 6440d9bc..cadd0dc5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,6 @@ # Changelog -## 3.5.5.dev +## 3.5.5 * Improve: [auth] ldap: do not read server info by bind to avoid needless network traffic * Fix: [storage] broken support of 'folder_umask' * Improve: add details about platform and effective user on startup diff --git a/pyproject.toml b/pyproject.toml index 4032a769..d8a49c18 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -3,7 +3,7 @@ name = "Radicale" # When the version is updated, a new section in the CHANGELOG.md file must be # added too. readme = "README.md" -version = "3.5.5.dev" +version = "3.5.5" authors = [{name = "Guillaume Ayoub", email = "guillaume.ayoub@kozea.fr"}, {name = "Unrud", email = "unrud@outlook.com"}, {name = "Peter Bieringer", email = "pb@bieringer.de"}] license = {text = "GNU GPL v3"} description = "CalDAV and CardDAV Server" diff --git a/setup.py.legacy b/setup.py.legacy index 1c44d272..e89bb5c5 100644 --- a/setup.py.legacy +++ b/setup.py.legacy @@ -20,7 +20,7 @@ from setuptools import find_packages, setup # When the version is updated, a new section in the CHANGELOG.md file must be # added too. -VERSION = "3.5.5.dev" +VERSION = "3.5.5" with open("README.md", encoding="utf-8") as f: long_description = f.read() From 9d5772901dd4158897577fbfb4baa3b53dec80ff Mon Sep 17 00:00:00 2001 From: Peter Bieringer Date: Sat, 23 Aug 2025 07:28:35 +0200 Subject: [PATCH 021/290] run rabbitmq tests only if module pika is available --- radicale/tests/test_hook_rabbitmq.py | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/radicale/tests/test_hook_rabbitmq.py b/radicale/tests/test_hook_rabbitmq.py index 42cedfce..a50dfaa2 100644 --- a/radicale/tests/test_hook_rabbitmq.py +++ b/radicale/tests/test_hook_rabbitmq.py @@ -21,6 +21,7 @@ Radicale tests related to hook 'rabbitmq' import logging import os +import pytest from radicale.tests import BaseTest from radicale.tests.helpers import get_file_content @@ -29,6 +30,14 @@ from radicale.tests.helpers import get_file_content class TestHooks(BaseTest): """Tests with hooks.""" + # test for available pika module + try: + import pika + except ImportError: + has_pika = 0 + else: + has_pika = 1 + def setup_method(self) -> None: BaseTest.setup_method(self) rights_file_path = os.path.join(self.colpath, "rights") @@ -63,6 +72,7 @@ permissions: RrWw""") self.configure({"hook": {"type": "rabbitmq", "dryrun": "True"}}) + @pytest.mark.skipif(has_pika == 0, reason="No pika module installed") def test_add_event(self, caplog) -> None: caplog.set_level(logging.WARNING) """Add an event.""" @@ -83,6 +93,7 @@ permissions: RrWw""") if (found is False): raise ValueError("Logging misses expected log line") + @pytest.mark.skipif(has_pika == 0, reason="No pika module installed") def test_delete_event(self, caplog) -> None: caplog.set_level(logging.WARNING) """Delete an event.""" From 5f7f41031038451e9ca510baa3a84669b69fdafd Mon Sep 17 00:00:00 2001 From: Jochen Sprickerhof Date: Sat, 23 Aug 2025 20:59:37 +0200 Subject: [PATCH 022/290] Fix acquire_lock interface signature See multifilesystem/lock.py and different calls. --- radicale/storage/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/radicale/storage/__init__.py b/radicale/storage/__init__.py index fb17f834..ba4d1038 100644 --- a/radicale/storage/__init__.py +++ b/radicale/storage/__init__.py @@ -355,7 +355,7 @@ class BaseStorage: raise NotImplementedError @types.contextmanager - def acquire_lock(self, mode: str, user: str = "") -> Iterator[None]: + def acquire_lock(self, mode: str, user: str = "", *args, **kwargs) -> Iterator[None]: """Set a context manager to lock the whole storage. ``mode`` must either be "r" for shared access or "w" for exclusive From 6d3cd8146f7ba42526e639973a5f509c03bd4f9d Mon Sep 17 00:00:00 2001 From: Peter Bieringer Date: Sun, 24 Aug 2025 10:14:28 +0200 Subject: [PATCH 023/290] fix lint issue related to 9d5772901dd4158897577fbfb4baa3b53dec80ff --- radicale/tests/test_hook_rabbitmq.py | 1 + 1 file changed, 1 insertion(+) diff --git a/radicale/tests/test_hook_rabbitmq.py b/radicale/tests/test_hook_rabbitmq.py index a50dfaa2..80abb55c 100644 --- a/radicale/tests/test_hook_rabbitmq.py +++ b/radicale/tests/test_hook_rabbitmq.py @@ -21,6 +21,7 @@ Radicale tests related to hook 'rabbitmq' import logging import os + import pytest from radicale.tests import BaseTest From a44e4cf335a22397150014b2708f80959726a58f Mon Sep 17 00:00:00 2001 From: Peter Bieringer Date: Sun, 24 Aug 2025 18:50:58 +0200 Subject: [PATCH 024/290] 3.5.6.dev --- CHANGELOG.md | 2 ++ pyproject.toml | 2 +- setup.py.legacy | 2 +- 3 files changed, 4 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index cadd0dc5..5b4a58fc 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,7 @@ # Changelog +## 3.5.6.dev + ## 3.5.5 * Improve: [auth] ldap: do not read server info by bind to avoid needless network traffic * Fix: [storage] broken support of 'folder_umask' diff --git a/pyproject.toml b/pyproject.toml index d8a49c18..a3f317bf 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -3,7 +3,7 @@ name = "Radicale" # When the version is updated, a new section in the CHANGELOG.md file must be # added too. readme = "README.md" -version = "3.5.5" +version = "3.5.6.dev" authors = [{name = "Guillaume Ayoub", email = "guillaume.ayoub@kozea.fr"}, {name = "Unrud", email = "unrud@outlook.com"}, {name = "Peter Bieringer", email = "pb@bieringer.de"}] license = {text = "GNU GPL v3"} description = "CalDAV and CardDAV Server" diff --git a/setup.py.legacy b/setup.py.legacy index e89bb5c5..12079205 100644 --- a/setup.py.legacy +++ b/setup.py.legacy @@ -20,7 +20,7 @@ from setuptools import find_packages, setup # When the version is updated, a new section in the CHANGELOG.md file must be # added too. -VERSION = "3.5.5" +VERSION = "3.5.6.dev" with open("README.md", encoding="utf-8") as f: long_description = f.read() From 550f522e9db951392224847a5e34910ecb444144 Mon Sep 17 00:00:00 2001 From: David Fernandez Alcoba Date: Fri, 29 Aug 2025 13:00:23 +0200 Subject: [PATCH 025/290] Fix broken start when UID does not exist --- radicale/pathutils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/radicale/pathutils.py b/radicale/pathutils.py index b204635a..8ee25b4e 100644 --- a/radicale/pathutils.py +++ b/radicale/pathutils.py @@ -327,6 +327,6 @@ def path_permissions_as_string(path): try: pp = path_permissions(path) s = "path=%r owner=%s group=%s mode=%o" % (path, pp[0], pp[1], pp[2]) - except NotImplementedError: + except (KeyError, NotImplementedError): s = "path=%r owner=UNKNOWN(unsupported on this system)" % (path) return s From c4e897f997f364b7726f6307ae2bf6e4a57b1cb3 Mon Sep 17 00:00:00 2001 From: Peter Bieringer Date: Sun, 31 Aug 2025 17:38:59 +0200 Subject: [PATCH 026/290] cosmetics --- DOCUMENTATION.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/DOCUMENTATION.md b/DOCUMENTATION.md index 256a5bfa..ec41848a 100644 --- a/DOCUMENTATION.md +++ b/DOCUMENTATION.md @@ -1250,6 +1250,7 @@ _(>= 3.1.9)_ Global control of permission to delete complete collection (default: True) If False it can be permitted by permissions per section with: D + If True it can be forbidden by permissions per section with: d ##### permit_overwrite_collection @@ -1259,6 +1260,7 @@ _(>= 3.3.0)_ Global control of permission to overwrite complete collection (default: True) If False it can be permitted by permissions per section with: O + If True it can be forbidden by permissions per section with: o #### storage From 001d44faae7e58e598653466c86eef610ac8c231 Mon Sep 17 00:00:00 2001 From: Peter Bieringer Date: Sun, 31 Aug 2025 17:42:26 +0200 Subject: [PATCH 027/290] changelog for https://github.com/Kozea/Radicale/pull/1857 --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 5b4a58fc..348b7eb7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,7 @@ # Changelog ## 3.5.6.dev +* Fix: broken start when UID does not exist (potential container startup case) ## 3.5.5 * Improve: [auth] ldap: do not read server info by bind to avoid needless network traffic From ca3fd9a3ffaaf61362f882c08757cde886b5774a Mon Sep 17 00:00:00 2001 From: Peter Bieringer Date: Mon, 1 Sep 2025 20:31:23 +0200 Subject: [PATCH 028/290] Improve: user/group retrievement for running service and directories --- radicale/pathutils.py | 37 ++++++++++++++++++++++++++++++------- radicale/utils.py | 42 +++++++++++++++++++++++++++++++++--------- 2 files changed, 63 insertions(+), 16 deletions(-) diff --git a/radicale/pathutils.py b/radicale/pathutils.py index 8ee25b4e..e4e65928 100644 --- a/radicale/pathutils.py +++ b/radicale/pathutils.py @@ -31,7 +31,7 @@ import threading from tempfile import TemporaryDirectory from typing import Iterator, Type, Union -from radicale import storage, types +from radicale import storage, types, utils if sys.platform == "win32": import ctypes @@ -320,13 +320,36 @@ def name_from_path(path: str, collection: "storage.BaseCollection") -> str: def path_permissions(path): path = pathlib.Path(path) - return [path.owner(), path.group(), path.stat().st_mode] + + try: + uid = utils.unknown_if_empty(path.stat().st_uid) + except (KeyError, NotImplementedError): + uid = "UNKNOWN" + + try: + gid = utils.unknown_if_empty(path.stat().st_gid) + except (KeyError, NotImplementedError): + gid = "UNKNOWN" + + try: + mode = utils.unknown_if_empty("%o" % path.stat().st_mode) + except (KeyError, NotImplementedError): + mode = "UNKNOWN" + + try: + owner = utils.unknown_if_empty(path.owner()) + except (KeyError, NotImplementedError): + owner = "UNKNOWN" + + try: + group = utils.unknown_if_empty(path.group()) + except (KeyError, NotImplementedError): + group = "UNKNOWN" + + return [owner, uid, group, gid, mode] def path_permissions_as_string(path): - try: - pp = path_permissions(path) - s = "path=%r owner=%s group=%s mode=%o" % (path, pp[0], pp[1], pp[2]) - except (KeyError, NotImplementedError): - s = "path=%r owner=UNKNOWN(unsupported on this system)" % (path) + pp = path_permissions(path) + s = "path=%r owner=%s(%s) group=%s(%s) mode=%s" % (path, pp[0], pp[1], pp[2], pp[3], pp[4]) return s diff --git a/radicale/utils.py b/radicale/utils.py index 096864b6..ed6c4ab2 100644 --- a/radicale/utils.py +++ b/radicale/utils.py @@ -226,25 +226,49 @@ def ssl_get_protocols(context): return protocols +def unknown_if_empty(value): + if value == "": + return "UNKNOWN" + else: + return value + + def user_groups_as_string(): if sys.platform != "win32": euid = os.geteuid() - egid = os.getegid() try: username = pwd.getpwuid(euid)[0] + user = "%s(%d)" % (unknown_if_empty(username), euid) except Exception: # name of user not found - s = "user=(%d) group=(%d)" % (euid, egid) - return s - gids = os.getgrouplist(username, egid) + user = "UNKNOWN(%d)" % euid + + egid = os.getegid() groups = [] - for gid in gids: + try: + gids = os.getgrouplist(username, egid) + for gid in gids: + try: + gi = grp.getgrgid(gid) + groups.append("%s(%d)" % (unknown_if_empty(gi.gr_name), gid)) + except Exception: + groups.append("UNKNOWN(%d)" % gid) + except Exception: try: - gi = grp.getgrgid(gid) - groups.append("%s(%d)" % (gi.gr_name, gid)) + groups.append("%s(%d)" % (grp.getgrnam(egid)[0], egid)) except Exception: - groups.append("%s(%d)" % (gid, gid)) - s = "user=%s(%d) groups=%s" % (username, euid, ','.join(groups)) + # workaround to get groupid by name + groups_all = grp.getgrall() + found = False + for entry in groups_all: + if entry[2] == egid: + groups.append("%s(%d)" % (unknown_if_empty(entry[0]), egid)) + found = True + break + if not found: + groups.append("UNKNOWN(%d)" % egid) + + s = "user=%s groups=%s" % (user, ','.join(groups)) else: username = os.getlogin() s = "user=%s" % (username) From 7edcd0bf083834fc73a6e5219fdb7a6eae4b9c82 Mon Sep 17 00:00:00 2001 From: Peter Bieringer Date: Mon, 1 Sep 2025 20:31:32 +0200 Subject: [PATCH 029/290] Changelog: Improve: user/group retrievement for running service and directorie --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 348b7eb7..c6b65355 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,7 @@ ## 3.5.6.dev * Fix: broken start when UID does not exist (potential container startup case) +* Improve: user/group retrievement for running service and directories ## 3.5.5 * Improve: [auth] ldap: do not read server info by bind to avoid needless network traffic From 2ecfe1c952821f36c5a6479ff2265992d9f11481 Mon Sep 17 00:00:00 2001 From: Peter Bieringer Date: Sun, 7 Sep 2025 08:55:47 +0200 Subject: [PATCH 030/290] merge extensions from ce9b2cf5d2b76056e58358a1e1ce3a5f7ce68b20 --- setup.cfg.legacy | 20 +++++++++++++++++++- 1 file changed, 19 insertions(+), 1 deletion(-) diff --git a/setup.cfg.legacy b/setup.cfg.legacy index e27241b4..399767f0 100644 --- a/setup.cfg.legacy +++ b/setup.cfg.legacy @@ -35,7 +35,25 @@ known_third_party = defusedxml,passlib,pkg_resources,pytest,vobject # Only enable default tests (https://github.com/PyCQA/flake8/issues/790#issuecomment-812823398) # DNE: DOES-NOT-EXIST select = E,F,W,C90,DNE000 -ignore = E121,E123,E126,E226,E24,E704,W503,W504,DNE000,E501 +ignore = E121,E123,E126,E226,E24,E704,W503,W504,DNE000,E501,E261 +exclude = .git, + __pycache__, + build, + dist, + *.egg, + *.egg-info, + *.eggs, + *.pyc, + *.pyo, + *.pyd, + .tox, + venv, + venv3, + .venv, + .venv3, + .env, + .mypy_cache, + .pytest_cache extend-exclude = build [mypy] From dd365d1f4b6c7c08cecb75fbe298e3aa51a3a110 Mon Sep 17 00:00:00 2001 From: Peter Bieringer Date: Sun, 7 Sep 2025 08:58:41 +0200 Subject: [PATCH 031/290] explicit define pyproject.toml as tox at least 4.30.2 is otherwise using EOL setup.cfg --- .github/workflows/test.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 82ac574f..34774410 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -19,7 +19,7 @@ jobs: - name: Install Test dependencies run: pip install tox - name: Test - run: tox -e py + run: tox -c pyproject.toml -e py - name: Install Coveralls if: github.event_name == 'push' run: pip install coveralls @@ -55,4 +55,4 @@ jobs: - name: Install tox run: pip install tox - name: Lint - run: tox -e flake8,mypy,isort + run: tox -c pyproject.toml -e flake8,mypy,isort From 8821612fa83a1a10f53f3b5aeff3eb2401899331 Mon Sep 17 00:00:00 2001 From: Peter Marschall Date: Mon, 21 Jul 2025 21:11:32 +0200 Subject: [PATCH 032/290] LDAP auth: allow finding groups based on separate search Instead of searching for the membership attribute on the user side (usually AD: memberOf, Novell eDirectory: groupMembership) to determine the groups the user loging on is a member of, allow performing a separate search for the groups having the user as member and use the found groups' DNs. The group search is performed in the context of 'ldap_reader_dn', after the user DN has been found in the directory, but before the authentication has been performed by doing an LDAP bind in the user's context. Although this may - in the case of unsuccessful login attempts - double the number of queries to the LDAP server, it has been done this way to keep the number of LDAP contexts minimal. Doing the group search in the context of the user logging on is no viable option, because there are known implementations where regular users do not have the necessary permissions to query the groups they are a member in. --- radicale/auth/ldap.py | 128 ++++++++++++++++++++++++++++++++++-------- radicale/config.py | 12 ++++ 2 files changed, 116 insertions(+), 24 deletions(-) diff --git a/radicale/auth/ldap.py b/radicale/auth/ldap.py index 2c4d63c3..fa2a4891 100644 --- a/radicale/auth/ldap.py +++ b/radicale/auth/ldap.py @@ -30,6 +30,10 @@ Following parameters controls SSL connections: ldap_security The encryption mode to be used: *none*|tls|starttls ldap_ssl_verify_mode The certificate verification mode. Works for tls and starttls. NONE, OPTIONAL, default is REQUIRED ldap_ssl_ca_file + The following parameters are optional: + ldap_group_base Base DN to search for groups. Only if it differs from ldap_base and if ldap_group_members_attribute is set + ldap_group_filter Search filter to search for groups having the user as member. Only if ldap_group_members_attribute is set + ldap_group_members_attribute Attribute in the group entries to read the group's members from """ import ssl @@ -47,6 +51,9 @@ class Auth(auth.BaseAuth): _ldap_attributes: list[str] = [] _ldap_user_attr: str _ldap_groups_attr: str + _ldap_group_base: str + _ldap_group_filter: str + _ldap_group_members_attr: str _ldap_module_version: int = 3 _ldap_use_ssl: bool = False _ldap_security: str = "none" @@ -78,6 +85,9 @@ class Auth(auth.BaseAuth): self._ldap_filter = configuration.get("auth", "ldap_filter") self._ldap_user_attr = configuration.get("auth", "ldap_user_attribute") self._ldap_groups_attr = configuration.get("auth", "ldap_groups_attribute") + self._ldap_group_base = configuration.get("auth", "ldap_group_base") + self._ldap_group_filter = configuration.get("auth", "ldap_group_filter") + self._ldap_group_members_attr = configuration.get("auth", "ldap_group_members_attribute") ldap_secret_file_path = configuration.get("auth", "ldap_secret_file") if ldap_secret_file_path: with open(ldap_secret_file_path, 'r') as file: @@ -110,6 +120,19 @@ class Auth(auth.BaseAuth): logger.info("auth.ldap_groups_attribute: %r" % self._ldap_groups_attr) else: logger.info("auth.ldap_groups_attribute: (not provided)") + if self._ldap_group_base: + logger.info("auth.ldap_group_base : %r" % self._ldap_group_base) + else: + logger.info("auth.ldap_group_base : (not provided, using ldap_base)") + self._ldap_group_base = self._ldap_base + if self._ldap_group_filter: + logger.info("auth.ldap_group_filter: %r" % self._ldap_group_filter) + else: + logger.info("auth.ldap_group_filter: (not provided)") + if self._ldap_group_members_attr: + logger.info("auth.ldap_group_members_attr: %r" % self._ldap_group_members_attr) + else: + logger.info("auth.ldap_group_members_attr: (not provided)") if ldap_secret_file_path: logger.info("auth.ldap_secret_file_path: %r" % ldap_secret_file_path) if self._ldap_secret: @@ -160,6 +183,30 @@ class Auth(auth.BaseAuth): user_entry = res[0] user_dn = user_entry[0] logger.debug(f"_login2 found LDAP user DN {user_dn}") + + """Let's collect the groups of the user.""" + groupDNs: list[str] = [] + if self._ldap_groups_attr: + groupDNs = user_entry[1][self._ldap_groups_attr] + + """Search for all groups having the user_dn found as member.""" + if self._ldap_group_members_attr: + groupDNs = [] + res = conn.search_s( + self._ldap_group_base, + self.ldap.SCOPE_SUBTREE, + filterstr="(&{0}({1}={2}))".format( + self._ldap_group_filter, + self._ldap_group_members_attr, + self.ldap.filter.escape_filter_chars(user_dn)), + attrlist=['1.1'] + ) + """Fill groupDNs with DNs of groups found""" + if len(res) > 0: + groupDNs = [] + for dn,entry in res: + groupDNs.append(dn) + """Close LDAP connection""" conn.unbind() except Exception as e: @@ -171,23 +218,23 @@ class Auth(auth.BaseAuth): conn.protocol_version = 3 conn.set_option(self.ldap.OPT_REFERRALS, 0) conn.simple_bind_s(user_dn, password) - tmp: list[str] = [] - if self._ldap_groups_attr: - tmp = [] - for g in user_entry[1][self._ldap_groups_attr]: - """Get group g's RDN's attribute value""" - try: - rdns = self.ldap.dn.explode_dn(g, notypes=True) - tmp.append(rdns[0]) - except Exception: - tmp.append(g.decode('utf8')) - self._ldap_groups = set(tmp) - logger.debug("_login2 LDAP groups of user: %s", ",".join(self._ldap_groups)) if self._ldap_user_attr: if user_entry[1][self._ldap_user_attr]: tmplogin = user_entry[1][self._ldap_user_attr][0] login = tmplogin.decode('utf-8') logger.debug(f"_login2 user set to: '{login}'") + + """Get RDNs of groups' DNs""" + tmp: list[str] = [] + for g in groupDNs: + try: + rdns = self.ldap.dn.explode_dn(g, notypes=True) + tmp.append(rdns[0]) + except Exception: + tmp.append(g.decode('utf8')) + self._ldap_groups = set(tmp) + logger.debug("_login2 LDAP groups of user: %s", ",".join(self._ldap_groups)) + conn.unbind() logger.debug(f"_login2 {login} successfully authenticated") return login @@ -249,9 +296,42 @@ class Auth(auth.BaseAuth): return "" user_entry = conn.response[0] - conn.unbind() user_dn = user_entry['dn'] logger.debug(f"_login3 found LDAP user DN {user_dn}") + + """Let's collect the groups of the user.""" + groupDNs: list[str] = [] + if self._ldap_groups_attr: + if user_entry['attributes'][self._ldap_groups_attr]: + if isinstance(user_entry['attributes'][self._ldap_groups_attr], list): + groupDNs = user_entry['attributes'][self._ldap_groups_attr] + else: + groupDNs.append(user_entry['attributes'][self._ldap_groups_attr]) + + """Search for all groups having the user_dn found as member.""" + if self._ldap_group_members_attr: + try: + conn.search( + search_base=self._ldap_group_base, + search_filter="(&{0}({1}={2}))".format( + self._ldap_group_filter, + self._ldap_group_members_attr, + self.ldap3.utils.conv.escape_filter_chars(user_dn)), + search_scope=self.ldap3.SUBTREE, + attributes=['1.1'] + ) + except Exception as e: + """LDAP search failed: consider it as non-fatal - only groups missing""" + logger.debug(f"_ldap3: LDAP group search failed: {e}") + else: + """Fill groupDNs with DNs of groups found""" + groupDNs = [] + for group in conn.response: + groupDNs.append(group['dn']) + + """Close LDAP connection""" + conn.unbind() + try: """Try to bind as the user itself""" try: @@ -264,18 +344,18 @@ class Auth(auth.BaseAuth): if not conn.bind(read_server_info=False): logger.debug(f"_login3 user '{login}' cannot be found") return "" + + """Get RDNs of groups' DNs""" tmp: list[str] = [] - if self._ldap_groups_attr: - tmp = [] - for g in user_entry['attributes'][self._ldap_groups_attr]: - """Get group g's RDN's attribute value""" - try: - rdns = self.ldap3.utils.dn.parse_dn(g) - tmp.append(rdns[0][1]) - except Exception: - tmp.append(g) - self._ldap_groups = set(tmp) - logger.debug("_login3 LDAP groups of user: %s", ",".join(self._ldap_groups)) + for g in groupDNs: + try: + rdns = self.ldap3.utils.dn.parse_dn(g) + tmp.append(rdns[0][1]) + except Exception: + tmp.append(g) + self._ldap_groups = set(tmp) + logger.debug("_login3 LDAP groups of user: %s", ",".join(self._ldap_groups)) + if self._ldap_user_attr: if user_entry['attributes'][self._ldap_user_attr]: if isinstance(user_entry['attributes'][self._ldap_user_attr], list): diff --git a/radicale/config.py b/radicale/config.py index c6d93f41..77fcb04e 100644 --- a/radicale/config.py +++ b/radicale/config.py @@ -297,6 +297,18 @@ DEFAULT_CONFIG_SCHEMA: types.CONFIG_SCHEMA = OrderedDict([ "value": "", "help": "attribute to read the group memberships from", "type": str}), + ("ldap_group_members_attribute", { + "value": "", + "help": "Attribute in the group entries to read the group's members from", + "type": str}), + ("ldap_group_base", { + "value": "", + "help": "Base DN to search for groups. Only if it differs from ldap_base and if ldap_group_members_attribute is set", + "type": str}), + ("ldap_group_filter", { + "value": "", + "help": "Search filter to search for groups having the user as member. Only if ldap_group_members_attribute is set", + "type": str}), ("ldap_use_ssl", { "value": "False", "help": "Use ssl on the ldap connection. Soon to be deprecated, use ldap_security instead", From 5f677fc77ed9c4c0cfb2a5d462b6a21f68049d2c Mon Sep 17 00:00:00 2001 From: Peter Marschall Date: Sun, 31 Aug 2025 17:51:23 +0200 Subject: [PATCH 033/290] LDAP auth: document all paramters at the top of the file --- radicale/auth/ldap.py | 46 +++++++++++++++++++++++++++---------------- 1 file changed, 29 insertions(+), 17 deletions(-) diff --git a/radicale/auth/ldap.py b/radicale/auth/ldap.py index fa2a4891..15bf89ea 100644 --- a/radicale/auth/ldap.py +++ b/radicale/auth/ldap.py @@ -16,24 +16,36 @@ # along with Radicale. If not, see . """ Authentication backend that checks credentials with a LDAP server. -Following parameters are needed in the configuration: - ldap_uri The LDAP URL to the server like ldap://localhost - ldap_base The baseDN of the LDAP server - ldap_reader_dn The DN of a LDAP user with read access to get the user accounts - ldap_secret The password of the ldap_reader_dn - ldap_secret_file The path of the file containing the password of the ldap_reader_dn - ldap_filter The search filter to find the user to authenticate by the username - ldap_user_attribute The attribute to be used as username after authentication - ldap_groups_attribute The attribute containing group memberships in the LDAP user entry -Following parameters controls SSL connections: - ldap_use_ssl If ssl encryption should be used (to be deprecated) - ldap_security The encryption mode to be used: *none*|tls|starttls - ldap_ssl_verify_mode The certificate verification mode. Works for tls and starttls. NONE, OPTIONAL, default is REQUIRED - ldap_ssl_ca_file + The following parameters are needed in the configuration: + ldap_uri URI to the LDAP server + ldap_base Base DN of the LDAP server + ldap_reader_dn DN of an LDAP user with read access to get the user accounts + ldap_secret Password of the 'ldap_reader_dn' + Better: use 'ldap_secret_file'! + ldap_secret_file Path of the file containing the password of the 'ldap_reader_dn' + ldap_filter Search filter to find the user DN to authenticate + The following parameters control TLS connections: + ldap_use_ssl Use ssl on the ldap connection. + Deprecated, use 'ldap_security' instead! + ldap_security Encryption mode to be used, + one of: *none* | tls | starttls + ldap_ssl_verify_mode Certificate verification mode for tls and starttls; + one of: *REQUIRED* | OPTIONAL | NONE + ldap_ssl_ca_file Path to the CA file in PEM format to certify the server certificate The following parameters are optional: - ldap_group_base Base DN to search for groups. Only if it differs from ldap_base and if ldap_group_members_attribute is set - ldap_group_filter Search filter to search for groups having the user as member. Only if ldap_group_members_attribute is set - ldap_group_members_attribute Attribute in the group entries to read the group's members from + ldap_user_attribute Attribute to be used as username after authentication, e.g. cn; + if not given, the name used to logon is used. + ldap_groups_attribute Attribute in the user entry to read the user's group memberships from, + e.g. memberof, groupMememberShip. This may even be a non-DN attribute! + ldap_group_base Base DN to search for groups; + only if it differs from 'ldap_base' and if 'ldap_group_members_attribute' is set + ldap_group_filter Search filter to search for groups having the user DN found as member; + only if 'ldap_group_members_attribute' is set + ldap_group_members_attribute Attribute in the group entries to read the group's members from, + e.g. member. + The following parameters are for LDAP servers with oddities + ldap_ignore_attribute_create_modify_timestamp + Ignore modifyTimestamp and createTimestamp attributes. Needed for Authentik LDAP server """ import ssl From 5a183e3c2b3504ccd59f4f74300e8c05055fbfc5 Mon Sep 17 00:00:00 2001 From: Peter Marschall Date: Sun, 31 Aug 2025 20:43:10 +0200 Subject: [PATCH 034/290] LDAP auth: make flake8 happy "fix" small lint to keep flake8 happy. --- radicale/auth/ldap.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/radicale/auth/ldap.py b/radicale/auth/ldap.py index 15bf89ea..54f29e08 100644 --- a/radicale/auth/ldap.py +++ b/radicale/auth/ldap.py @@ -216,7 +216,7 @@ class Auth(auth.BaseAuth): """Fill groupDNs with DNs of groups found""" if len(res) > 0: groupDNs = [] - for dn,entry in res: + for dn, entry in res: groupDNs.append(dn) """Close LDAP connection""" From 5c4a0578b02953dfbeb55677fbaf48cf60d14f5f Mon Sep 17 00:00:00 2001 From: Peter Marschall Date: Sat, 6 Sep 2025 10:46:35 +0200 Subject: [PATCH 035/290] LDAP auth: fix _login2() by importing ldap.filter --- radicale/auth/ldap.py | 1 + 1 file changed, 1 insertion(+) diff --git a/radicale/auth/ldap.py b/radicale/auth/ldap.py index 54f29e08..a1c0ec6c 100644 --- a/radicale/auth/ldap.py +++ b/radicale/auth/ldap.py @@ -80,6 +80,7 @@ class Auth(auth.BaseAuth): except ImportError: try: import ldap + import ldap.filter self._ldap_module_version = 2 self.ldap = ldap except ImportError as e: From 9b216a9f2408a51e6b4b6ae98bcea8edf4475bf5 Mon Sep 17 00:00:00 2001 From: Peter Marschall Date: Sat, 6 Sep 2025 10:58:23 +0200 Subject: [PATCH 036/290] LDAP auth: define fallback value for _use_encryption --- radicale/auth/ldap.py | 1 + 1 file changed, 1 insertion(+) diff --git a/radicale/auth/ldap.py b/radicale/auth/ldap.py index a1c0ec6c..2b17257c 100644 --- a/radicale/auth/ldap.py +++ b/radicale/auth/ldap.py @@ -67,6 +67,7 @@ class Auth(auth.BaseAuth): _ldap_group_filter: str _ldap_group_members_attr: str _ldap_module_version: int = 3 + _use_encryption: bool = False _ldap_use_ssl: bool = False _ldap_security: str = "none" _ldap_ssl_verify_mode: int = ssl.CERT_REQUIRED From cde4c5f2e831a4f41ee6e3d0e8c44b805d60bae1 Mon Sep 17 00:00:00 2001 From: Peter Marschall Date: Sun, 7 Sep 2025 14:35:58 +0200 Subject: [PATCH 037/290] LDAP auth: stop giving type hints for local list variables --- radicale/auth/ldap.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/radicale/auth/ldap.py b/radicale/auth/ldap.py index 2b17257c..39b1593c 100644 --- a/radicale/auth/ldap.py +++ b/radicale/auth/ldap.py @@ -199,7 +199,7 @@ class Auth(auth.BaseAuth): logger.debug(f"_login2 found LDAP user DN {user_dn}") """Let's collect the groups of the user.""" - groupDNs: list[str] = [] + groupDNs = [] if self._ldap_groups_attr: groupDNs = user_entry[1][self._ldap_groups_attr] @@ -239,7 +239,7 @@ class Auth(auth.BaseAuth): logger.debug(f"_login2 user set to: '{login}'") """Get RDNs of groups' DNs""" - tmp: list[str] = [] + tmp = [] for g in groupDNs: try: rdns = self.ldap.dn.explode_dn(g, notypes=True) @@ -314,7 +314,7 @@ class Auth(auth.BaseAuth): logger.debug(f"_login3 found LDAP user DN {user_dn}") """Let's collect the groups of the user.""" - groupDNs: list[str] = [] + groupDNs = [] if self._ldap_groups_attr: if user_entry['attributes'][self._ldap_groups_attr]: if isinstance(user_entry['attributes'][self._ldap_groups_attr], list): @@ -360,7 +360,7 @@ class Auth(auth.BaseAuth): return "" """Get RDNs of groups' DNs""" - tmp: list[str] = [] + tmp = [] for g in groupDNs: try: rdns = self.ldap3.utils.dn.parse_dn(g) From 9eb955653660b86391a3f23c8a543ff8b9c34475 Mon Sep 17 00:00:00 2001 From: Peter Marschall Date: Sun, 7 Sep 2025 14:38:56 +0200 Subject: [PATCH 038/290] LDAP auth: decode UTF-8 byte sequences to strings only if necessary --- radicale/auth/ldap.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/radicale/auth/ldap.py b/radicale/auth/ldap.py index 39b1593c..84dcee0b 100644 --- a/radicale/auth/ldap.py +++ b/radicale/auth/ldap.py @@ -234,8 +234,9 @@ class Auth(auth.BaseAuth): conn.simple_bind_s(user_dn, password) if self._ldap_user_attr: if user_entry[1][self._ldap_user_attr]: - tmplogin = user_entry[1][self._ldap_user_attr][0] - login = tmplogin.decode('utf-8') + login = user_entry[1][self._ldap_user_attr][0] + if isinstance(login, bytes): + login = login.decode('utf-8') logger.debug(f"_login2 user set to: '{login}'") """Get RDNs of groups' DNs""" @@ -245,7 +246,9 @@ class Auth(auth.BaseAuth): rdns = self.ldap.dn.explode_dn(g, notypes=True) tmp.append(rdns[0]) except Exception: - tmp.append(g.decode('utf8')) + if isinstance(g, bytes): + g = g.decode('utf-8') + tmp.append(g) self._ldap_groups = set(tmp) logger.debug("_login2 LDAP groups of user: %s", ",".join(self._ldap_groups)) From 57a4d8d47d1f8cc476a146ce1e92d5f212eeac17 Mon Sep 17 00:00:00 2001 From: Peter Marschall Date: Mon, 8 Sep 2025 21:59:29 +0200 Subject: [PATCH 039/290] LDAP auth: update, consolidate & extend documentation --- DOCUMENTATION.md | 143 ++++++++++++++++++++++++++++++++------------- config | 42 +++++++------ radicale/config.py | 54 ++++++++--------- 3 files changed, 154 insertions(+), 85 deletions(-) diff --git a/DOCUMENTATION.md b/DOCUMENTATION.md index ec41848a..632c477a 100644 --- a/DOCUMENTATION.md +++ b/DOCUMENTATION.md @@ -987,7 +987,8 @@ Default: `Radicale - Password Required` _(>= 3.3.0)_ -The URI to the ldap server +URI to the LDAP server. +Mandatory for auth type `ldap`. Default: `ldap://localhost` @@ -995,39 +996,44 @@ Default: `ldap://localhost` _(>= 3.3.0)_ -LDAP base DN of the ldap server. This parameter must be provided if auth type is ldap. +Base DN of the LDAP server. +Mandatory for auth type `ldap`. -Default: +Default: (unset) ##### ldap_reader_dn _(>= 3.3.0)_ -The DN of a ldap user with read access to get the user accounts. This parameter must be provided if auth type is ldap. +DN of a LDAP user with read access users and - if defined - groups. +Mandatory for auth type `ldap`. -Default: +Default: (unset) ##### ldap_secret _(>= 3.3.0)_ -The password of the ldap_reader_dn. Either this parameter or `ldap_secret_file` must be provided if auth type is ldap. +Password of `ldap_reader_dn`. +Mandatory for auth type `ldap` unless `ldap_secret_file` is given. -Default: +Default: (unset) ##### ldap_secret_file _(>= 3.3.0)_ -Path of the file containing the password of the ldap_reader_dn. Either this parameter or `ldap_secret` must be provided if auth type is ldap. +Path to the file containing the password of `ldap_reader_dn`. +Mandatory for auth type `ldap` unless `ldap_secret` is given. -Default: +Default: (unset) ##### ldap_filter _(>= 3.3.0)_ -The search filter to find the user DN to authenticate by the username. User '{0}' as placeholder for the user name. +Filter to search for the LDAP entry of the user to authenticate. +It must contain '{0}' as placeholder for the login name. Default: `(cn={0})` @@ -1035,66 +1041,117 @@ Default: `(cn={0})` _(>= 3.4.0)_ -The LDAP attribute whose value shall be used as the user name after successful authentication +LDAP attribute whose value shall be used as the username after successful authentication. -Default: not set, i.e. the login name given is used directly. +If set, you can use flexible logins in `ldap_filter` and still have consolidated usernames, +e.g. to allow login in using mail addresses as an alternative to cn, simply set +``` +ldap_filter = (&(objectclass=inetOrgPerson)(|(cn={0})(mail={0}))) +ldap_user_attribute = cn +``` +Even for simple filter setups, it is recommended to set it in order to get usernames exactly +as they are stored in LDAP and to avoid inconsistencies in the upper-/lower-case spelling of the +login names. -##### ldap_groups_attribute - -_(>= 3.4.0)_ - -The LDAP attribute to read the group memberships from in the authenticated user's LDAP entry. - -If set, load the LDAP group memberships from the attribute given -These memberships can be used later on to define rights. -This also gives you access to the group calendars, if they exist. -* The group calendar will be placed under collection_root_folder/GROUPS -* The name of the calendar directory is the base64 encoded group name. -* The group calendar folders will not be created automatically. This must be done manually. In the [LDAP-authentication section of Radicale's wiki](https://github.com/Kozea/Radicale/wiki/LDAP-authentication) you can find a script to create a group calendar. - -Use 'memberOf' if you want to load groups on Active Directory and alikes, 'groupMembership' on Novell eDirectory, ... - -Default: (unset) +Default: (unset, in which case the login name is directly used as the username) ##### ldap_use_ssl _(>= 3.3.0)_ -Use ssl on the ldap connection (soon to be deprecated, use ldap_security instead) +Use ssl on the LDAP connection. **Deprecated**, use `ldap_security` instead**!** ##### ldap_security _(>= 3.5.2)_ -Use encryption on the ldap connection. none, tls, starttls +Use encryption on the LDAP connection. One of `none`, `tls`, `starttls`. -Default: none +Default: `none` ##### ldap_ssl_verify_mode _(>= 3.3.0)_ -The certificate verification mode. Works for tls and starttls. NONE, OPTIONAL or REQUIRED +Certificate verification mode for tls and starttls. One of `NONE`, `OPTIONAL`, `REQUIRED`. -Default: REQUIRED +Default: `REQUIRED` ##### ldap_ssl_ca_file _(>= 3.3.0)_ -The path to the CA file in pem format which is used to certificate the server certificate +Path to the CA file in PEM format which is used to certify the server certificate -Default: +Default: (unset) + +##### ldap_groups_attribute + +_(>= 3.4.0)_ + +LDAP attribute in the authenticated user's LDAP entry to read the group memberships from. + +E.g. `memberOf` to get groups on Active Directory and alikes, `groupMembership` on Novell eDirectory, ... + +If set, get the user's LDAP groups from the attribute given. + +For DN-valued attributes, the value of the RDN is used to determine the group names. +The implementation also supports non-DN-valued attributes: their values are taken directly. + +The user's group names can be used later on to define rights. +They also give you access to the group calendars, if those exist. +* Group calendars are placed directly under *collection_root_folder*`/GROUPS/` + with the base64-encoded group name as the calendar folder name. +* Group calendar folders are not created automatically. + This must be done manually. In the [LDAP-authentication section of Radicale's wiki](https://github.com/Kozea/Radicale/wiki/LDAP-authentication) you can find a script to create a group calendar. + +Default: (unset) + +##### ldap_group_members_attribute + +_(>= 3.5.6)_ + +Attribute in the group entries to read the group's members from. + +E.g. `member` for groups with objectclass `groupOfNames`. + +Using `ldap_group_members_attribute`, `ldap_group_base` and `ldap_group_filter` is an alternative +approach to getting the user's groups. Instead of reading them from `ldap_groups_attribute` +in the user's entry, an additional query is performed to seach for those groups beneath `ldap_group_base`, +that have the user's DN in their `ldap_group_members_attribute` and additionally fulfil `ldap_group_filter`. + +As with DN-valued `ldap_groups_attribute`, the value of the RDN is used to determine the group names. + +Default: (unset) + +##### ldap_group_base + +_(>= 3.5.6)_ + +Base DN to search for groups. +Only necessary if `ldap_group_members_attribute` is set, and if the base DN for groups differs from `ldap_base`. + +Default: (unset, in which case `ldap_base` is used as fallback) + +##### ldap_group_filter + +_(>= 3.5.6)_ + +Search filter to search for groups having the user DN found as member. +Only necessary `ldap_group_members_attribute` is set, and you want the groups returned to be restricted +instead of all groups the user's DN is in. + +Default: (unset) ##### ldap_ignore_attribute_create_modify_timestamp _(>= 3.5.1)_ -Add modifyTimestamp and createTimestamp to the exclusion list of internal ldap3 client -so that these schema attributes are not checked. This is needed at least for Authentik -LDAP server as not providing these both attributes. +Quirks for Authentik LDAP server, which violates the LDAP RFCs: +add modifyTimestamp and createTimestamp to the exclusion list of internal ldap3 client +so that these schema attributes are not checked. -Default: false +Default: `false` ##### dovecot_connection_type = AF_UNIX @@ -1177,7 +1234,9 @@ providers like ldap, kerberos Default: `False` -Note: cannot be enabled together with `uc_username` +Notes: +* `lc_username` and `uc_username` are mutually exclusive +* for auth type `ldap` the use of `ldap_user_attribute` is preferred ##### uc_username @@ -1188,7 +1247,9 @@ providers like ldap, kerberos Default: `False` -Note: cannot be enabled together with `lc_username` +Notes: +* `uc_username` and `lc_username` are mutually exclusive +* for auth type `ldap` the use of `ldap_user_attribute` is preferred ##### strip_domain diff --git a/config b/config index 0e08659b..b51c5dfc 100644 --- a/config +++ b/config @@ -75,46 +75,54 @@ ## Expiration time of caching failed logins in seconds #cache_failed_logins_expiry = 90 -# Ignore modifyTimestamp and createTimestamp attributes. Required e.g. for Authentik LDAP server -#ldap_ignore_attribute_create_modify_timestamp = false - # URI to the LDAP server #ldap_uri = ldap://localhost -# The base DN where the user accounts have to be searched +# Base DN of the LDAP server to search for user accounts #ldap_base = ##BASE_DN## -# The reader DN of the LDAP server +# Reader DN of the LDAP server; (needs read access to users and - if defined - groups) #ldap_reader_dn = CN=ldapreader,CN=Users,##BASE_DN## -# Password of the reader DN +# Password of the reader DN (better: use 'ldap_secret_file'!) #ldap_secret = ldapreader-secret -# Path of the file containing password of the reader DN +# Path to the file containing the password of the reader DN #ldap_secret_file = /run/secrets/ldap_password -# the attribute to read the group memberships from in the user's LDAP entry (default: not set) -#ldap_groups_attribute = memberOf - -# The filter to find the DN of the user. This filter must contain a python-style placeholder for the login +# Filter to search for the LDAP entry of the user to authenticate. It must contain '{0}' as placeholder for the login name. #ldap_filter = (&(objectClass=person)(uid={0})) -# the attribute holding the value to be used as username after authentication +# Attribute holding the value to be used as username after authentication #ldap_user_attribute = cn -# Use ssl on the ldap connection -# Soon to be deprecated, use ldap_security instead +# Use ssl on the LDAP connection (DEPRECATED - use 'ldap_security'!) #ldap_use_ssl = False -# the encryption mode to be used: tls, starttls, default is none +# Encryption mode to be used. Default: none; one of: none, tls, starttls #ldap_security = none -# The certificate verification mode. Works for ssl and starttls. NONE, OPTIONAL, default is REQUIRED +# Certificate verification mode for tls & starttls. Default: REQUIRED; one of NONE, OPTIONAL, REQUIRED #ldap_ssl_verify_mode = REQUIRED -# The path to the CA file in pem format which is used to certificate the server certificate +# Path to the CA file in PEM format to certify the server certificate #ldap_ssl_ca_file = +# Attribute in the user's LDAP entry to read the group memberships from; default: not set +#ldap_groups_attribute = memberOf + +# Attribute in the group entries to read the group's members from, e.g. member; default: not set +#ldap_group_members_attribute = member + +# Base DN to search for groups; only if it differs from 'ldap_base' and if 'ldap_group_members_attribute' is set +#ldap_group_base = ##GROUP_BASE_DN## + +# Search filter to search for groups having the user DN found as member; only if 'ldap_group_members_attribute' is set +#ldap_group_filter = (objectclass=groupOfNames) + +# Quirks for Authentik LDAP server: ignore modifyTimestamp and createTimestamp attributes +#ldap_ignore_attribute_create_modify_timestamp = false + # Connection type for dovecot authentication (AF_UNIX|AF_INET|AF_INET6) # Note: credentials are transmitted in cleartext #dovecot_connection_type = AF_UNIX diff --git a/radicale/config.py b/radicale/config.py index 77fcb04e..adab9567 100644 --- a/radicale/config.py +++ b/radicale/config.py @@ -261,41 +261,53 @@ DEFAULT_CONFIG_SCHEMA: types.CONFIG_SCHEMA = OrderedDict([ "value": "1", "help": "incorrect authentication delay", "type": positive_float}), - ("ldap_ignore_attribute_create_modify_timestamp", { - "value": "false", - "help": "Ignore modifyTimestamp and createTimestamp attributes. Need if Authentik LDAP server is used.", - "type": bool}), ("ldap_uri", { "value": "ldap://localhost", - "help": "URI to the ldap server", + "help": "URI to the LDAP server", "type": str}), ("ldap_base", { "value": "", - "help": "LDAP base DN of the ldap server", + "help": "Base DN of the LDAP server", "type": str}), ("ldap_reader_dn", { "value": "", - "help": "the DN of a ldap user with read access to get the user accounts", + "help": "DN of an LDAP user with read access to users anmd - if defined - groups", "type": str}), ("ldap_secret", { "value": "", - "help": "the password of the ldap_reader_dn", + "help": "Password of ldap_reader_dn (better: use ldap_secret_file)", "type": str}), ("ldap_secret_file", { "value": "", - "help": "path of the file containing the password of the ldap_reader_dn", + "help": "Path to the file containing the password of ldap_reader_dn", "type": str}), ("ldap_filter", { "value": "(cn={0})", - "help": "the search filter to find the user DN to authenticate by the username", + "help": "Filter to search for the LDAP entry of the user to authenticate", "type": str}), ("ldap_user_attribute", { "value": "", - "help": "the attribute to be used as username after authentication", + "help": "Attribute to be used as username after authentication", + "type": str}), + ("ldap_use_ssl", { + "value": "False", + "help": "Use ssl on the LDAP connection. Deprecated, use ldap_security instead!", + "type": bool}), + ("ldap_security", { + "value": "none", + "help": "Encryption mode to be used: *none*|tls|starttls", + "type": str}), + ("ldap_ssl_verify_mode", { + "value": "REQUIRED", + "help": "Certificate verification mode for tls and starttls. NONE, OPTIONAL, default is REQUIRED", + "type": str}), + ("ldap_ssl_ca_file", { + "value": "", + "help": "Path to the CA file in PEM format which is used to certify the server certificate", "type": str}), ("ldap_groups_attribute", { "value": "", - "help": "attribute to read the group memberships from", + "help": "Attribute in the user's LDAP entry to read the group memberships from", "type": str}), ("ldap_group_members_attribute", { "value": "", @@ -309,22 +321,10 @@ DEFAULT_CONFIG_SCHEMA: types.CONFIG_SCHEMA = OrderedDict([ "value": "", "help": "Search filter to search for groups having the user as member. Only if ldap_group_members_attribute is set", "type": str}), - ("ldap_use_ssl", { - "value": "False", - "help": "Use ssl on the ldap connection. Soon to be deprecated, use ldap_security instead", + ("ldap_ignore_attribute_create_modify_timestamp", { + "value": "false", + "help": "Quirk for Authentik LDAP server: ignore modifyTimestamp and createTimestamp attributes.", "type": bool}), - ("ldap_security", { - "value": "none", - "help": "the encryption mode to be used: *none*|tls|starttls", - "type": str}), - ("ldap_ssl_verify_mode", { - "value": "REQUIRED", - "help": "The certificate verification mode. Works for tls and starttls. NONE, OPTIONAL, default is REQUIRED", - "type": str}), - ("ldap_ssl_ca_file", { - "value": "", - "help": "The path to the CA file in pem format which is used to certificate the server certificate", - "type": str}), ("imap_host", { "value": "localhost", "help": "IMAP server hostname: address|address:port|[address]:port|*localhost*", From 1bac038f5a3fd0164aaa3c72b49dfafd6cf90034 Mon Sep 17 00:00:00 2001 From: Peter Bieringer Date: Tue, 9 Sep 2025 07:39:56 +0200 Subject: [PATCH 040/290] changelog for https://github.com/Kozea/Radicale/pull/1861 --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index c6b65355..f5de8c23 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,6 +3,7 @@ ## 3.5.6.dev * Fix: broken start when UID does not exist (potential container startup case) * Improve: user/group retrievement for running service and directories +* Extend/Improve: [auth] ldap: group membership lookup ## 3.5.5 * Improve: [auth] ldap: do not read server info by bind to avoid needless network traffic From b5a1ea911d0cfff2e9f55a72d74eaf4866be42cb Mon Sep 17 00:00:00 2001 From: Johannes Berg Date: Sat, 30 Aug 2025 21:59:22 +0200 Subject: [PATCH 041/290] auth: dovecot: pass remote IP (rip=) to auth server If known, let the auth server know where the client came from, using REMOTE_ADDR or, optionally/configurably, the X-Remote-Addr header value (which is needed when running behind a trusted proxy.) Addresses #1859. --- CHANGELOG.md | 1 + DOCUMENTATION.md | 20 ++++++++++++ config | 3 ++ radicale/app/__init__.py | 13 ++++++-- radicale/auth/__init__.py | 30 ++++++++++++++++-- radicale/auth/dovecot.py | 18 +++++++++-- radicale/config.py | 4 +++ radicale/tests/test_auth.py | 62 +++++++++++++++++++++++++++++++++---- 8 files changed, 136 insertions(+), 15 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index f5de8c23..c3f3cb51 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,7 @@ * Fix: broken start when UID does not exist (potential container startup case) * Improve: user/group retrievement for running service and directories * Extend/Improve: [auth] ldap: group membership lookup +* Add: option [auth] dovecot_rip_x_remote_addr ## 3.5.5 * Improve: [auth] ldap: do not read server info by bind to avoid needless network traffic diff --git a/DOCUMENTATION.md b/DOCUMENTATION.md index 632c477a..4f6b7ed0 100644 --- a/DOCUMENTATION.md +++ b/DOCUMENTATION.md @@ -1187,6 +1187,26 @@ Port of via network exposed dovecot socket Default: `12345` +##### dovecot_rip_x_remote_addr + +_(>= 3.5.6)_ + +Use the `X-Remote-Addr` value for the remote IP (rip) parameter in the +dovecot authentication protocol. + +If set, Radicale must be running behind a proxy that you control and +that sets/overwrites the `X-Remote-Addr` header (doesn't pass it) so +that the value passed to dovecot is reliable. For example, for nginx, +add + +``` + proxy_set_header X-Remote-Addr $remote_addr; +``` + +to the configuration sample. + +Default: `False` + ##### imap_host _(>= 3.4.1)_ diff --git a/config b/config index b51c5dfc..37040eff 100644 --- a/config +++ b/config @@ -136,6 +136,9 @@ # Port of via network exposed dovecot socket #dovecot_port = 12345 +# Use X-Remote-Addr for remote IP (rip) in dovecot authentication +#dovecot_rip_x_remote_addr = False + # IMAP server hostname # Syntax: address | address:port | [address]:port | imap.server.tld #imap_host = localhost diff --git a/radicale/app/__init__.py b/radicale/app/__init__.py index 9a06d34e..0fcbd328 100644 --- a/radicale/app/__init__.py +++ b/radicale/app/__init__.py @@ -49,6 +49,7 @@ from radicale.app.propfind import ApplicationPartPropfind from radicale.app.proppatch import ApplicationPartProppatch from radicale.app.put import ApplicationPartPut from radicale.app.report import ApplicationPartReport +from radicale.auth import AuthContext from radicale.log import logger # Combination of types.WSGIStartResponse and WSGI application return value @@ -156,6 +157,8 @@ class Application(ApplicationPartDelete, ApplicationPartHead, unsafe_path = environ.get("PATH_INFO", "") https = environ.get("HTTPS", "") + context = AuthContext() + """Manage a request.""" def response(status: int, headers: types.WSGIResponseHeaders, answer: Union[None, str, bytes]) -> _IntermediateResponse: @@ -201,12 +204,16 @@ class Application(ApplicationPartDelete, ApplicationPartHead, remote_host = "unknown" if environ.get("REMOTE_HOST"): remote_host = repr(environ["REMOTE_HOST"]) - elif environ.get("REMOTE_ADDR"): - remote_host = environ["REMOTE_ADDR"] + if environ.get("REMOTE_ADDR"): + if remote_host == 'unknown': + remote_host = environ["REMOTE_ADDR"] + context.remote_addr = environ["REMOTE_ADDR"] if environ.get("HTTP_X_FORWARDED_FOR"): reverse_proxy = True remote_host = "%s (forwarded for %r)" % ( remote_host, environ["HTTP_X_FORWARDED_FOR"]) + if environ.get("HTTP_X_REMOTE_ADDR"): + context.x_remote_addr = environ["HTTP_X_REMOTE_ADDR"] if environ.get("HTTP_X_FORWARDED_HOST") or environ.get("HTTP_X_FORWARDED_PROTO") or environ.get("HTTP_X_FORWARDED_SERVER"): reverse_proxy = True remote_useragent = "" @@ -295,7 +302,7 @@ class Application(ApplicationPartDelete, ApplicationPartHead, self.configuration, environ, base64.b64decode( authorization.encode("ascii"))).split(":", 1) - (user, info) = self._auth.login(login, password) or ("", "") if login else ("", "") + (user, info) = self._auth.login(login, password, context) or ("", "") if login else ("", "") if self.configuration.get("auth", "type") == "ldap": try: logger.debug("Groups received from LDAP: %r", ",".join(self._auth._ldap_groups)) diff --git a/radicale/auth/__init__.py b/radicale/auth/__init__.py index 2de8c4e9..d6371383 100644 --- a/radicale/auth/__init__.py +++ b/radicale/auth/__init__.py @@ -91,6 +91,15 @@ def load(configuration: "config.Configuration") -> "BaseAuth": configuration) +class AuthContext: + remote_addr: str + x_remote_addr: str + + def __init__(self): + self.remote_addr = None + self.x_remote_addr = None + + class BaseAuth: _ldap_groups: Set[str] = set([]) @@ -187,6 +196,21 @@ class BaseAuth: raise NotImplementedError + def _login_ext(self, login: str, password: str, context: AuthContext) -> str: + """Check credentials and map login to internal user + + ``login`` the login name + + ``password`` the password + + ``context`` additional data for the login, e.g. IP address used + + Returns the username or ``""`` for invalid credentials. + """ + + # override this method instead of _login() if you want the context + return self._login(login, password) + def _sleep_for_constant_exec_time(self, time_ns_begin: int): """Sleep some time to reach a constant execution time for failed logins @@ -216,7 +240,7 @@ class BaseAuth: time.sleep(sleep) @final - def login(self, login: str, password: str) -> Tuple[str, str]: + def login(self, login: str, password: str, context: AuthContext) -> Tuple[str, str]: time_ns_begin = time.time_ns() result_from_cache = False if self._lc_username: @@ -284,7 +308,7 @@ class BaseAuth: if result == "": # verify login+password via configured backend logger.debug("Login verification for user+password via backend: '%s'", login) - result = self._login(login, password) + result = self._login_ext(login, password, context) if result != "": logger.debug("Login successful for user+password via backend: '%s'", login) if digest == "": @@ -314,7 +338,7 @@ class BaseAuth: return (result, self._type) else: # self._cache_logins is False - result = self._login(login, password) + result = self._login_ext(login, password, context) if result == "": self._sleep_for_constant_exec_time(time_ns_begin) return (result, self._type) diff --git a/radicale/auth/dovecot.py b/radicale/auth/dovecot.py index b3f3fb81..b4d28c32 100644 --- a/radicale/auth/dovecot.py +++ b/radicale/auth/dovecot.py @@ -19,6 +19,7 @@ import base64 import itertools import os +import re import socket from contextlib import closing @@ -32,6 +33,8 @@ class Auth(auth.BaseAuth): self.timeout = 5 self.request_id_gen = itertools.count(1) + self.use_x_remote_addr = configuration.get("auth", "dovecot_rip_x_remote_addr") + config_family = configuration.get("auth", "dovecot_connection_type") if config_family == "AF_UNIX": self.family = socket.AF_UNIX @@ -46,7 +49,7 @@ class Auth(auth.BaseAuth): else: self.family = socket.AF_INET6 - def _login(self, login, password): + def _login_ext(self, login, password, context): """Validate credentials. Check if the ``login``/``password`` pair is valid according to Dovecot. @@ -148,10 +151,19 @@ class Auth(auth.BaseAuth): "Authenticating with request id: '{}'" .format(request_id) ) + rip = b'' + if self.use_x_remote_addr and context.x_remote_addr: + rip = context.x_remote_addr.encode('ascii') + elif context.remote_addr: + rip = context.remote_addr.encode('ascii') + # squash all whitespace - shouldn't be there and auth protocol + # is sensitive to whitespace (in particular \t and \n) + if rip: + rip = b'\trip=' + re.sub(br'\s', b'', rip) sock.send( - b'AUTH\t%u\tPLAIN\tservice=radicale\tresp=%b\n' % + b'AUTH\t%u\tPLAIN\tservice=radicale%s\tresp=%b\n' % ( - request_id, base64.b64encode( + request_id, rip, base64.b64encode( b'\0%b\0%b' % (login.encode(), password.encode()) ) diff --git a/radicale/config.py b/radicale/config.py index adab9567..a228ed97 100644 --- a/radicale/config.py +++ b/radicale/config.py @@ -253,6 +253,10 @@ DEFAULT_CONFIG_SCHEMA: types.CONFIG_SCHEMA = OrderedDict([ "value": "12345", "help": "dovecot auth port", "type": int}), + ("dovecot_rip_x_remote_addr", { + "value": "False", + "help": "use X-Remote-Addr for dovecot auth remote IP (rip) parameter", + "type": bool}), ("realm", { "value": "Radicale - Password Required", "help": "message displayed when a password is needed", diff --git a/radicale/tests/test_auth.py b/radicale/tests/test_auth.py index 88cd3ea4..e34256ed 100644 --- a/radicale/tests/test_auth.py +++ b/radicale/tests/test_auth.py @@ -282,13 +282,23 @@ class TestBaseAuthRequests(BaseTest): @pytest.mark.skipif(sys.platform == 'win32', reason="Not supported on Windows") def _test_dovecot( - self, user, password, expected_status, - response=b'FAIL\n1\n', mech=[b'PLAIN'], broken=None): + self, user, password, expected_status, expected_rip=None, + response=b'FAIL\t1', mech=[b'PLAIN'], broken=None, + extra_config=None, extra_env=None): import socket from unittest.mock import DEFAULT, patch - self.configure({"auth": {"type": "dovecot", - "dovecot_socket": "./dovecot.sock"}}) + if extra_env is None: + extra_env = {} + if extra_config is None: + extra_config = {} + + config = {"auth": {"type": "dovecot", + "dovecot_socket": "./dovecot.sock"}} + for toplvl, entries in extra_config.items(): + for key, val in entries.items(): + config[toplvl][key] = val + self.configure(config) if broken is None: broken = [] @@ -311,10 +321,18 @@ class TestBaseAuthRequests(BaseTest): if "done" not in broken: handshake += b'DONE\n' + sent_rip = None + + def record_sent_data(s, data, flags=None): + nonlocal sent_rip + if b'\trip=' in data: + sent_rip = data.split(b'\trip=')[1].split(b'\t')[0] + return len(data) + with patch.multiple( 'socket.socket', connect=DEFAULT, - send=DEFAULT, + send=record_sent_data, recv=DEFAULT ) as mock_socket: if "socket" in broken: @@ -325,7 +343,9 @@ class TestBaseAuthRequests(BaseTest): status, _, answer = self.request( "PROPFIND", "/", HTTP_AUTHORIZATION="Basic %s" % base64.b64encode( - ("%s:%s" % (user, password)).encode()).decode()) + ("%s:%s" % (user, password)).encode()).decode(), + **extra_env) + assert sent_rip == expected_rip assert status == expected_status @pytest.mark.skipif(sys.platform == 'win32', reason="Not supported on Windows") @@ -392,6 +412,36 @@ class TestBaseAuthRequests(BaseTest): def test_dovecot_auth_id_mismatch(self): self._test_dovecot("user", "password", 401, response=b'OK\t2') + @pytest.mark.skipif(sys.platform == 'win32', reason="Not supported on Windows") + def test_dovecot_remote_addr(self): + self._test_dovecot("user", "password", 401, expected_rip=b'172.17.16.15', + extra_env={ + 'REMOTE_ADDR': '172.17.16.15', + 'HTTP_X_REMOTE_ADDR': '127.0.0.1', + }) + + @pytest.mark.skipif(sys.platform == 'win32', reason="Not supported on Windows") + def test_dovecot_x_remote_addr(self): + self._test_dovecot("user", "password", 401, expected_rip=b'172.17.16.15', + extra_env={ + 'REMOTE_ADDR': '127.0.0.1', + 'HTTP_X_REMOTE_ADDR': '172.17.16.15', + }, + extra_config={ + 'auth': {"dovecot_rip_x_remote_addr": "True"}, + }) + + @pytest.mark.skipif(sys.platform == 'win32', reason="Not supported on Windows") + def test_dovecot_x_remote_addr_whitespace(self): + self._test_dovecot("user", "password", 401, expected_rip=b'172.17.16.15rip=127.0.0.1', + extra_env={ + 'REMOTE_ADDR': '127.0.0.1', + 'HTTP_X_REMOTE_ADDR': '172.17.16.15\trip=127.0.0.1', + }, + extra_config={ + 'auth': {"dovecot_rip_x_remote_addr": "True"}, + }) + def test_custom(self) -> None: """Custom authentication.""" self.configure({"auth": {"type": "radicale.tests.custom.auth"}}) From 256ca59aaf3586b00a10eb7c769dd2caf5338463 Mon Sep 17 00:00:00 2001 From: Johannes Berg Date: Tue, 9 Sep 2025 20:22:44 +0200 Subject: [PATCH 042/290] auth: clean up remote IP parameter/documentation Make the remote IP parameter more generic and make it an enum determining the source instead of a boolean. Also fix the changelog entry. Both as requested, I managed to miss those comments previously. --- CHANGELOG.md | 2 +- DOCUMENTATION.md | 24 ++++++++++++++++-------- config | 5 +++-- radicale/auth/__init__.py | 2 ++ radicale/auth/dovecot.py | 3 ++- radicale/config.py | 9 +++++---- radicale/tests/test_auth.py | 4 ++-- 7 files changed, 31 insertions(+), 18 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index c3f3cb51..8ef6be2c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,7 +4,7 @@ * Fix: broken start when UID does not exist (potential container startup case) * Improve: user/group retrievement for running service and directories * Extend/Improve: [auth] ldap: group membership lookup -* Add: option [auth] dovecot_rip_x_remote_addr +* Add: [auth] remote_ip_source: set the remote IP source for auth algorithms ## 3.5.5 * Improve: [auth] ldap: do not read server info by bind to avoid needless network traffic diff --git a/DOCUMENTATION.md b/DOCUMENTATION.md index 4f6b7ed0..0d0ab6cd 100644 --- a/DOCUMENTATION.md +++ b/DOCUMENTATION.md @@ -1187,17 +1187,25 @@ Port of via network exposed dovecot socket Default: `12345` -##### dovecot_rip_x_remote_addr +##### remote_ip_source _(>= 3.5.6)_ -Use the `X-Remote-Addr` value for the remote IP (rip) parameter in the -dovecot authentication protocol. +For authentication mechanisms that are made aware of the remote IP +(such as dovecot via the `rip=` auth protocol parameter), determine +the source to use. Currently, valid values are -If set, Radicale must be running behind a proxy that you control and -that sets/overwrites the `X-Remote-Addr` header (doesn't pass it) so -that the value passed to dovecot is reliable. For example, for nginx, -add +`REMOTE_ADDR` (default) +: Use the REMOTE_ADDR environment variable that captures the remote + address of the socket connection. + +`X-Remote-Addr` +: Use the `X-Remote-Addr` HTTP header value. + +In the case of `X-Remote-Addr`, Radicale must be running be running +behind a proxy that you control and that sets/overwrites the +`X-Remote-Addr` header (doesn't pass it) so that the value passed +to dovecot is reliable. For example, for nginx, add ``` proxy_set_header X-Remote-Addr $remote_addr; @@ -1205,7 +1213,7 @@ add to the configuration sample. -Default: `False` +Default: `REMOTE_ADDR` ##### imap_host diff --git a/config b/config index 37040eff..79bb275a 100644 --- a/config +++ b/config @@ -136,8 +136,9 @@ # Port of via network exposed dovecot socket #dovecot_port = 12345 -# Use X-Remote-Addr for remote IP (rip) in dovecot authentication -#dovecot_rip_x_remote_addr = False +# Remote address source for authentication mechanisms (such as dovecot) +# that are passed this information. +#remote_ip_source = REMOTE_ADDR # IMAP server hostname # Syntax: address | address:port | [address]:port | imap.server.tld diff --git a/radicale/auth/__init__.py b/radicale/auth/__init__.py index d6371383..c32a8306 100644 --- a/radicale/auth/__init__.py +++ b/radicale/auth/__init__.py @@ -64,6 +64,8 @@ INSECURE_IF_NO_LOOPBACK_TYPES: Sequence[str] = ( AUTH_SOCKET_FAMILY: Sequence[str] = ("AF_UNIX", "AF_INET", "AF_INET6") +REMOTE_ADDR_SOURCE: Sequence[str] = ("REMOTE_ADDR", "X-Remote-Addr") + def load(configuration: "config.Configuration") -> "BaseAuth": """Load the authentication module chosen in configuration.""" diff --git a/radicale/auth/dovecot.py b/radicale/auth/dovecot.py index b4d28c32..479f4111 100644 --- a/radicale/auth/dovecot.py +++ b/radicale/auth/dovecot.py @@ -33,7 +33,8 @@ class Auth(auth.BaseAuth): self.timeout = 5 self.request_id_gen = itertools.count(1) - self.use_x_remote_addr = configuration.get("auth", "dovecot_rip_x_remote_addr") + remote_ip_source = configuration.get("auth", "remote_ip_source") + self.use_x_remote_addr = remote_ip_source == 'X-Remote-Addr' config_family = configuration.get("auth", "dovecot_connection_type") if config_family == "AF_UNIX": diff --git a/radicale/config.py b/radicale/config.py index a228ed97..7693e9e6 100644 --- a/radicale/config.py +++ b/radicale/config.py @@ -253,10 +253,11 @@ DEFAULT_CONFIG_SCHEMA: types.CONFIG_SCHEMA = OrderedDict([ "value": "12345", "help": "dovecot auth port", "type": int}), - ("dovecot_rip_x_remote_addr", { - "value": "False", - "help": "use X-Remote-Addr for dovecot auth remote IP (rip) parameter", - "type": bool}), + ("remote_ip_source", { + "value": "REMOTE_ADDR", + "help": "remote address source for passing it to auth method", + "type": str, + "internal": auth.REMOTE_ADDR_SOURCE}), ("realm", { "value": "Radicale - Password Required", "help": "message displayed when a password is needed", diff --git a/radicale/tests/test_auth.py b/radicale/tests/test_auth.py index e34256ed..5ffc540d 100644 --- a/radicale/tests/test_auth.py +++ b/radicale/tests/test_auth.py @@ -428,7 +428,7 @@ class TestBaseAuthRequests(BaseTest): 'HTTP_X_REMOTE_ADDR': '172.17.16.15', }, extra_config={ - 'auth': {"dovecot_rip_x_remote_addr": "True"}, + 'auth': {"remote_ip_source": "X-Remote-Addr"}, }) @pytest.mark.skipif(sys.platform == 'win32', reason="Not supported on Windows") @@ -439,7 +439,7 @@ class TestBaseAuthRequests(BaseTest): 'HTTP_X_REMOTE_ADDR': '172.17.16.15\trip=127.0.0.1', }, extra_config={ - 'auth': {"dovecot_rip_x_remote_addr": "True"}, + 'auth': {"remote_ip_source": "X-Remote-Addr"}, }) def test_custom(self) -> None: From fe93f88d173fcfede1ca316e2790a924a6047c20 Mon Sep 17 00:00:00 2001 From: Peter Bieringer Date: Wed, 10 Sep 2025 19:42:02 +0200 Subject: [PATCH 043/290] extend config sniplet triggered by https://github.com/Kozea/Radicale/issues/1869 --- DOCUMENTATION.md | 6 ++++++ contrib/apache/radicale.conf | 12 ++++++++++++ 2 files changed, 18 insertions(+) diff --git a/DOCUMENTATION.md b/DOCUMENTATION.md index 0d0ab6cd..6e7b7c69 100644 --- a/DOCUMENTATION.md +++ b/DOCUMENTATION.md @@ -427,6 +427,9 @@ RewriteRule ^/radicale$ /radicale/ [R,L] RequestHeader set X-Script-Name /radicale RequestHeader set X-Forwarded-Port "%{SERVER_PORT}s" RequestHeader set X-Forwarded-Proto expr=%{REQUEST_SCHEME} + = 2.4.40> + Proxy100Continue Off + ``` @@ -517,6 +520,9 @@ RewriteRule ^/radicale$ /radicale/ [R,L] ProxyPass http://localhost:5232/ retry=0 ProxyPassReverse http://localhost:5232/ + = 2.4.40> + Proxy100Continue Off + RequestHeader set X-Script-Name /radicale RequestHeader set X-Remote-User expr=%{REMOTE_USER} diff --git a/contrib/apache/radicale.conf b/contrib/apache/radicale.conf index d92c5c31..385ee159 100644 --- a/contrib/apache/radicale.conf +++ b/contrib/apache/radicale.conf @@ -59,6 +59,9 @@ ProxyPass http://localhost:5232/ retry=0 ProxyPassReverse http://localhost:5232/ + = 2.4.40> + Proxy100Continue Off + Require local @@ -74,6 +77,9 @@ ProxyPass http://localhost:5232/ retry=0 ProxyPassReverse http://localhost:5232/ + = 2.4.40> + Proxy100Continue Off + ## User authentication handled by "radicale" @@ -221,6 +227,9 @@ CustomLog logs/ssl_request_log "%t %h %{SSL_PROTOCOL}x %{SSL_CIPHER}x \"%r\" %b" ProxyPass http://localhost:5232/ retry=0 ProxyPassReverse http://localhost:5232/ + = 2.4.40> + Proxy100Continue Off + Require local @@ -234,6 +243,9 @@ CustomLog logs/ssl_request_log "%t %h %{SSL_PROTOCOL}x %{SSL_CIPHER}x \"%r\" %b" ProxyPass http://localhost:5232/ retry=0 ProxyPassReverse http://localhost:5232/ + = 2.4.40> + Proxy100Continue Off + ## User authentication handled by "radicale" From 50043e5ec7fd036b1d5e8eb9785029c6fc670e9f Mon Sep 17 00:00:00 2001 From: Peter Marschall Date: Tue, 9 Sep 2025 22:21:32 +0200 Subject: [PATCH 044/290] documentation updates * config sections in [brackets] * config values as `code` * config value alternatives as lists * standardized format for config options * consolidate multiple markup variants into one * fix hierarchy for some options * grammar fixes * fix some "Germanisms" --- DOCUMENTATION.md | 781 ++++++++++++++++++++++++++--------------------- 1 file changed, 439 insertions(+), 342 deletions(-) diff --git a/DOCUMENTATION.md b/DOCUMENTATION.md index 6e7b7c69..cde35fdb 100644 --- a/DOCUMENTATION.md +++ b/DOCUMENTATION.md @@ -10,7 +10,8 @@ Radicale is a small but powerful CalDAV (calendars, to-do lists) and CardDAV * Shares calendars and contact lists through CalDAV, CardDAV and HTTP. * Supports events, todos, journal entries and business cards. * Works out-of-the-box, no complicated setup or configuration required. -* Can limit access by authentication. +* Offers flexible authentication options. +* Can limit access by authorization. * Can secure connections with TLS. * Works with many [CalDAV and CardDAV clients](#supported-clients). @@ -25,11 +26,9 @@ Check * [Tutorials](#tutorials) * [Documentation](#documentation-1) * [Wiki on GitHub](https://github.com/Kozea/Radicale/wiki) -* [Disussions on GitHub](https://github.com/Kozea/Radicale/discussions) +* [Discussions on GitHub](https://github.com/Kozea/Radicale/discussions) * [Open and already Closed Issues on GitHub](https://github.com/Kozea/Radicale/issues?q=is%3Aissue) -Hint: instead of downloading from PyPI look for packages provided by used [distribution](#linux-distribution-packages), they contain also startup scripts to run daemonized. - #### What's New? Read the [Changelog on GitHub](https://github.com/Kozea/Radicale/blob/master/CHANGELOG.md). @@ -38,18 +37,23 @@ Read the [Changelog on GitHub](https://github.com/Kozea/Radicale/blob/master/CHA ### Simple 5-minute setup -You want to try Radicale but only have 5 minutes free in your calendar? Let's -go right now and play a bit with Radicale! +You want to try Radicale but only have 5 minutes free in your calendar? +Let's go right now and play a bit with Radicale! -When everything works, you can get a [client](#supported-clients) -and start creating calendars and address books. The server, configured with settings from this section, only binds to localhost (is not reachable over the network) -and you can log in with any user name and password. When everything works, you may get a local client and start creating calendars and address books. -If Radicale fits your needs, it may be time for some [basic configuration](#basic-configuration) to support remote clients and desired authentication type. +The server, configured with settings from this section, only binds to localhost +(i.e. it is not reachable over the network), and you can log in with any username and password. +When everything works, you may get a local [client](#supported-clients) +and start creating calendars and address books. +If Radicale fits your needs, it may be time for some [basic configuration](#basic-configuration) +to support remote clients and desired authentication type. Follow one of the chapters below depending on your operating system. #### Linux / \*BSD +Hint: instead of downloading from PyPI, look for packages provided by your [distribution](#linux-distribution-packages). +They contain also startup scripts integrated into your distributions, that allow Radicale to run daemonized. + First, make sure that **python** 3.9 or later and **pip** are installed. On most distributions it should be enough to install the package ``python3-pip``. @@ -62,7 +66,8 @@ Recommended only for testing - open a console and type: python3 -m pip install --user --upgrade https://github.com/Kozea/Radicale/archive/master.tar.gz ``` -If _install_ is not working and instead `error: externally-managed-environment` is displayed, create and activate a virtual environment in advance +If _install_ is not working and instead `error: externally-managed-environment` is displayed, +create and activate a virtual environment in advance. ```bash python3 -m venv ~/venv @@ -84,15 +89,15 @@ python3 -m radicale --storage-filesystem-folder=~/.var/lib/radicale/collections ##### as system user (or as root) -Alternative one can install and run as system user or as root (not recommended) +Alternatively, you can install and run as system user or as root (not recommended): ```bash -# Run the following command as root (not required) -# or non-root system user (can require --user in case of dependencies are not available system-wide and/or virtual environment) +# Run the following command as root (not recommended) or non-root system user +# (the later may require --user in case dependencies are not available system-wide and/or virtual environment) python3 -m pip install --upgrade https://github.com/Kozea/Radicale/archive/master.tar.gz ``` -Start the service manually, data is stored in a system folder +Start the service manually, with data stored in a system folder under `/var/lib/radicale/collections`: ```bash # Start, data is stored in a system folder (requires write permissions to /var/lib/radicale/collections) @@ -116,12 +121,12 @@ python -m radicale --storage-filesystem-folder=~/radicale/collections --auth-typ ##### Common -Victory! Open in your browser! +Success!!! Open in your browser! You can log in with any username and password as no authentication is required by example option `--auth-type none`. -But this is INSECURE, see [Configuration/Authentication](#auth) for more. +This is **INSECURE**, see [Configuration/Authentication](#auth) for more details. Just note that default configuration for security reason binds the server to `localhost` (IPv4: `127.0.0.1`, IPv6: `::1`). -See [Addresses](#addresses) and [Configuration/Server](#server) for more. +See [Addresses](#addresses) and [Configuration/Server](#server) for more details. ### Basic Configuration @@ -144,9 +149,12 @@ All configuration options are described in detail in the #### Authentication -In its default configuration since 3.5.0 Radicale rejects by default all authentication by `type = denyall` (introduced with 3.2.2) until explicitly configured. +In its default configuration since version 3.5.0, Radicale rejects all +authentication attempts by using config option `type = denyall` (introduced +with 3.2.2) as default until explicitly configured. -Before 3.5.0 it didn't check usernames or passwords if not explicitly configured, and if the server is reachable over a network, you should change this as soon as possible. +Versions before 3.5.0 did not check usernames or passwords at all, unless explicitly configured. +If such a server is reachable over a network, you should change this as soon as possible. First a `users` file with all usernames and passwords must be created. It can be stored in the same directory as the configuration file. @@ -156,11 +164,12 @@ It can be stored in the same directory as the configuration file. The `users` file can be created and managed with [htpasswd](https://httpd.apache.org/docs/current/programs/htpasswd.html): -Note: some OS contain unpatched `htpasswd` (< 2.4.59) without supporting SHA-256 or SHA-512 -(e.g. Ubuntu LTS 22), in this case use '-B' for "bcrypt" hash method or stay with -insecure MD5 (default) or SHA-1 ('-s'). +Note: some OSes or distributions contain outdated versions of `htpasswd` (< 2.4.59) without +support for SHA-256 or SHA-512 (e.g. Ubuntu LTS 22). +In these cases use `htpasswd`'s command line option `-B` for the `bcrypt` hash method (recommended), +or stay with the insecure (not recommended) MD5 (default) or SHA-1 (command line option `-s`). -Note that support of SHA-256 or SHA-512 was introduced with 3.1.9 +Note: support of SHA-256 and SHA-512 was introduced with 3.1.9 ```bash # Create a new htpasswd file with the user "user1" using SHA-512 as hash method @@ -204,7 +213,7 @@ htpasswd_encryption = plain #### Addresses -The default configuration binds the server to localhost. It can't be reached +The default configuration binds the server to localhost. It cannot be reached from other computers. This can be changed with the following configuration options (IPv4 and IPv6): @@ -223,7 +232,7 @@ be changed with the following configuration: filesystem_folder = /path/to/storage ``` -> **Security:** The storage folder should not be readable by unauthorized users. +> **Security:** The storage folder shall not be readable by unauthorized users. > Otherwise, they can read the calendar data and lock the storage. > You can find OS dependent instructions in the > [Running as a service](#running-as-a-service) section. @@ -256,20 +265,30 @@ requirements. #### Linux with systemd system-wide -Recommendation: check support by [Linux Distribution Packages](#linux-distribution-packages) instead of manual setup / initial configuration. +Recommendation: check support by [Linux Distribution Packages](#linux-distribution-packages) +instead of manual setup / initial configuration. -Create the **radicale** user and group for the Radicale service. (Run -`useradd --system --user-group --home-dir / --shell /sbin/nologin radicale` as root.) -The storage folder must be writable by **radicale**. (Run -`mkdir -p /var/lib/radicale/collections && chown -R radicale:radicale /var/lib/radicale/collections` -as root.) +Create the **radicale** user and group for the Radicale service by running (as `root`: +```bash +useradd --system --user-group --home-dir / --shell /sbin/nologin radicale +``` -If a dedicated cache folder is configured (see option 'storage' -> 'filesystem_cache_folder'), it also must be also writable by **radicale**. (Run -`mkdir -p /var/cache/radicale && chown -R radicale:radicale /var/cache/radicale` -as root.) +The storage folder must be writable by the **radicale** user by running (as `root`): +```bash +mkdir -p /var/lib/radicale/collections && chown -R radicale:radicale /var/lib/radicale/collections +``` -> **Security:** The storage should not be readable by others. -> (Run `chmod -R o= /var/lib/radicale/collections` as root.) +If a dedicated cache folder is configured (see option [filesystem_cache_folder](#filesystem_cache_folder)), +it also must be also writable by **radicale**. To achieva that, run (as `root`): +```bash +mkdir -p /var/cache/radicale && chown -R radicale:radicale /var/cache/radicale +```` + +> **Security:** The storage shall not be readable by others. +> To make sure this is the case, run (as `root`): +> ```bash +> chmod -R o= /var/lib/radicale/collections +> ``` Create the file `/etc/systemd/system/radicale.service`: @@ -295,14 +314,14 @@ ProtectKernelModules=true ProtectControlGroups=true NoNewPrivileges=true ReadWritePaths=/var/lib/radicale/ -# Replace with following in case of dedicated cache folder should be used +# Replace with following in case dedicated cache folder should be used #ReadWritePaths=/var/lib/radicale/ /var/cache/radicale/ [Install] WantedBy=multi-user.target ``` -Radicale will load the configuration file from `/etc/radicale/config`. +In this system-wide implementation, Radicale will load the configuration from the file `/etc/radicale/config`. To enable and manage the service run: @@ -333,7 +352,8 @@ Restart=on-failure WantedBy=default.target ``` -Radicale will load the configuration file from `~/.config/radicale/config`. +In this user-specific configuration, Radicale will load the configuration from +the file `~/.config/radicale/config`. You should set the configuration option `filesystem_folder` in the `storage` section to something like `~/.var/lib/radicale/collections`. @@ -475,7 +495,7 @@ incorrect authentication attempts. Connections are terminated after a timeout. Set the configuration option `type` in the `auth` section to `http_x_remote_user`. Radicale uses the username provided in the `X-Remote-User` HTTP header and -disables HTTP authentication. +disables its internal HTTP authentication. Example **nginx** configuration: @@ -547,8 +567,8 @@ RequestHeader set X-Remote-User expr=%{REMOTE_USER} > **Security:** Untrusted clients should not be able to access the Radicale > server directly. Otherwise, they can authenticate as any user by simply -> setting related HTTP header. This can be prevented by restrict listen to -> loopback interface only or at least a local firewall rule. +> setting related HTTP header. This can be prevented by listening to the +> loopback interface only or local firewall rules. #### Secure connection between Radicale and the reverse proxy @@ -556,8 +576,8 @@ SSL certificates can be used to encrypt and authenticate the connection between Radicale and the reverse proxy. First you have to generate a certificate for Radicale and a certificate for the reverse proxy. The following commands generate self-signed certificates. You will be asked to enter additional -information about the certificate, the values don't matter and you can keep the -defaults. +information about the certificate, these values do not really matter and you can +keep the defaults. ```bash openssl req -x509 -newkey rsa:4096 -keyout server_key.pem -out server_cert.pem \ @@ -576,7 +596,7 @@ key = /path/to/server_key.pem certificate_authority = /path/to/client_cert.pem ``` -If you're using the Let's Encrypt's Certbot, the configuration should look similar to this: +If you are using the Let's Encrypt Certbot, the configuration should look similar to this: ```ini [server] @@ -626,10 +646,10 @@ gunicorn --bind '127.0.0.1:5232' --env 'RADICALE_CONFIG=/etc/radicale/config' \ #### Manage user accounts with the WSGI server Set the configuration option `type` in the `auth` section to `remote_user`. -Radicale uses the username provided by the WSGI server and disables -authentication over HTTP. +This way Radicale uses the username provided by the WSGI server and disables +its internal authentication over HTTP. -### Versioning with Git +### Versioning collections with Git This tutorial describes how to keep track of all changes to calendars and address books with **git** (or any other version control system). @@ -695,6 +715,9 @@ Reason for problems can be Radicale can be configured with a configuration file or with command line arguments. +Configuration files have INI-style syntax comprising key-value pairs +grouped into sections with section headers enclosed in brackets. + An example configuration file looks like: ```ini @@ -729,7 +752,7 @@ python3 -m radicale --server-hosts 0.0.0.0:5232,[::]:5232 \ Add the argument `--config ""` to stop Radicale from loading the default configuration files. Run `python3 -m radicale --help` for more information. -One can also use command line options in startup scripts using following examples: +You can also use command-line options in startup scripts as shown in the following examples: ```bash ## simple variable containing multiple options @@ -747,12 +770,12 @@ RADICALE_OPTIONS+=("--config=/etc/radicale/config") /usr/bin/radicale ${RADICALE_OPTIONS[@]} ``` -In the following, all configuration categories and options are described. +The following describes all configuration sections and options. -#### server +#### [server] -The configuration options in this category are only relevant in standalone -mode. All options are ignored, when Radicale runs via WSGI. +The configuration options in this section are only relevant in standalone +mode; they are ignored, when Radicale runs on WSGI. ##### hosts @@ -835,7 +858,7 @@ Strip script name from URI if called by reverse proxy Default: (taken from HTTP_X_SCRIPT_NAME or SCRIPT_NAME) -#### encoding +#### [encoding] ##### request @@ -849,55 +872,57 @@ Encoding for storing local collections Default: `utf-8` -#### auth +#### [auth] ##### type The method to verify usernames and passwords. -Available backends: +Available types are: -`none` -: Just allows all usernames and passwords. +* `none` + Just allows all usernames and passwords. -`denyall` _(>= 3.2.2)_ -: Just denies all usernames and passwords. +* `denyall` _(>= 3.2.2)_ + Just denies all usernames and passwords. -`htpasswd` -: Use an +* `htpasswd` + Use an [Apache htpasswd file](https://httpd.apache.org/docs/current/programs/htpasswd.html) to store usernames and passwords. -`remote_user` -: Takes the username from the `REMOTE_USER` environment variable and disables - HTTP authentication. This can be used to provide the username from a WSGI - server which authenticated the client upfront. Required to validate, otherwise - client can supply the header itself which is unconditionally trusted then. +* `remote_user` + Takes the username from the `REMOTE_USER` environment variable and disables + Radicale's internal HTTP authentication. This can be used to provide the + username from a WSGI server which authenticated the client upfront. + Requires validation, otherwise clients can supply the header themselves, + which then is unconditionally trusted. -`http_x_remote_user` -: Takes the username from the `X-Remote-User` HTTP header and disables HTTP - authentication. This can be used to provide the username from a reverse - proxy which authenticated the client upfront. Required to validate, otherwise - client can supply the header itself which is unconditionally trusted then. +* `http_x_remote_user` + Takes the username from the `X-Remote-User` HTTP header and disables + Radicale's internal HTTP authentication. This can be used to provide the + username from a reverse proxy which authenticated the client upfront. + Requires validation, otherwise clients can supply the header themselves, + which then is unconditionally trusted. -`ldap` _(>= 3.3.0)_ -: Use a LDAP or AD server to authenticate users by relaying credentials from client and handle result. +* `ldap` _(>= 3.3.0)_ + Use a LDAP or AD server to authenticate users by relaying credentials from clients and handle results. -`dovecot` _(>= 3.3.1)_ -: Use a Dovecot server to authenticate users by relaying credentials from client and handle result. +* `dovecot` _(>= 3.3.1)_ + Use a Dovecot server to authenticate users by relaying credentials from clients and handle results. -`imap` _(>= 3.4.1)_ -: Use an IMAP server to authenticate users by relaying credentials from client and handle result. +* `imap` _(>= 3.4.1)_ + Use an IMAP server to authenticate users by relaying credentials from clients and handle results. -`oauth2` _(>= 3.5.0)_ -: Use an OAuth2 server to authenticate users by relaying credentials from client and handle result. - Oauth2 authentication (SSO) directly on client is not supported. Use herefore `http_x_remote_user` +* `oauth2` _(>= 3.5.0)_ + Use an OAuth2 server to authenticate users by relaying credentials from clients and handle results. + OAuth2 authentication (SSO) directly on client is not supported. Use herefore `http_x_remote_user` in combination with SSO support in reverse proxy (e.g. Apache+mod_auth_openidc). -`pam` _(>= 3.5.0)_ -: Use local PAM to authenticate users by relaying credentials from client and handle result.. +* `pam` _(>= 3.5.0)_ + Use local PAM to authenticate users by relaying credentials from client and handle result.. -Default: `none` _(< 3.5.0)_ `denyall` _(>= 3.5.0)_ +Default: `none` _(< 3.5.0)_ / `denyall` _(>= 3.5.0)_ ##### cache_logins @@ -906,7 +931,7 @@ _(>= 3.4.0)_ Cache successful/failed logins until expiration time. Enable this to avoid overload of authentication backends. -Default: `false` +Default: `False` ##### cache_successful_logins_expiry @@ -932,14 +957,15 @@ Default: `/etc/radicale/users` ##### htpasswd_encryption -The encryption method that is used in the htpasswd file. Use the +The encryption method that is used in the htpasswd file. Use [htpasswd](https://httpd.apache.org/docs/current/programs/htpasswd.html) or similar to generate this files. Available methods: -`plain` -: Passwords are stored in plaintext. This is obviously not secure! +* `plain` + Passwords are stored in plaintext. + This is not recommended. as it is obviously **insecure!** The htpasswd file for this can be created by hand and looks like: ```htpasswd @@ -947,27 +973,27 @@ Available methods: user2:password2 ``` -`bcrypt` -: This uses a modified version of the Blowfish stream cipher. It's very secure. - The installation of **bcrypt** is required for this. +* `bcrypt` + This uses a modified version of the Blowfish stream cipher, which is considered very secure. + The installation of Python's **bcrypt** module is required for this to work. -`md5` -: This uses an iterated MD5 digest of the password with a salt (nowadays insecure). +* `md5` + Use an iterated MD5 digest of the password with salt (nowadays insecure). -`sha256` _(>= 3.1.9)_ -: This uses an iterated SHA-256 digest of the password with a salt. +* `sha256` _(>= 3.1.9)_ + Use an iterated SHA-256 digest of the password with salt. -`sha512` _(>= 3.1.9)_ -: This uses an iterated SHA-512 digest of the password with a salt. +* `sha512` _(>= 3.1.9)_ + Use an iterated SHA-512 digest of the password with salt. -`argon2` _(>= 3.5.3)_ -: This uses an iterated ARGON2 digest of the password with a salt. - The installation of **argon2-cffi** is required for this. +* `argon2` _(>= 3.5.3)_ + Use an iterated ARGON2 digest of the password with salt. + The installation of Python's **argon2-cffi** module is required for this to work. -`autodetect` _(>= 3.1.9)_ -: This selects autodetection of method per entry. +* `autodetect` _(>= 3.1.9)_ + Automatically detect the encryption method used per user entry. -Default: `md5` _(< 3.3.0)_ `autodetect` _(>= 3.3.0)_ +Default: `md5` _(< 3.3.0)_ / `autodetect` _(>= 3.3.0)_ ##### htpasswd_cache @@ -979,7 +1005,7 @@ Default: `False` ##### delay -Average delay after failed login attempts in seconds. +Average delay (in seconds) after failed login attempts. Default: `1` @@ -1050,8 +1076,8 @@ _(>= 3.4.0)_ LDAP attribute whose value shall be used as the username after successful authentication. If set, you can use flexible logins in `ldap_filter` and still have consolidated usernames, -e.g. to allow login in using mail addresses as an alternative to cn, simply set -``` +e.g. to allow users to login using mail addresses as an alternative to cn, simply set +```ini ldap_filter = (&(objectclass=inetOrgPerson)(|(cn={0})(mail={0}))) ldap_user_attribute = cn ``` @@ -1065,13 +1091,18 @@ Default: (unset, in which case the login name is directly used as the username) _(>= 3.3.0)_ -Use ssl on the LDAP connection. **Deprecated**, use `ldap_security` instead**!** +Use ssl on the LDAP connection. **Deprecated!** Use `ldap_security` instead. ##### ldap_security _(>= 3.5.2)_ -Use encryption on the LDAP connection. One of `none`, `tls`, `starttls`. +Use encryption on the LDAP connection. + +One of +* `none` +* `tls` +* `starttls` Default: `none` @@ -1079,7 +1110,12 @@ Default: `none` _(>= 3.3.0)_ -Certificate verification mode for tls and starttls. One of `NONE`, `OPTIONAL`, `REQUIRED`. +Certificate verification mode for tls and starttls. + +One of +* `NONE` +* `OPTIONAL` +* `REQUIRED`. Default: `REQUIRED` @@ -1157,13 +1193,18 @@ Quirks for Authentik LDAP server, which violates the LDAP RFCs: add modifyTimestamp and createTimestamp to the exclusion list of internal ldap3 client so that these schema attributes are not checked. -Default: `false` +Default: `False` -##### dovecot_connection_type = AF_UNIX +##### dovecot_connection_type _(>= 3.4.1)_ -Connection type for dovecot authentication (AF_UNIX|AF_INET|AF_INET6) +Connection type for dovecot authentication. + +One of: +* `AF_UNIX` +* `AF_INET` +* `AF_INET6` Note: credentials are transmitted in cleartext @@ -1173,7 +1214,8 @@ Default: `AF_UNIX` _(>= 3.3.1)_ -The path to the Dovecot client authentication socket (eg. /run/dovecot/auth-client on Fedora). Radicale must have read / write access to the socket. +Path to the Dovecot client authentication socket (eg. /run/dovecot/auth-client on Fedora). +Radicale must have read & write access to the socket. Default: `/var/run/dovecot/auth-client` @@ -1181,7 +1223,7 @@ Default: `/var/run/dovecot/auth-client` _(>= 3.4.1)_ -Host of via network exposed dovecot socket +Host of dovecot socket exposed via network Default: `localhost` @@ -1189,7 +1231,7 @@ Default: `localhost` _(>= 3.4.1)_ -Port of via network exposed dovecot socket +Port of dovecot socket exposed via network Default: `12345` @@ -1225,7 +1267,13 @@ Default: `REMOTE_ADDR` _(>= 3.4.1)_ -IMAP server hostname: address | address:port | [address]:port | imap.server.tld +IMAP server hostname. + +One of: +* address +* address:port +* [address]:port (for IPv5 addresses) +* imap.server.tld Default: `localhost` @@ -1233,7 +1281,12 @@ Default: `localhost` _(>= 3.4.1)_ -Secure the IMAP connection: tls | starttls | none +Secure the IMAP connection: + +One of: +* `tls` +* `starttls` +* `none` Default: `tls` @@ -1241,17 +1294,17 @@ Default: `tls` _(>= 3.5.0)_ -OAuth2 token endpoint URL +Endpoint URL for the OAuth2 token -Default: +Default: (unset) ##### pam_service _(>= 3.5.0)_ -PAM service +PAM service name -Default: radicale +Default: `radicale` ##### pam_group_membership @@ -1259,31 +1312,31 @@ _(>= 3.5.0)_ PAM group user should be member of -Default: +Default: (unset) ##### lc_username -Сonvert username to lowercase, must be true for case-insensitive auth -providers like ldap, kerberos +Сonvert username to lowercase. +Recommended to be `True` for case-insensitive auth providers like ldap, kerberos, ... Default: `False` Notes: * `lc_username` and `uc_username` are mutually exclusive -* for auth type `ldap` the use of `ldap_user_attribute` is preferred +* for auth type `ldap` the use of `ldap_user_attribute` is preferred over `lc_username` ##### uc_username _(>= 3.3.2)_ -Сonvert username to uppercase, must be true for case-insensitive auth -providers like ldap, kerberos +Сonvert username to uppercase. +Recommended to be `True` for case-insensitive auth providers like ldap, kerberos, ... Default: `False` Notes: * `uc_username` and `lc_username` are mutually exclusive -* for auth type `ldap` the use of `ldap_user_attribute` is preferred +* for auth type `ldap` the use of `ldap_user_attribute` is preferred over `uc_username` ##### strip_domain @@ -1297,87 +1350,93 @@ Default: `False` _(>= 3.5.3)_ -URL Decode the username. When the username is an email, some clients send the username URL-encoded (notably iOS devices) -breaking the authentication process (user@example.com becomes user%40example.com). This setting will force decoding the username. +URL-decode the username. +If the username is an email address, some clients send the username URL-encoded +(notably iOS devices) breaking the authentication process +(user@example.com becomes user%40example.com). +This setting forces decoding the username. Default: `False` -#### rights +#### [rights] ##### type -The backend that is used to check the access rights of collections. +Authorization backend that is used to check the access rights to collections. -The recommended backend is `owner_only`. If access to calendars -and address books outside the home directory of users (that's `/USERNAME/`) -is granted, clients won't detect these collections and will not show them to -the user. Choosing any other method is only useful if you access calendars and -address books directly via URL. +The default and recommended backend is `owner_only`. If access to calendars +and address books outside the user's collection directory (that's `/username/`) +is granted, clients will not detect these collections automatically and +will not show them to the users. +Choosing any other authorization backend is only useful if you access +calendars and address books directly via URL. -Available backends: +Available backends are: -`authenticated` -: Authenticated users can read and write everything. +* `authenticated` + Authenticated users can read and write everything. -`owner_only` -: Authenticated users can read and write their own collections under the path +* `owner_only` + Authenticated users can read and write their own collections under the path */USERNAME/*. -`owner_write` -: Authenticated users can read everything and write their own collections under +* `owner_write` + Authenticated users can read everything and write their own collections under the path */USERNAME/*. -`from_file` -: Load the rules from a file. +* `from_file` + Load the rules from a file. Default: `owner_only` ##### file -File for the rights backend `from_file`. See the -[Rights](#authentication-and-rights) section. +Name of the file containing the authorization rules for the `from_file` backend. +See the [Rights](#authorization-and-rights) section for details. + +Default: `/etc/radicale/rights` ##### permit_delete_collection _(>= 3.1.9)_ -Global control of permission to delete complete collection (default: True) +Global permission to delete complete collections. +* If `False` it can be explicitly granted per collection by `permissions: D` +* If `True` it can be explicitly forbidden per collection by `permissions: d` -If False it can be permitted by permissions per section with: D - -If True it can be forbidden by permissions per section with: d +Default: `True` ##### permit_overwrite_collection _(>= 3.3.0)_ -Global control of permission to overwrite complete collection (default: True) +Global permission to overwrite complete collections. +* If `False` it can be explicitly granted per collection by `permissions: O` +* If `True` it can be explicitly forbidden per collection by `permissions: o` -If False it can be permitted by permissions per section with: O +Default: `True` -If True it can be forbidden by permissions per section with: o - -#### storage +#### [storage] ##### type -The backend that is used to store data. +Backend used to store data. -Available backends: +Available backends are: -`multifilesystem` -: Stores the data in the filesystem. +* `multifilesystem` + Stores the data in the filesystem. -`multifilesystem_nolock` -: The `multifilesystem` backend without file-based locking. +* `multifilesystem_nolock` + The `multifilesystem` backend without file-based locking. Must only be used with a single process. Default: `multifilesystem` ##### filesystem_folder -Folder for storing local collections, created if not present. +Folder for storing local collections; will be auto-created if not present. Default: `/var/lib/radicale/collections` @@ -1385,11 +1444,11 @@ Default: `/var/lib/radicale/collections` _(>= 3.3.2)_ -Folder for storing cache of local collections, created if not present +Folder for storing cache of local collections; will be auto-created if not present Default: (filesystem_folder) -Note: only used in case of use_cache_subfolder_* options are active +Note: only used if use_cache_subfolder_* options are active Note: can be used on multi-instance setup to cache files on local node (see below) @@ -1411,7 +1470,7 @@ Use subfolder `collection-cache` for cache file structure of 'history' instead o Default: `False` -Note: use only on single-instance setup, will break consistency with client in multi-instance setup +Note: only use on single-instance setup: it will break consistency with clients in multi-instance setup ##### use_cache_subfolder_for_synctoken @@ -1421,33 +1480,38 @@ Use subfolder `collection-cache` for cache file structure of 'sync-token' instea Default: `False` -Note: use only on single-instance setup, will break consistency with client in multi-instance setup +Note: only use on single-instance setup: it will break consistency with clients in multi-instance setup ##### use_mtime_and_size_for_item_cache _(>= 3.3.2)_ -Use last modifiction time (nanoseconds) and size (bytes) for 'item' cache instead of SHA256 (improves speed) +Use last modification time (in nanoseconds) and size (in bytes) for 'item' cache instead of SHA256 (improves speed) Default: `False` -Note: check used filesystem mtime precision before enabling - -Note: conversion is done on access, bulk conversion can be done offline using storage verification option `radicale --verify-storage` +Notes: +* check used filesystem mtime precision before enabling +* conversion is done on access +* bulk conversion can be done offline using the storage verification option `radicale --verify-storage` ##### folder_umask _(>= 3.3.2)_ -Use configured umask for folder creation (not applicable for OS Windows) +umask to use for folder creation (not applicable for OS Windows) -Default: (system-default, usual `0022`) +Default: (system-default, usually `0022`) -Useful value: `0077` (user:rw group:- other:-) or `0027` (user:rw group:r other:-) or `0007` (user:rw group:rw other:-) or `0022` (user:rw group:r other:r) +Useful values: +* `0077` (user:rw group:- other:-) +* `0027` (user:rw group:r other:-) +* `0007` (user:rw group:rw other:-) +* `0022` (user:rw group:r other:r) ##### max_sync_token_age -Delete sync-token that are older than the specified time. (seconds) +Delete sync-tokens that are older than the specified time (in seconds). Default: `2592000` @@ -1462,9 +1526,10 @@ Default: `True` ##### hook Command that is run after changes to storage. Take a look at the -[Versioning with Git](#versioning-with-git) tutorial for an example. +[Versioning collections with Git](#versioning-collections-with-git) +tutorial for an example. -Default: +Default: (unset) Supported placeholders: - `%(user)s`: logged-in user @@ -1473,53 +1538,58 @@ Supported placeholders: - `%(to_path)s`: full path of destination item (only set on MOVE request) _(>= 3.5.5)_ - `%(request)s`: request method _(>= 3.5.5)_ -Command will be executed with base directory defined in `filesystem_folder` (see above) +The command will be executed with base directory defined in `filesystem_folder` (see above) ##### predefined_collections -Create predefined user collections +Create predefined user collections. - Example: +Example: +```json +{ + "def-addressbook": { + "D:displayname": "Personal Address Book", + "tag": "VADDRESSBOOK" + }, + "def-calendar": { + "C:supported-calendar-component-set": "VEVENT,VJOURNAL,VTODO", + "D:displayname": "Personal Calendar", + "tag": "VCALENDAR" + } +} +``` +Default: (unset) - { - "def-addressbook": { - "D:displayname": "Personal Address Book", - "tag": "VADDRESSBOOK" - }, - "def-calendar": { - "C:supported-calendar-component-set": "VEVENT,VJOURNAL,VTODO", - "D:displayname": "Personal Calendar", - "tag": "VCALENDAR" - } - } - -Default: - -#### web +#### [web] ##### type The backend that provides the web interface of Radicale. -Available backends: +Available backends are: -`none` -: Just shows the message "Radicale works!". +* `none` + Simply shows the message "Radicale works!". -`internal` -: Allows creation and management of address books and calendars. +* `internal` + Allows creation and management of address books and calendars. Default: `internal` -#### logging +#### [logging] ##### level Set the logging level. -Available levels: **debug**, **info**, **warning**, **error**, **critical** +Available levels are: +* `debug` +* `info` +* `warning` +* `error` +* `critical` -Default: `warning` _(< 3.2.0)_ `info` _(>= 3.2.0)_ +Default: `warning` _(< 3.2.0)_ / `info` _(>= 3.2.0)_ ##### trace_on_debug @@ -1535,13 +1605,13 @@ _(> 3.5.4)_ Filter debug messages starting with 'TRACE/' -Precondition: `trace_on_debug = True` +Prerequisite: `trace_on_debug = True` Default: (empty) ##### mask_passwords -Don't include passwords in logs. +Do not include passwords in logs. Default: `True` @@ -1557,7 +1627,7 @@ Default: `False` _(>= 3.2.2)_ -Log backtrace on level=debug +Log backtrace on `level = debug` Default: `False` @@ -1565,7 +1635,7 @@ Default: `False` _(>= 3.2.2)_ -Log request on level=debug +Log request on `level = debug` Default: `False` @@ -1573,7 +1643,7 @@ Default: `False` _(>= 3.2.2)_ -Log request on level=debug +Log request on `level = debug` Default: `False` @@ -1581,7 +1651,7 @@ Default: `False` _(>= 3.2.2)_ -Log response on level=debug +Log response on `level = debug` Default: `False` @@ -1589,7 +1659,7 @@ Default: `False` _(>= 3.2.3)_ -Log rights rule which doesn't match on level=debug +Log rights rule which doesn't match on `level = debug` Default: `False` @@ -1597,14 +1667,13 @@ Default: `False` _(>= 3.3.2)_ -Log storage cache actions on level=debug +Log storage cache actions on `level = debug` Default: `False` -#### headers +#### [headers] -In this section additional HTTP headers that are sent to clients can be -specified. +This section can be used to specify additional HTTP headers that will be sent to clients. An example to relax the same-origin policy: @@ -1612,21 +1681,22 @@ An example to relax the same-origin policy: Access-Control-Allow-Origin = * ``` -#### hook +#### [hook] + ##### type Hook binding for event changes and deletion notifications. -Available types: +Available types are: -`none` -: Disabled. Nothing will be notified. +* `none` + Disabled. Nothing will be notified. -`rabbitmq` _(>= 3.2.0)_ -: Push the message to the rabbitmq server. +* `rabbitmq` _(>= 3.2.0)_ + Push the message to the rabbitmq server. -`email` _(>= 3.5.5)_ -: Send an email notification to event attendees. +* `email` _(>= 3.5.5)_ + Send an email notification to event attendees. Default: `none` @@ -1634,7 +1704,7 @@ Default: `none` _(> 3.5.4)_ -Dry-Run (do not really trigger hook action) +Dry-Run / simulate (i.e. do not really trigger) the hook action. Default: `False` @@ -1643,17 +1713,17 @@ Default: `False` _(>= 3.2.0)_ End-point address for rabbitmq server. -Ex: amqp://user:password@localhost:5672/ +E.g.: `amqp://user:password@localhost:5672/` -Default: +Default: (unset) ##### rabbitmq_topic _(>= 3.2.0)_ -RabbitMQ topic to publish message. +RabbitMQ topic to publish message in. -Default: +Default: (unset) ##### rabbitmq_queue_type @@ -1661,21 +1731,21 @@ _(>= 3.2.0)_ RabbitMQ queue type for the topic. -Default: classic +Default: `classic` ##### smtp_server _(>= 3.5.5)_ -Address to connect to SMTP server. +Address of SMTP server to connect to. -Default: +Default: (unset) ##### smtp_port _(>= 3.5.5)_ -Port to connect to SMTP server. +Port on SMTP server to connect to. Default: @@ -1683,33 +1753,45 @@ Default: _(>= 3.5.5)_ -Use encryption on the SMTP connection. none, tls, starttls +Use encryption on the SMTP connection. -Default: none +One of: +* `none` +* `tls` +* `starttls` + +Default: `none` ##### smtp_ssl_verify_mode _(>= 3.5.5)_ -The certificate verification mode. Works for tls and starttls. NONE, OPTIONAL or REQUIRED +The certificate verification mode for tls and starttls. -Default: REQUIRED +One of: +* `NONE` +* `OPTIONAL` +* `REQUIRED` + +Default: `REQUIRED` ##### smtp_username _(>= 3.5.5)_ -Username to authenticate with SMTP server. Leave empty to disable authentication (e.g. using local mail server). +Username to authenticate with SMTP server. +Leave empty to disable authentication (e.g. using local mail server). -Default: +Default: (unset) ##### smtp_password _(>= 3.5.5)_ -Password to authenticate with SMTP server. Leave empty to disable authentication (e.g. using local mail server). +Password to authenticate with SMTP server. +Leave empty to disable authentication (e.g. using local mail server). -Default: +Default: (unset) ##### from_email @@ -1717,13 +1799,14 @@ _(>= 3.5.5)_ Email address to use as sender in email notifications. -Default: +Default: (unset) ##### mass_email _(>= 3.5.5)_ -When enabled, send one email to all attendee email addresses. When disabled, send one email per attendee email address. +When enabled, send one email to all attendee email addresses. +When disabled, send one email per attendee email address. Default: `False` @@ -1731,16 +1814,16 @@ Default: `False` _(>= 3.5.5)_ -Template to use for added/updated event email body (sent to an attendee when the event is created or they are added to a pre-existing event). +Template to use for added/updated event email body sent to an attendee when the event is created or they are added to a pre-existing event. The following placeholders will be replaced: -- `$organizer_name`: Name of the organizer, or "Unknown Organizer" if not set in event -- `$from_email`: Email address the email is sent from -- `$attendee_name`: Name of the attendee (email recipient), or "everyone" if mass email enabled. -- `$event_name`: Name/summary of the event, or "No Title" if not set in event -- `$event_start_time`: Start time of the event in ISO 8601 format -- `$event_end_time`: End time of the event in ISO 8601 format, or "No End Time" if the event has no end time -- `$event_location`: Location of the event, or "No Location Specified" if not set in event +* `$organizer_name`: Name of the organizer, or "Unknown Organizer" if not set in event +* `$from_email`: Email address the email is sent from +* `$attendee_name`: Name of the attendee (email recipient), or "everyone" if mass email enabled. +* `$event_name`: Name/summary of the event, or "No Title" if not set in event +* `$event_start_time`: Start time of the event in ISO 8601 format +* `$event_end_time`: End time of the event in ISO 8601 format, or "No End Time" if the event has no end time +* `$event_location`: Location of the event, or "No Location Specified" if not set in event Providing any words prefixed with $ not included in the list above will result in an error. @@ -1761,20 +1844,20 @@ This is an automated message. Please do not reply. _(>= 3.5.5)_ -Template to use for deleted/removed event email body (sent to an attendee when the event is deleted or they are removed from the event). +Template to use for deleted/removed event email body sent to an attendee when the event is deleted or they are removed from the event. The following placeholders will be replaced: -- `$organizer_name`: Name of the organizer, or "Unknown Organizer" if not set in event -- `$from_email`: Email address the email is sent from -- `$attendee_name`: Name of the attendee (email recipient), or "everyone" if mass email enabled. -- `$event_name`: Name/summary of the event, or "No Title" if not set in event -- `$event_start_time`: Start time of the event in ISO 8601 format -- `$event_end_time`: End time of the event in ISO 8601 format, or "No End Time" if the event has no end time -- `$event_location`: Location of the event, or "No Location Specified" if not set in event +* `$organizer_name`: Name of the organizer, or "Unknown Organizer" if not set in event +* `$from_email`: Email address the email is sent from +* `$attendee_name`: Name of the attendee (email recipient), or "everyone" if mass email enabled. +* `$event_name`: Name/summary of the event, or "No Title" if not set in event +* `$event_start_time`: Start time of the event in ISO 8601 format +* `$event_end_time`: End time of the event in ISO 8601 format, or "No End Time" if the event has no end time +* `$event_location`: Location of the event, or "No Location Specified" if not set in event Providing any words prefixed with $ not included in the list above will result in an error. -Default: +Default: ``` Hello $attendee_name, @@ -1787,26 +1870,26 @@ The following event has been deleted. This is an automated message. Please do not reply. ``` -#### updated_event_template +##### updated_event_template _(>= 3.5.5)_ -Template to use for updated event email body (sent to an attendee when non-attendee-related details of the event are updated). +Template to use for updated event email body sent to an attendee when non-attendee-related details of the event are updated. Existing attendees will NOT be notified of a modified event if the only changes are adding/removing other attendees. The following placeholders will be replaced: -- `$organizer_name`: Name of the organizer, or "Unknown Organizer" if not set in event -- `$from_email`: Email address the email is sent from -- `$attendee_name`: Name of the attendee (email recipient), or "everyone" if mass email enabled. -- `$event_name`: Name/summary of the event, or "No Title" if not set in event -- `$event_start_time`: Start time of the event in ISO 8601 format -- `$event_end_time`: End time of the event in ISO 8601 format, or "No End Time" if the event has no end time -- `$event_location`: Location of the event, or "No Location Specified" if not set in event +* `$organizer_name`: Name of the organizer, or "Unknown Organizer" if not set in event +* `$from_email`: Email address the email is sent from +* `$attendee_name`: Name of the attendee (email recipient), or "everyone" if mass email enabled. +* `$event_name`: Name/summary of the event, or "No Title" if not set in event +* `$event_start_time`: Start time of the event in ISO 8601 format +* `$event_end_time`: End time of the event in ISO 8601 format, or "No End Time" if the event has no end time +* `$event_location`: Location of the event, or "No Location Specified" if not set in event Providing any words prefixed with $ not included in the list above will result in an error. -Default: +Default: ``` Hello $attendee_name, @@ -1819,7 +1902,7 @@ The following event has been updated. This is an automated message. Please do not reply. ``` -#### reporting +#### [reporting] ##### max_freebusy_occurrence @@ -1844,6 +1927,8 @@ Radicale has been tested with: * [GNOME Calendar](https://wiki.gnome.org/Apps/Calendar), [Contacts](https://wiki.gnome.org/Apps/Contacts) and [Evolution](https://wiki.gnome.org/Apps/Evolution) +* [KDE PIM Applications](https://kontact.kde.org/), + [KDE Merkuro](https://apps.kde.org/de/merkuro/) * [Mozilla Thunderbird](https://www.mozilla.org/thunderbird/) ([Thunderbird/Radicale](https://github.com/Kozea/Radicale/wiki/Client-Thunderbird)) with [CardBook](https://addons.mozilla.org/thunderbird/addon/cardbook/) and [Lightning](https://www.mozilla.org/projects/calendar/) @@ -1857,10 +1942,9 @@ Many clients do not support the creation of new calendars and address books. You can use Radicale's web interface (e.g. ) to create and manage address books and calendars. -In some clients you can just enter the URL of the Radicale server +In some clients, it is sufficient to simply enter the URL of the Radicale server (e.g. `http://localhost:5232`) and your username. In others, you have to -enter the URL of the collection directly -(e.g. `http://localhost:5232/user/calendar`). +enter the URL of the collection directly (e.g. `http://localhost:5232/user/calendar`). Some clients (notably macOS's Calendar.app) may silently refuse to include account credentials over unsecured HTTP, leading to unexpected authentication @@ -1871,7 +1955,7 @@ failures. In these cases, you want to make sure the Radicale server is Enter the URL of the Radicale server (e.g. `http://localhost:5232`) and your username. DAVx⁵ will show all existing calendars and address books and you -can create new. +can create new ones. #### OneCalendar @@ -1884,7 +1968,10 @@ you want to see. OneCalendar supports many other server types too. GNOME 46 added CalDAV and CardDAV support to _GNOME Online Accounts_. -Open GNOME Settings, navigate to _Online Accounts_ > _Connect an Account_ > _Calendar, Contacts and Files_. Enter the URL (e.g. `https://example.com/radicale`) and your credentials then click _Sign In_. In the pop-up dialog, turn off _Files_. After adding Radicale in _GNOME Online Accounts_, it should be available in GNOME Contacts and GNOME Calendar. +Open GNOME Settings, navigate to _Online Accounts_ > _Connect an Account_ > _Calendar, Contacts and Files_. +Enter the URL (e.g. `https://example.com/radicale`) and your credentials then click _Sign In_. +In the pop-up dialog, turn off _Files_. After adding Radicale in _GNOME Online Accounts_, +it should be available in GNOME Contacts and GNOME Calendar. #### Evolution @@ -1893,7 +1980,16 @@ Enter the URL of the Radicale server (e.g. `http://localhost:5232`) and your username. Clicking on the search button will list the existing calendars and address books. -Adding CalDAV and CardDAV accounts in Evolution will automatically make them available in GNOME Contacts and GNOME Calendar. +Adding CalDAV and CardDAV accounts in Evolution will automatically make them +available in GNOME Contacts and GNOME Calendar. + +#### KDE PIM Applications + +In **Kontact** add a _DAV Groupware resource_ to Akonadi under +_Settings > Configure Kontact > Calendar > General > Calendars_, +select the protocol (CalDAV or CardDAV), add the URL to the Radicale collections +and enter the credentials. After synchronization of the calendar resp. +addressbook items, you can manage them in Kontact. #### Thunderbird @@ -1980,16 +2076,16 @@ curl -u user -X DELETE 'http://localhost:5232/user/calendar' Note: requires config/option `permit_delete_collection = True` -### Authentication and Rights +### Authorization and Rights This section describes the format of the rights file for the `from_file` authentication backend. The configuration option `file` in the `rights` section must point to the rights file. -The recommended rights method is `owner_only`. If access to calendars -and address books outside the home directory of users (that's `/USERNAME/`) -is granted, clients won't detect these collections and will not show them to -the user. +The recommended rights method is `owner_only`. If access is granted +to calendars and address books outside the home directory of users +(that's `/USERNAME/`), clients will not detect these collections automatically, +and will not show them to the users. This is only useful if you access calendars and address books directly via URL. An example rights file: @@ -2041,40 +2137,40 @@ The following `permissions` are recognized: (CalDAV/CardDAV is susceptible to expensive search requests) * **W:** write collections (excluding address books and calendars) * **w:** write address book and calendar collections -* **D:** permit delete of collection in case permit_delete_collection=False _(>= 3.3.0)_ -* **d:** forbid delete of collection in case permit_delete_collection=True _(>= 3.3.0)_ -* **O:** permit overwrite of collection in case permit_overwrite_collection=False -* **o:** forbid overwrite of collection in case permit_overwrite_collection=True +* **D:** permit delete of collection in case `permit_delete_collection=False` _(>= 3.3.0)_ +* **d:** forbid delete of collection in case `permit_delete_collection=True` _(>= 3.3.0)_ +* **O:** permit overwrite of collection in case `permit_overwrite_collection=False` +* **o:** forbid overwrite of collection in case `permit_overwrite_collection=True` ### Storage -This document describes the layout and format of the file system storage -(`multifilesystem` backend). +This document describes the layout and format of the file system storage, +the `multifilesystem` backend. -It's safe to access and manipulate the data by hand or with scripts. -Scripts can be invoked manually, periodically (e.g. with +It is safe to access and manipulate the data by hand or with scripts. +Scripts can be invoked manually, periodically (e.g. using [cron](https://manpages.debian.org/unstable/cron/cron.8.en.html)) or after each change to the storage with the configuration option `hook` in the `storage` -section (e.g. [Versioning with Git](#versioning-with-git)). +section (e.g. [Versioning collections with Git](#versioning-collections-with-git)). #### Layout -The file system contains the following files and folders: - +The file system comprises the following files and folders: * `.Radicale.lock`: The lock file for locking the storage. * `collection-root`: This folder contains all collections and items. -A collection is represented by a folder. This folder may contain the file +Each collection is represented by a folder. This folder may contain the file `.Radicale.props` with all WebDAV properties of the collection encoded as [JSON](https://en.wikipedia.org/wiki/JSON). -An item is represented by a file containing the iCalendar data. +Each item in a calendar or address book collection is represented by +a file containing the item's iCalendar resp. vCard data. -All files and folders, whose names start with a dot but not `.Radicale.` +All files and folders, whose names start with a dot but not with `.Radicale.` (internal files) are ignored. -If you introduce syntax errors in any of the files, all requests that access -the faulty data will fail. The logging output should contain the names of the +Syntax errors in any of the files will cause all requests accessing +the faulty data to fail. The logging output should contain the names of the culprits. Caches and sync-tokens are stored in the `.Radicale.cache` folder inside of @@ -2082,14 +2178,14 @@ collections. This folder may be created or modified, while the storage is locked for shared access. In theory, it should be safe to delete the folder. Caches will be recreated -automatically and clients will be told that their sync-token isn't valid +automatically and clients will be told that their sync-token is not valid anymore. You may encounter files or folders that start with `.Radicale.tmp-`. Radicale uses them for atomic creation and deletion of files and folders. -They should be deleted after requests are finished but it's possible that +They should be deleted after requests are finished but it is possible that they are left behind when Radicale or the computer crashes. -It's safe to delete them. +You can safely delete them. #### Locking @@ -2102,12 +2198,13 @@ The storage is locked with exclusive access while the `hook` runs. Use the [flock](https://manpages.debian.org/unstable/util-linux/flock.1.en.html) -utility. +utility to acquire exclusive or shared locks for the commands you want to run +on Radicale's data. ```bash -# Exclusive +# Exclusive lock for COMMAND $ flock --exclusive /path/to/storage/.Radicale.lock COMMAND -# Shared +# Shared lock for COMMAND $ flock --shared /path/to/storage/.Radicale.lock COMMAND ``` @@ -2129,17 +2226,17 @@ and `nNumberOfBytesToLockHigh` to `0` works. #### Manually creating collections -To create a new collection, you have to create the corresponding folder in the +To create a new collection, you need to create the corresponding folder in the file system storage (e.g. `collection-root/user/calendar`). -To tell Radicale and clients that the collection is a calendar, you have to +To indicate to Radicale and clients that the collection is a calendar, you have to create the file ``.Radicale.props`` with the following content in the folder: ```json {"tag": "VCALENDAR"} ``` -The calendar is now available at the URL path ``/user/calendar``. -For address books the file must contain: +The calendar is now available at the URL path (e.g. ``/user/calendar``). +For address books ``.Radicale.props`` must contain: ```json {"tag": "VADDRESSBOOK"} @@ -2179,12 +2276,12 @@ an address book through network: Radicale is **only the server part** of this architecture. -Please note that: +Please note: -* CalDAV and CardDAV are superset protocols of WebDAV, -* WebDAV is a superset protocol of HTTP. +* CalDAV and CardDAV are extension protocols of WebDAV, +* WebDAV is an extension of the HTTP protocol. -Radicale being a CalDAV/CardDAV server, it also can be seen as a special WebDAV +Radicale being a CalDAV/CardDAV server, can also be seen as a special WebDAV and HTTP server. Radicale is **not the client part** of this architecture. It means that @@ -2200,59 +2297,59 @@ icons and buttons, a terminal or another web application. The ``radicale`` package offers the following modules. -`__init__` -: Contains the entry point for WSGI. +* `__init__` + : Contains the entry point for WSGI. -`__main__` -: Provides the entry point for the ``radicale`` executable and +* `__main__` + : Provides the entry point for the ``radicale`` executable and includes the command line parser. It loads configuration files from the default (or specified) paths and starts the internal server. -`app` -: This is the core part of Radicale, with the code for the CalDAV/CardDAV +* `app` + : This is the core part of Radicale, with the code for the CalDAV/CardDAV server. The code managing the different HTTP requests according to the CalDAV/CardDAV specification can be found here. -`auth` -: Used for authenticating users based on username and password, mapping +* `auth` + : Used for authenticating users based on username and password, mapping usernames to internal users and optionally retrieving credentials from the environment. -`config` -: Contains the code for managing configuration and loading settings from files. +* `config` + : Contains the code for managing configuration and loading settings from files. -`ìtem` -: Internal representation of address book and calendar entries. Based on +* `ìtem` + : Internal representation of address book and calendar entries. Based on [VObject](https://github.com/py-vobject/vobject/). -`log` -: The logger for Radicale based on the default Python logging module. +* `log` + : The logger for Radicale based on the default Python logging module. -`rights` -: This module is used by Radicale to manage access rights to collections, +* `rights` + : This module is used by Radicale to manage access rights to collections, address books and calendars. -`server` +* `server` : The integrated HTTP server for standalone use. -`storage` -: This module contains the classes representing collections in Radicale and +* `storage` + : This module contains the classes representing collections in Radicale and the code for storing and loading them in the filesystem. -`web` -: This module contains the web interface. +* `web` + : This module contains the web interface. -`utils` -: Contains general helper functions. +* `utils` + : Contains general helper functions. -`httputils` -: Contains helper functions for working with HTTP. +* `httputils` + : Contains helper functions for working with HTTP. -`pathutils` -: Helper functions for working with paths and the filesystem. +* `pathutils` + : Helper functions for working with paths and the filesystem. -`xmlutils` -: Helper functions for working with the XML part of CalDAV/CardDAV requests +* `xmlutils` + : Helper functions for working with the XML part of CalDAV/CardDAV requests and responses. It's based on the ElementTree XML API. ### Plugins @@ -2260,7 +2357,7 @@ The ``radicale`` package offers the following modules. Radicale can be extended by plugins for authentication, rights management and storage. Plugins are **python** modules. -#### Getting started +#### Getting started with plugin development To get started we walk through the creation of a simple authentication plugin, that accepts login attempts with a static password. From de1ce0d1d3c0ff9a77725338ce8a943a213ddd36 Mon Sep 17 00:00:00 2001 From: Peter Bieringer Date: Thu, 11 Sep 2025 08:01:09 +0200 Subject: [PATCH 045/290] manual apply from https://github.com/Kozea/Radicale/pull/1866 --- contrib/nginx/radicale.conf | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/contrib/nginx/radicale.conf b/contrib/nginx/radicale.conf index 990ebe4d..80369e27 100644 --- a/contrib/nginx/radicale.conf +++ b/contrib/nginx/radicale.conf @@ -8,7 +8,7 @@ rewrite ^/.well-known/caldav /radicale/ redirect; ## Base URI: /radicale/ location /radicale/ { - proxy_pass http://localhost:5232/; + proxy_pass http://localhost:5232; proxy_set_header X-Script-Name /radicale; proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; proxy_set_header X-Forwarded-Host $host; @@ -20,7 +20,7 @@ location /radicale/ { ## Base URI: / #location / { -# proxy_pass http://localhost:5232/; +# proxy_pass http://localhost:5232; # proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; # proxy_set_header X-Forwarded-Host $host; # proxy_set_header X-Forwarded-Port $server_port; From fba2a7caef242a491c3ba17129e20f7cc9bd5e2d Mon Sep 17 00:00:00 2001 From: Peter Bieringer Date: Thu, 11 Sep 2025 08:02:40 +0200 Subject: [PATCH 046/290] according to https://github.com/Kozea/Radicale/pull/1866 the trailing / should be removed --- DOCUMENTATION.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/DOCUMENTATION.md b/DOCUMENTATION.md index cde35fdb..1560ca90 100644 --- a/DOCUMENTATION.md +++ b/DOCUMENTATION.md @@ -410,7 +410,7 @@ See also for latest examples: https://github.com/Kozea/Radicale/tree/master/cont ```nginx location /radicale/ { # The trailing / is important! - proxy_pass http://localhost:5232/; # The / is important! + proxy_pass http://localhost:5232; proxy_set_header X-Script-Name /radicale; proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; proxy_set_header X-Forwarded-Host $host; From a04a9ba8be8034649a06b6310d186980d2cf42c8 Mon Sep 17 00:00:00 2001 From: Peter Bieringer Date: Mon, 15 Sep 2025 20:40:50 +0200 Subject: [PATCH 047/290] release 3.5.6 --- CHANGELOG.md | 2 +- pyproject.toml | 2 +- setup.py.legacy | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 8ef6be2c..48cce209 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,6 @@ # Changelog -## 3.5.6.dev +## 3.5.6 * Fix: broken start when UID does not exist (potential container startup case) * Improve: user/group retrievement for running service and directories * Extend/Improve: [auth] ldap: group membership lookup diff --git a/pyproject.toml b/pyproject.toml index a3f317bf..962d289e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -3,7 +3,7 @@ name = "Radicale" # When the version is updated, a new section in the CHANGELOG.md file must be # added too. readme = "README.md" -version = "3.5.6.dev" +version = "3.5.6" authors = [{name = "Guillaume Ayoub", email = "guillaume.ayoub@kozea.fr"}, {name = "Unrud", email = "unrud@outlook.com"}, {name = "Peter Bieringer", email = "pb@bieringer.de"}] license = {text = "GNU GPL v3"} description = "CalDAV and CardDAV Server" diff --git a/setup.py.legacy b/setup.py.legacy index 12079205..c276f0b1 100644 --- a/setup.py.legacy +++ b/setup.py.legacy @@ -20,7 +20,7 @@ from setuptools import find_packages, setup # When the version is updated, a new section in the CHANGELOG.md file must be # added too. -VERSION = "3.5.6.dev" +VERSION = "3.5.6" with open("README.md", encoding="utf-8") as f: long_description = f.read() From 243931000b0cca40786036dfb7d2fa0cbb806cb7 Mon Sep 17 00:00:00 2001 From: Peter Marschall Date: Fri, 19 Sep 2025 17:44:05 +0200 Subject: [PATCH 048/290] DOCUMENTATION.md: fix small glitches: typos, ... --- DOCUMENTATION.md | 32 ++++++++++++++++---------------- 1 file changed, 16 insertions(+), 16 deletions(-) diff --git a/DOCUMENTATION.md b/DOCUMENTATION.md index 1560ca90..42aa64b6 100644 --- a/DOCUMENTATION.md +++ b/DOCUMENTATION.md @@ -166,7 +166,7 @@ The `users` file can be created and managed with Note: some OSes or distributions contain outdated versions of `htpasswd` (< 2.4.59) without support for SHA-256 or SHA-512 (e.g. Ubuntu LTS 22). -In these cases use `htpasswd`'s command line option `-B` for the `bcrypt` hash method (recommended), +In these cases, use `htpasswd`'s command line option `-B` for the `bcrypt` hash method (recommended), or stay with the insecure (not recommended) MD5 (default) or SHA-1 (command line option `-s`). Note: support of SHA-256 and SHA-512 was introduced with 3.1.9 @@ -273,13 +273,13 @@ Create the **radicale** user and group for the Radicale service by running (as ` useradd --system --user-group --home-dir / --shell /sbin/nologin radicale ``` -The storage folder must be writable by the **radicale** user by running (as `root`): +The storage folder must be made writable by the **radicale** user by running (as `root`): ```bash mkdir -p /var/lib/radicale/collections && chown -R radicale:radicale /var/lib/radicale/collections ``` If a dedicated cache folder is configured (see option [filesystem_cache_folder](#filesystem_cache_folder)), -it also must be also writable by **radicale**. To achieva that, run (as `root`): +it also must be made writable by **radicale**. To achieve that, run (as `root`): ```bash mkdir -p /var/cache/radicale && chown -R radicale:radicale /var/cache/radicale ```` @@ -573,10 +573,10 @@ RequestHeader set X-Remote-User expr=%{REMOTE_USER} #### Secure connection between Radicale and the reverse proxy SSL certificates can be used to encrypt and authenticate the connection between -Radicale and the reverse proxy. First you have to generate a certificate for +Radicale and the reverse proxy. First you need to generate a certificate for Radicale and a certificate for the reverse proxy. The following commands generate self-signed certificates. You will be asked to enter additional -information about the certificate, these values do not really matter and you can +information about the certificate, these values do not really matter, and you can keep the defaults. ```bash @@ -811,7 +811,7 @@ Default: `False` ##### certificate -Path of the SSL certifcate. +Path of the SSL certificate. Default: `/etc/ssl/radicale.cert.pem` @@ -959,7 +959,7 @@ Default: `/etc/radicale/users` The encryption method that is used in the htpasswd file. Use [htpasswd](https://httpd.apache.org/docs/current/programs/htpasswd.html) -or similar to generate this files. +or similar to generate this file. Available methods: @@ -1140,7 +1140,7 @@ If set, get the user's LDAP groups from the attribute given. For DN-valued attributes, the value of the RDN is used to determine the group names. The implementation also supports non-DN-valued attributes: their values are taken directly. -The user's group names can be used later on to define rights. +The user's group names can be used later to define rights. They also give you access to the group calendars, if those exist. * Group calendars are placed directly under *collection_root_folder*`/GROUPS/` with the base64-encoded group name as the calendar folder name. @@ -1525,7 +1525,7 @@ Default: `True` ##### hook -Command that is run after changes to storage. Take a look at the +Command that is run after changes to storage. See the [Versioning collections with Git](#versioning-collections-with-git) tutorial for an example. @@ -1959,7 +1959,7 @@ can create new ones. #### OneCalendar -When adding account, select CalDAV account type, then enter user name, password and the +When adding account, select CalDAV account type, then enter username, password and the Radicale server (e.g. `https://yourdomain:5232`). OneCalendar will show all existing calendars and (FIXME: address books), you need to select which ones you want to see. OneCalendar supports many other server types too. @@ -2006,8 +2006,8 @@ It will list your existing address books. #### InfCloud, CalDavZAP and CardDavMATE You can integrate InfCloud into Radicale's web interface with by simply -download latest package from [InfCloud](https://www.inf-it.com/open-source/clients/infcloud/) -and extract content to new folder `infcloud` in `radicale/web/internal_data/`. +downloading the latest package from [InfCloud](https://www.inf-it.com/open-source/clients/infcloud/) +and extract the content into a folder named `infcloud` in `radicale/web/internal_data/`. No further adjustments are required as content is adjusted on the fly (tested with 0.13.1). @@ -2137,10 +2137,10 @@ The following `permissions` are recognized: (CalDAV/CardDAV is susceptible to expensive search requests) * **W:** write collections (excluding address books and calendars) * **w:** write address book and calendar collections -* **D:** permit delete of collection in case `permit_delete_collection=False` _(>= 3.3.0)_ -* **d:** forbid delete of collection in case `permit_delete_collection=True` _(>= 3.3.0)_ -* **O:** permit overwrite of collection in case `permit_overwrite_collection=False` -* **o:** forbid overwrite of collection in case `permit_overwrite_collection=True` +* **D:** allow deleting a collection in case `permit_delete_collection=False` _(>= 3.3.0)_ +* **d:** deny deleting a collection in case `permit_delete_collection=True` _(>= 3.3.0)_ +* **O:** allow overwriting a collection in case `permit_overwrite_collection=False` +* **o:** deny overwriting a collection in case `permit_overwrite_collection=True` ### Storage From 4b2e63dafed2fa4efb0cc6a88c48d0ac3db0d60f Mon Sep 17 00:00:00 2001 From: Peter Bieringer Date: Wed, 24 Sep 2025 06:24:31 +0200 Subject: [PATCH 049/290] prepare 3.5.7.dev --- CHANGELOG.md | 2 ++ pyproject.toml | 2 +- setup.py.legacy | 2 +- 3 files changed, 4 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 48cce209..3b4ececa 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,7 @@ # Changelog +## 3.5.7.dev + ## 3.5.6 * Fix: broken start when UID does not exist (potential container startup case) * Improve: user/group retrievement for running service and directories diff --git a/pyproject.toml b/pyproject.toml index 962d289e..6816745a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -3,7 +3,7 @@ name = "Radicale" # When the version is updated, a new section in the CHANGELOG.md file must be # added too. readme = "README.md" -version = "3.5.6" +version = "3.5.7.dev" authors = [{name = "Guillaume Ayoub", email = "guillaume.ayoub@kozea.fr"}, {name = "Unrud", email = "unrud@outlook.com"}, {name = "Peter Bieringer", email = "pb@bieringer.de"}] license = {text = "GNU GPL v3"} description = "CalDAV and CardDAV Server" diff --git a/setup.py.legacy b/setup.py.legacy index c276f0b1..f8af3ead 100644 --- a/setup.py.legacy +++ b/setup.py.legacy @@ -20,7 +20,7 @@ from setuptools import find_packages, setup # When the version is updated, a new section in the CHANGELOG.md file must be # added too. -VERSION = "3.5.6" +VERSION = "3.5.7.dev" with open("README.md", encoding="utf-8") as f: long_description = f.read() From b46916fca9a835b13015e798758bdb13201dd07b Mon Sep 17 00:00:00 2001 From: Peter Bieringer Date: Wed, 24 Sep 2025 06:35:27 +0200 Subject: [PATCH 050/290] fix according to https://github.com/Kozea/Radicale/issues/1878#issue-3438629348 --- radicale/auth/dovecot.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/radicale/auth/dovecot.py b/radicale/auth/dovecot.py index 479f4111..bffed4ed 100644 --- a/radicale/auth/dovecot.py +++ b/radicale/auth/dovecot.py @@ -92,6 +92,7 @@ class Auth(auth.BaseAuth): # Hence, we try to read just once with a buffer big # enough to hold all of it. buf = sock.recv(1024) + version_sent = False while b'\n' in buf and not done: line, buf = buf.split(b'\n', 1) parts = line.split(b'\t') @@ -114,6 +115,10 @@ class Auth(auth.BaseAuth): ) return "" seen_part[0] += 1 + if int(version[1]) >= 3: + sock.send(b'VERSION\t1\t1\n') + buf += sock.recv(1024) + version_sent = True elif first == b'MECH': supported_mechs.append(parts[0]) seen_part[1] += 1 @@ -144,7 +149,8 @@ class Auth(auth.BaseAuth): # Handshake logger.debug("Sending auth handshake") - sock.send(b'VERSION\t1\t1\n') + if not version_sent: + sock.send(b'VERSION\t1\t1\n') sock.send(b'CPID\t%u\n' % os.getpid()) request_id = next(self.request_id_gen) From 8d224f976862012f60c485cdffb6117f8006af15 Mon Sep 17 00:00:00 2001 From: Peter Bieringer Date: Wed, 24 Sep 2025 06:35:48 +0200 Subject: [PATCH 051/290] changelog for https://github.com/Kozea/Radicale/issues/1878 --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 3b4ececa..a4ff0bf5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,7 @@ # Changelog ## 3.5.7.dev +* Extend: [auth] dovecot: add support for version >= 2.4 ## 3.5.6 * Fix: broken start when UID does not exist (potential container startup case) From d1679a53b1d4983d59cbe3ce8552187543e03d3d Mon Sep 17 00:00:00 2001 From: Peter Bieringer Date: Wed, 24 Sep 2025 21:15:20 +0200 Subject: [PATCH 052/290] new test items --- .../tests/static/event_issue1812_getetag.ics | 30 ++ radicale/tests/static/event_issue1880_1.ics | 29 ++ radicale/tests/static/event_issue1880_2.ics | 396 ++++++++++++++++++ 3 files changed, 455 insertions(+) create mode 100644 radicale/tests/static/event_issue1812_getetag.ics create mode 100644 radicale/tests/static/event_issue1880_1.ics create mode 100644 radicale/tests/static/event_issue1880_2.ics diff --git a/radicale/tests/static/event_issue1812_getetag.ics b/radicale/tests/static/event_issue1812_getetag.ics new file mode 100644 index 00000000..8b9936eb --- /dev/null +++ b/radicale/tests/static/event_issue1812_getetag.ics @@ -0,0 +1,30 @@ +BEGIN:VCALENDAR +VERSION:2.0 +PRODID:-//algoo.fr//NONSGML Open Calendar v0.9//EN +BEGIN:VTIMEZONE +TZID:Europe/Paris +LAST-MODIFIED:20250523T094234Z +BEGIN:STANDARD +DTSTART:19701025T030000Z +RRULE:BYDAY=-1SU;BYMONTH=10;FREQ=YEARLY +TZNAME:CET +TZOFFSETFROM:+0200 +TZOFFSETTO:+0100 +END:STANDARD +BEGIN:DAYLIGHT +DTSTART:19700329T020000Z +RRULE:BYDAY=-1SU;BYMONTH=3;FREQ=YEARLY +TZNAME:CEST +TZOFFSETFROM:+0100 +TZOFFSETTO:+0200 +END:DAYLIGHT +END:VTIMEZONE +BEGIN:VEVENT +UID:070a3478-4411-4364-844d-26f3542fc364 +DTSTART;TZID=Europe/Paris;VALUE=DATE:20250716 +DTEND;TZID=Europe/Paris;VALUE=DATE:20250717 +DTSTAMP;VALUE=DATE-TIME:20250723T080354Z +SEQUENCE:1 +SUMMARY:Filtered event +END:VEVENT +END:VCALENDAR diff --git a/radicale/tests/static/event_issue1880_1.ics b/radicale/tests/static/event_issue1880_1.ics new file mode 100644 index 00000000..74484364 --- /dev/null +++ b/radicale/tests/static/event_issue1880_1.ics @@ -0,0 +1,29 @@ +BEGIN:VCALENDAR +VERSION:2.0 +PRODID:-//algoo.fr//NONSGML Open Calendar v0.9//EN +BEGIN:VTIMEZONE +TZID:Europe/Paris +LAST-MODIFIED:20250523T094234Z +BEGIN:STANDARD +DTSTART:19701025T030000Z +RRULE:BYDAY=-1SU;BYMONTH=10;FREQ=YEARLY +TZNAME:CET +TZOFFSETFROM:+0200 +TZOFFSETTO:+0100 +END:STANDARD +BEGIN:DAYLIGHT +DTSTART:19700329T020000Z +RRULE:BYDAY=-1SU;BYMONTH=3;FREQ=YEARLY +TZNAME:CEST +TZOFFSETFROM:+0100 +TZOFFSETTO:+0200 +END:DAYLIGHT +END:VTIMEZONE +BEGIN:VEVENT +UID:f5b69821-addc-4010-9ab8-891df1c33c01 +DTSTART;TZID=Europe/Paris;VALUE=DATE-TIME:20250925T093000 +DTEND;TZID=Europe/Paris;VALUE=DATE-TIME:20250925T140000 +DTSTAMP:20250923T114003Z +SUMMARY:event from opencalendar +END:VEVENT +END:VCALENDAR diff --git a/radicale/tests/static/event_issue1880_2.ics b/radicale/tests/static/event_issue1880_2.ics new file mode 100644 index 00000000..791f3d2e --- /dev/null +++ b/radicale/tests/static/event_issue1880_2.ics @@ -0,0 +1,396 @@ +BEGIN:VCALENDAR +VERSION:2.0 +PRODID:-//Mozilla.org/NONSGML Mozilla Calendar V1.1//EN +BEGIN:VTIMEZONE +TZID:Europe/Paris +BEGIN:STANDARD +DTSTART:19110311T000000 +RDATE:19110311T000000 +TZNAME:Europe/Paris(STD) +TZOFFSETFROM:+000921 +TZOFFSETTO:+000000 +END:STANDARD +BEGIN:STANDARD +DTSTART:19161002T000000 +RDATE:19161002T000000 +TZNAME:Europe/Paris(STD) +TZOFFSETFROM:+010000 +TZOFFSETTO:+000000 +END:STANDARD +BEGIN:STANDARD +DTSTART:19171008T000000 +RDATE:19171008T000000 +TZNAME:Europe/Paris(STD) +TZOFFSETFROM:+010000 +TZOFFSETTO:+000000 +END:STANDARD +BEGIN:STANDARD +DTSTART:19181007T000000 +RRULE:FREQ=YEARLY;BYMONTH=10;BYDAY=1MO;UNTIL=19191006T000000 +TZNAME:Europe/Paris(STD) +TZOFFSETFROM:+010000 +TZOFFSETTO:+000000 +END:STANDARD +BEGIN:STANDARD +DTSTART:19201024T000000 +RDATE:19201024T000000 +TZNAME:Europe/Paris(STD) +TZOFFSETFROM:+010000 +TZOFFSETTO:+000000 +END:STANDARD +BEGIN:STANDARD +DTSTART:19211026T000000 +RDATE:19211026T000000 +TZNAME:Europe/Paris(STD) +TZOFFSETFROM:+010000 +TZOFFSETTO:+000000 +END:STANDARD +BEGIN:STANDARD +DTSTART:19221008T000000 +RDATE:19221008T000000 +TZNAME:Europe/Paris(STD) +TZOFFSETFROM:+010000 +TZOFFSETTO:+000000 +END:STANDARD +BEGIN:STANDARD +DTSTART:19231007T000000 +RRULE:FREQ=YEARLY;BYMONTH=10;BYDAY=1SU;UNTIL=19321002T000000 +TZNAME:Europe/Paris(STD) +TZOFFSETFROM:+010000 +TZOFFSETTO:+000000 +END:STANDARD +BEGIN:STANDARD +DTSTART:19331008T000000 +RDATE:19331008T000000 +TZNAME:Europe/Paris(STD) +TZOFFSETFROM:+010000 +TZOFFSETTO:+000000 +END:STANDARD +BEGIN:STANDARD +DTSTART:19341007T000000 +RRULE:FREQ=YEARLY;BYMONTH=10;BYDAY=1SU;UNTIL=19381002T000000 +TZNAME:Europe/Paris(STD) +TZOFFSETFROM:+010000 +TZOFFSETTO:+000000 +END:STANDARD +BEGIN:STANDARD +DTSTART:19391119T000000 +RDATE:19391119T000000 +TZNAME:Europe/Paris(STD) +TZOFFSETFROM:+010000 +TZOFFSETTO:+000000 +END:STANDARD +BEGIN:STANDARD +DTSTART:19421102T030000 +RDATE:19421102T030000 +TZNAME:Europe/Paris(STD) +TZOFFSETFROM:+020000 +TZOFFSETTO:+010000 +END:STANDARD +BEGIN:STANDARD +DTSTART:19431004T030000 +RDATE:19431004T030000 +TZNAME:Europe/Paris(STD) +TZOFFSETFROM:+020000 +TZOFFSETTO:+010000 +END:STANDARD +BEGIN:STANDARD +DTSTART:19450916T030000 +RDATE:19450916T030000 +TZNAME:Europe/Paris(STD) +TZOFFSETFROM:+020000 +TZOFFSETTO:+010000 +END:STANDARD +BEGIN:STANDARD +DTSTART:19760926T010000 +RDATE:19760926T010000 +TZNAME:Europe/Paris(STD) +TZOFFSETFROM:+020000 +TZOFFSETTO:+010000 +END:STANDARD +BEGIN:STANDARD +DTSTART:19770925T030000 +RDATE:19770925T030000 +TZNAME:Europe/Paris(STD) +TZOFFSETFROM:+020000 +TZOFFSETTO:+010000 +END:STANDARD +BEGIN:STANDARD +DTSTART:19781001T030000 +RDATE:19781001T030000 +TZNAME:Europe/Paris(STD) +TZOFFSETFROM:+020000 +TZOFFSETTO:+010000 +END:STANDARD +BEGIN:STANDARD +DTSTART:19790930T030000 +RRULE:FREQ=YEARLY;BYMONTH=9;BYDAY=-1SU;UNTIL=19950924T030000 +TZNAME:Europe/Paris(STD) +TZOFFSETFROM:+020000 +TZOFFSETTO:+010000 +END:STANDARD +BEGIN:STANDARD +DTSTART:19961027T030000 +RDATE:19961027T030000 +TZNAME:Europe/Paris(STD) +TZOFFSETFROM:+020000 +TZOFFSETTO:+010000 +END:STANDARD +BEGIN:STANDARD +DTSTART:19971026T030000 +RRULE:FREQ=YEARLY;BYMONTH=10;BYDAY=-1SU +TZNAME:(STD) +TZOFFSETFROM:+020000 +TZOFFSETTO:+010000 +END:STANDARD +BEGIN:DAYLIGHT +DTSTART:19160614T230000 +RDATE:19160614T230000 +TZNAME:Europe/Paris(DST) +TZOFFSETFROM:+000000 +TZOFFSETTO:+010000 +END:DAYLIGHT +BEGIN:DAYLIGHT +DTSTART:19170324T230000 +RDATE:19170324T230000 +TZNAME:Europe/Paris(DST) +TZOFFSETFROM:+000000 +TZOFFSETTO:+010000 +END:DAYLIGHT +BEGIN:DAYLIGHT +DTSTART:19180309T230000 +RDATE:19180309T230000 +TZNAME:Europe/Paris(DST) +TZOFFSETFROM:+000000 +TZOFFSETTO:+010000 +END:DAYLIGHT +BEGIN:DAYLIGHT +DTSTART:19190301T230000 +RDATE:19190301T230000 +TZNAME:Europe/Paris(DST) +TZOFFSETFROM:+000000 +TZOFFSETTO:+010000 +END:DAYLIGHT +BEGIN:DAYLIGHT +DTSTART:19200214T230000 +RDATE:19200214T230000 +TZNAME:Europe/Paris(DST) +TZOFFSETFROM:+000000 +TZOFFSETTO:+010000 +END:DAYLIGHT +BEGIN:DAYLIGHT +DTSTART:19210314T230000 +RDATE:19210314T230000 +TZNAME:Europe/Paris(DST) +TZOFFSETFROM:+000000 +TZOFFSETTO:+010000 +END:DAYLIGHT +BEGIN:DAYLIGHT +DTSTART:19220325T230000 +RDATE:19220325T230000 +TZNAME:Europe/Paris(DST) +TZOFFSETFROM:+000000 +TZOFFSETTO:+010000 +END:DAYLIGHT +BEGIN:DAYLIGHT +DTSTART:19230526T230000 +RDATE:19230526T230000 +TZNAME:Europe/Paris(DST) +TZOFFSETFROM:+000000 +TZOFFSETTO:+010000 +END:DAYLIGHT +BEGIN:DAYLIGHT +DTSTART:19240329T230000 +RDATE:19240329T230000 +TZNAME:Europe/Paris(DST) +TZOFFSETFROM:+000000 +TZOFFSETTO:+010000 +END:DAYLIGHT +BEGIN:DAYLIGHT +DTSTART:19250404T230000 +RDATE:19250404T230000 +TZNAME:Europe/Paris(DST) +TZOFFSETFROM:+000000 +TZOFFSETTO:+010000 +END:DAYLIGHT +BEGIN:DAYLIGHT +DTSTART:19260417T230000 +RDATE:19260417T230000 +TZNAME:Europe/Paris(DST) +TZOFFSETFROM:+000000 +TZOFFSETTO:+010000 +END:DAYLIGHT +BEGIN:DAYLIGHT +DTSTART:19270409T230000 +RRULE:FREQ=YEARLY;BYMONTH=4;BYDAY=2SA;UNTIL=19280414T230000 +TZNAME:Europe/Paris(DST) +TZOFFSETFROM:+000000 +TZOFFSETTO:+010000 +END:DAYLIGHT +BEGIN:DAYLIGHT +DTSTART:19290420T230000 +RDATE:19290420T230000 +TZNAME:Europe/Paris(DST) +TZOFFSETFROM:+000000 +TZOFFSETTO:+010000 +END:DAYLIGHT +BEGIN:DAYLIGHT +DTSTART:19300412T230000 +RDATE:19300412T230000 +TZNAME:Europe/Paris(DST) +TZOFFSETFROM:+000000 +TZOFFSETTO:+010000 +END:DAYLIGHT +BEGIN:DAYLIGHT +DTSTART:19310418T230000 +RDATE:19310418T230000 +TZNAME:Europe/Paris(DST) +TZOFFSETFROM:+000000 +TZOFFSETTO:+010000 +END:DAYLIGHT +BEGIN:DAYLIGHT +DTSTART:19320402T230000 +RDATE:19320402T230000 +TZNAME:Europe/Paris(DST) +TZOFFSETFROM:+000000 +TZOFFSETTO:+010000 +END:DAYLIGHT +BEGIN:DAYLIGHT +DTSTART:19330325T230000 +RDATE:19330325T230000 +TZNAME:Europe/Paris(DST) +TZOFFSETFROM:+000000 +TZOFFSETTO:+010000 +END:DAYLIGHT +BEGIN:DAYLIGHT +DTSTART:19340407T230000 +RDATE:19340407T230000 +TZNAME:Europe/Paris(DST) +TZOFFSETFROM:+000000 +TZOFFSETTO:+010000 +END:DAYLIGHT +BEGIN:DAYLIGHT +DTSTART:19350330T230000 +RDATE:19350330T230000 +TZNAME:Europe/Paris(DST) +TZOFFSETFROM:+000000 +TZOFFSETTO:+010000 +END:DAYLIGHT +BEGIN:DAYLIGHT +DTSTART:19360418T230000 +RDATE:19360418T230000 +TZNAME:Europe/Paris(DST) +TZOFFSETFROM:+000000 +TZOFFSETTO:+010000 +END:DAYLIGHT +BEGIN:DAYLIGHT +DTSTART:19370403T230000 +RDATE:19370403T230000 +TZNAME:Europe/Paris(DST) +TZOFFSETFROM:+000000 +TZOFFSETTO:+010000 +END:DAYLIGHT +BEGIN:DAYLIGHT +DTSTART:19380326T230000 +RDATE:19380326T230000 +TZNAME:Europe/Paris(DST) +TZOFFSETFROM:+000000 +TZOFFSETTO:+010000 +END:DAYLIGHT +BEGIN:DAYLIGHT +DTSTART:19390415T230000 +RDATE:19390415T230000 +TZNAME:Europe/Paris(DST) +TZOFFSETFROM:+000000 +TZOFFSETTO:+010000 +END:DAYLIGHT +BEGIN:DAYLIGHT +DTSTART:19400225T020000 +RDATE:19400225T020000 +TZNAME:Europe/Paris(DST) +TZOFFSETFROM:+000000 +TZOFFSETTO:+010000 +END:DAYLIGHT +BEGIN:DAYLIGHT +DTSTART:19400614T230000 +RDATE:19400614T230000 +TZNAME:Europe/Paris(DST) +TZOFFSETFROM:+010000 +TZOFFSETTO:+020000 +END:DAYLIGHT +BEGIN:DAYLIGHT +DTSTART:19430329T020000 +RDATE:19430329T020000 +TZNAME:Europe/Paris(DST) +TZOFFSETFROM:+010000 +TZOFFSETTO:+020000 +END:DAYLIGHT +BEGIN:DAYLIGHT +DTSTART:19440403T020000 +RDATE:19440403T020000 +TZNAME:Europe/Paris(DST) +TZOFFSETFROM:+010000 +TZOFFSETTO:+020000 +END:DAYLIGHT +BEGIN:DAYLIGHT +DTSTART:19440825T000000 +RDATE:19440825T000000 +TZNAME:Europe/Paris(DST) +TZOFFSETFROM:+020000 +TZOFFSETTO:+020000 +END:DAYLIGHT +BEGIN:DAYLIGHT +DTSTART:19441008T010000 +RDATE:19441008T010000 +TZNAME:Europe/Paris(DST) +TZOFFSETFROM:+020000 +TZOFFSETTO:+010000 +END:DAYLIGHT +BEGIN:DAYLIGHT +DTSTART:19450402T020000 +RDATE:19450402T020000 +TZNAME:Europe/Paris(DST) +TZOFFSETFROM:+010000 +TZOFFSETTO:+020000 +END:DAYLIGHT +BEGIN:DAYLIGHT +DTSTART:19760328T010000 +RDATE:19760328T010000 +TZNAME:Europe/Paris(DST) +TZOFFSETFROM:+010000 +TZOFFSETTO:+020000 +END:DAYLIGHT +BEGIN:DAYLIGHT +DTSTART:19770403T020000 +RRULE:FREQ=YEARLY;BYMONTH=4;BYDAY=1SU;UNTIL=19800406T020000 +TZNAME:Europe/Paris(DST) +TZOFFSETFROM:+010000 +TZOFFSETTO:+020000 +END:DAYLIGHT +BEGIN:DAYLIGHT +DTSTART:19810329T020000 +RRULE:FREQ=YEARLY;BYMONTH=3;BYDAY=-1SU;UNTIL=19960331T020000 +TZNAME:Europe/Paris(DST) +TZOFFSETFROM:+010000 +TZOFFSETTO:+020000 +END:DAYLIGHT +BEGIN:DAYLIGHT +DTSTART:19970330T020000 +RRULE:FREQ=YEARLY;BYMONTH=3;BYDAY=-1SU +TZNAME:(DST) +TZOFFSETFROM:+010000 +TZOFFSETTO:+020000 +END:DAYLIGHT +X-TZINFO:Europe/Paris[2024a] +END:VTIMEZONE +BEGIN:VEVENT +UID:50c08af4-295c-4bea-9ea4-7402b8e82143 +DTSTART;TZID=Europe/Paris:20250924T133000 +DTEND;TZID=Europe/Paris:20250924T143000 +CREATED:20250923T113902Z +DTSTAMP:20250923T113912Z +LAST-MODIFIED:20250923T113912Z +SUMMARY:event from thunderbird +TRANSP:OPAQUE +END:VEVENT +END:VCALENDAR From ec9ef124ff880a3939c6e935fb068949e3b55652 Mon Sep 17 00:00:00 2001 From: Peter Bieringer Date: Wed, 24 Sep 2025 21:17:37 +0200 Subject: [PATCH 053/290] add new test cases for #1880 and #1812 --- radicale/tests/test_expand.py | 182 ++++++++++++++++++++++++++++++++++ 1 file changed, 182 insertions(+) diff --git a/radicale/tests/test_expand.py b/radicale/tests/test_expand.py index 2cc4a49f..8eab1ced 100644 --- a/radicale/tests/test_expand.py +++ b/radicale/tests/test_expand.py @@ -512,3 +512,185 @@ permissions: RrWw""") status, event2_calendar_data = responses["/test/event2.ics"]["C:calendar-data"] assert event2_calendar_data.text assert "UID:c6be8b2c-3d72-453c-b698-4f25cdf1569e" in event2_calendar_data.text + + def test_report_getetag_expand_filter(self) -> None: + """Test getetag with time-range filter and expand (example from #1880).""" + self.mkcalendar("/test/") + self.put("/test/event_issue1880_1.ics", get_file_content("event_issue1880_1.ics")) + self.put("/test/event_issue1880_2.ics", get_file_content("event_issue1880_2.ics")) + + request = """ + + + + + + + + + + + + + + + """ + status, responses = self.report("/test", request) + assert status == 207 + assert len(responses) == 2 + assert "D:getetag" in responses["/test/event_issue1880_1.ics"] + assert "D:getetag" in responses["/test/event_issue1880_2.ics"] + + def test_report_getetag_expand_filter_positive1(self) -> None: + """Test getetag with time-range filter and expand (not applicable), should return as matching filter range (example from #1812).""" + self.mkcalendar("/test/") + self.put("/test/event_issue1812_getetag.ics", get_file_content("event_issue1812_getetag.ics")) + + request = """ + + + + + + + + + + + + + + + """ + status, responses = self.report("/test", request) + assert status == 207 + assert len(responses) == 1 + assert "D:getetag" in responses["/test/event_issue1812_getetag.ics"] + + def test_report_getetag_expand_filter_positive2(self) -> None: + """Test getetag with time-range filter and expand, should return as matching filter range (example from #1812).""" + self.mkcalendar("/test/") + self.put("/test/event_issue1812.ics", get_file_content("event_issue1812.ics")) + + request = """ + + + + + + + + + + + + + + + """ + status, responses = self.report("/test", request) + assert status == 207 + assert len(responses) == 1 + assert "D:getetag" in responses["/test/event_issue1812.ics"] + + def test_report_getetag_expand_filter_negative1(self) -> None: + """Test getetag with time-range filter and expand, should not return anything (example from #1812).""" + self.mkcalendar("/test/") + self.put("/test/event_issue1812_getetag.ics", get_file_content("event_issue1812_getetag.ics")) + + request = """ + + + + + + + + + + + + + + + """ + status, responses = self.report("/test", request) + assert status == 207 + assert len(responses) == 0 + + def test_report_getetag_expand_filter_negative2(self) -> None: + """Test getetag with time-range filter and expand, should not return anything (example from #1812).""" + self.mkcalendar("/test/") + self.put("/test/event_issue1812_getetag.ics", get_file_content("event_issue1812_getetag.ics")) + + request = """ + + + + + + + + + + + + + + + + """ + status, responses = self.report("/test", request) + assert status == 207 + assert len(responses) == 0 + + + def test_report_getetag_expand_filter_negative3(self) -> None: + """Test getetag with time-range filter and expand, should not return anything (example from #1812).""" + self.mkcalendar("/test/") + self.put("/test/event_issue1812_getetag.ics", get_file_content("event_issue1812_getetag.ics")) + + request = """ + + + + + + + + + + + + + + + """ + status, responses = self.report("/test", request) + assert status == 207 + assert len(responses) == 0 + + def test_report_getetag_expand_filter_negative4(self) -> None: + """Test getetag with time-range filter and expand, nothing returned as filter is not matching (example from #1812).""" + self.mkcalendar("/test/") + self.put("/test/event_issue1812.ics", get_file_content("event_issue1812.ics")) + + request = """ + + + + + + + + + + + + + + + """ + status, responses = self.report("/test", request) + assert status == 207 + assert len(responses) == 0 From 77e7745f9319b6b00a80f1d89c502d59c7ad0f0f Mon Sep 17 00:00:00 2001 From: Peter Bieringer Date: Wed, 24 Sep 2025 21:30:06 +0200 Subject: [PATCH 054/290] make mypy happy --- radicale/tests/test_expand.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/radicale/tests/test_expand.py b/radicale/tests/test_expand.py index 8eab1ced..28d650cf 100644 --- a/radicale/tests/test_expand.py +++ b/radicale/tests/test_expand.py @@ -538,6 +538,8 @@ permissions: RrWw""") status, responses = self.report("/test", request) assert status == 207 assert len(responses) == 2 + assert isinstance(responses["/test/event_issue1880_1.ics"], dict) + assert isinstance(responses["/test/event_issue1880_2.ics"], dict) assert "D:getetag" in responses["/test/event_issue1880_1.ics"] assert "D:getetag" in responses["/test/event_issue1880_2.ics"] @@ -565,6 +567,7 @@ permissions: RrWw""") status, responses = self.report("/test", request) assert status == 207 assert len(responses) == 1 + assert isinstance(responses["/test/event_issue1812_getetag.ics"], dict) assert "D:getetag" in responses["/test/event_issue1812_getetag.ics"] def test_report_getetag_expand_filter_positive2(self) -> None: @@ -591,6 +594,7 @@ permissions: RrWw""") status, responses = self.report("/test", request) assert status == 207 assert len(responses) == 1 + assert isinstance(responses["/test/event_issue1812.ics"], dict) assert "D:getetag" in responses["/test/event_issue1812.ics"] def test_report_getetag_expand_filter_negative1(self) -> None: From 2899c677c1e62a892f8978da4ca9a7614dc694ff Mon Sep 17 00:00:00 2001 From: Peter Bieringer Date: Wed, 24 Sep 2025 21:31:27 +0200 Subject: [PATCH 055/290] revert improper PR#1839, finally fix #1812 and #1880 --- radicale/app/report.py | 42 ++++++++++++++++++++++++------------------ 1 file changed, 24 insertions(+), 18 deletions(-) diff --git a/radicale/app/report.py b/radicale/app/report.py index 555154c3..b63681f7 100644 --- a/radicale/app/report.py +++ b/radicale/app/report.py @@ -156,6 +156,7 @@ def xml_report(base_prefix: str, path: str, xml_request: Optional[ET.Element], Read rfc3253-3.6 for info. """ + logger.debug("TRACE/REPORT/xml_report: base_prefix=%r path=%r", base_prefix, path) multistatus = ET.Element(xmlutils.make_clark("D:multistatus")) if xml_request is None: return client.MULTI_STATUS, multistatus @@ -239,6 +240,7 @@ def xml_report(base_prefix: str, path: str, xml_request: Optional[ET.Element], filter_copy = copy.deepcopy(filter_) if expand is not None: + logger.debug("TRACE/REPORT/xml_report: expand") for comp_filter in filter_copy.findall(".//" + xmlutils.make_clark("C:comp-filter")): if comp_filter.get("name", "").upper() == "VCALENDAR": continue @@ -275,21 +277,15 @@ def xml_report(base_prefix: str, path: str, xml_request: Optional[ET.Element], found_props = [] not_found_props = [] - item_etag: str = "" for prop in props: element = ET.Element(prop.tag) - if prop.tag == xmlutils.make_clark("D:getetag"): - if expand is not None: - item_etag = item.etag - else: - element.text = item.etag - found_props.append(element) - elif prop.tag == xmlutils.make_clark("D:getcontenttype"): + if prop.tag == xmlutils.make_clark("D:getcontenttype"): element.text = xmlutils.get_content_type(item, encoding) found_props.append(element) elif prop.tag in ( xmlutils.make_clark("C:calendar-data"), + xmlutils.make_clark("D:getetag"), xmlutils.make_clark("CR:address-data")): element.text = item.serialize() @@ -326,11 +322,24 @@ def xml_report(base_prefix: str, path: str, xml_request: Optional[ET.Element], continue n_vevents += n_vev - found_props.append(expanded_element) + if prop.tag == xmlutils.make_clark("D:getetag"): + if n_vev > 0: + logger.debug("TRACE/REPORT/xml_report: getetag/expanded element") + element.text = item.etag + found_props.append(element) + else: + logger.debug("TRACE/REPORT/xml_report: getetag/no expanded element") + else: + logger.debug("TRACE/REPORT/xml_report: default") + found_props.append(expanded_element) else: - found_props.append(element) - if hasattr(item.vobject_item, "vevent_list"): - n_vevents += len(item.vobject_item.vevent_list) + if prop.tag == xmlutils.make_clark("D:getetag"): + element.text = item.etag + found_props.append(element) + else: + found_props.append(element) + if hasattr(item.vobject_item, "vevent_list"): + n_vevents += len(item.vobject_item.vevent_list) # Avoid DoS with too many events if max_occurrence and n_vevents > max_occurrence: raise ValueError("REPORT occurrences limit of {} hit" @@ -345,7 +354,7 @@ def xml_report(base_prefix: str, path: str, xml_request: Optional[ET.Element], if found_props or not_found_props: multistatus.append(xml_item_response( base_prefix, uri, found_props=found_props, - not_found_props=not_found_props, found_item=True, item_etag=item_etag)) + not_found_props=not_found_props, found_item=True)) return client.MULTI_STATUS, multistatus @@ -664,7 +673,7 @@ def _find_overridden( def xml_item_response(base_prefix: str, href: str, found_props: Sequence[ET.Element] = (), not_found_props: Sequence[ET.Element] = (), - found_item: bool = True, item_etag: str = "") -> ET.Element: + found_item: bool = True) -> ET.Element: response = ET.Element(xmlutils.make_clark("D:response")) href_element = ET.Element(xmlutils.make_clark("D:href")) @@ -678,10 +687,6 @@ def xml_item_response(base_prefix: str, href: str, status = ET.Element(xmlutils.make_clark("D:status")) status.text = xmlutils.make_response(code) prop_element = ET.Element(xmlutils.make_clark("D:prop")) - if (item_etag != "") and (code == 200): - prop_etag = ET.Element(xmlutils.make_clark("D:getetag")) - prop_etag.text = item_etag - prop_element.append(prop_etag) for prop in props: prop_element.append(prop) propstat.append(prop_element) @@ -735,6 +740,7 @@ def retrieve_items( else: yield item, False if collection_requested: + logger.debug("TRACE/REPORT/retrieve_items: get_filtered") yield from collection.get_filtered(filters) From 7604d447011da01f2c9e94208abc14112e5c6384 Mon Sep 17 00:00:00 2001 From: Peter Bieringer Date: Wed, 24 Sep 2025 21:36:33 +0200 Subject: [PATCH 056/290] make flake8 happy --- radicale/tests/test_expand.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/radicale/tests/test_expand.py b/radicale/tests/test_expand.py index 28d650cf..d70af4b5 100644 --- a/radicale/tests/test_expand.py +++ b/radicale/tests/test_expand.py @@ -523,7 +523,7 @@ permissions: RrWw""") - + @@ -648,7 +648,6 @@ permissions: RrWw""") assert status == 207 assert len(responses) == 0 - def test_report_getetag_expand_filter_negative3(self) -> None: """Test getetag with time-range filter and expand, should not return anything (example from #1812).""" self.mkcalendar("/test/") From 81049df617fc796433d27696f0411f0365c95bfd Mon Sep 17 00:00:00 2001 From: Peter Bieringer Date: Thu, 25 Sep 2025 15:22:51 +0200 Subject: [PATCH 057/290] changelog for https://github.com/Kozea/Radicale/pull/1883 --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index a4ff0bf5..71407451 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,7 @@ ## 3.5.7.dev * Extend: [auth] dovecot: add support for version >= 2.4 +* Fix: report/getetag with enabled expand ## 3.5.6 * Fix: broken start when UID does not exist (potential container startup case) From 63b160c2b03ce5399dc2029181ca2e8174f6175d Mon Sep 17 00:00:00 2001 From: Peter Bieringer Date: Thu, 25 Sep 2025 15:29:04 +0200 Subject: [PATCH 058/290] move evaluation of quirk for Authentik where it belongs, superseeds https://github.com/Kozea/Radicale/pull/1877 --- radicale/auth/ldap.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/radicale/auth/ldap.py b/radicale/auth/ldap.py index 84dcee0b..ababe16a 100644 --- a/radicale/auth/ldap.py +++ b/radicale/auth/ldap.py @@ -89,8 +89,7 @@ class Auth(auth.BaseAuth): self._ldap_ignore_attribute_create_modify_timestamp = configuration.get("auth", "ldap_ignore_attribute_create_modify_timestamp") if self._ldap_ignore_attribute_create_modify_timestamp: - self.ldap3.utils.config._ATTRIBUTES_EXCLUDED_FROM_CHECK.extend(['createTimestamp', 'modifyTimestamp']) - logger.info("auth.ldap_ignore_attribute_create_modify_timestamp applied") + logger.info("auth.ldap_ignore_attribute_create_modify_timestamp will be applied") self._ldap_uri = configuration.get("auth", "ldap_uri") self._ldap_base = configuration.get("auth", "ldap_base") @@ -260,6 +259,8 @@ class Auth(auth.BaseAuth): def _login3(self, login: str, password: str) -> str: """Connect the server""" + if self._ldap_ignore_attribute_create_modify_timestamp: + self.ldap3.utils.config._ATTRIBUTES_EXCLUDED_FROM_CHECK.extend(['createTimestamp', 'modifyTimestamp']) try: logger.debug(f"_login3 {self._ldap_uri}, {self._ldap_reader_dn}") if self._use_encryption: From c316cdd24907bf01f1c02d67b1b8fd2015916601 Mon Sep 17 00:00:00 2001 From: Peter Bieringer Date: Thu, 25 Sep 2025 15:30:31 +0200 Subject: [PATCH 059/290] changelog for move evaluation of quirk for Authentik where it belongs --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 71407451..300ed425 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,6 +3,7 @@ ## 3.5.7.dev * Extend: [auth] dovecot: add support for version >= 2.4 * Fix: report/getetag with enabled expand +* Adjust: use of option [auth] ldap_ignore_attribute_create_modify_timestamp for support of Authentik LDAP server ## 3.5.6 * Fix: broken start when UID does not exist (potential container startup case) From 5f89d18df66f94f2ca839daa6dbe9b9389cce353 Mon Sep 17 00:00:00 2001 From: Peter Marschall Date: Fri, 19 Sep 2025 18:06:50 +0200 Subject: [PATCH 060/290] LDAP auth: move evaluation of quirk for Authentik where it belongs The evaluation of the quirk for the Authentik LDAP server changes the behaviour of Python's `ldap3` module, and that module only. Evaluating the quirk in `__init__` which is used for both, `ldap` and `ldap3` is thus wrong, and may lead to errors when this setting is used together with the `ldap` module. Signed-off-by: Peter Marschall --- radicale/auth/ldap.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/radicale/auth/ldap.py b/radicale/auth/ldap.py index ababe16a..94640f33 100644 --- a/radicale/auth/ldap.py +++ b/radicale/auth/ldap.py @@ -88,9 +88,6 @@ class Auth(auth.BaseAuth): raise RuntimeError("LDAP authentication requires the ldap3 module") from e self._ldap_ignore_attribute_create_modify_timestamp = configuration.get("auth", "ldap_ignore_attribute_create_modify_timestamp") - if self._ldap_ignore_attribute_create_modify_timestamp: - logger.info("auth.ldap_ignore_attribute_create_modify_timestamp will be applied") - self._ldap_uri = configuration.get("auth", "ldap_uri") self._ldap_base = configuration.get("auth", "ldap_base") self._ldap_reader_dn = configuration.get("auth", "ldap_reader_dn") @@ -165,6 +162,8 @@ class Auth(auth.BaseAuth): logger.info("auth.ldap_ssl_ca_file : %r" % self._ldap_ssl_ca_file) else: logger.info("auth.ldap_ssl_ca_file : (not provided)") + if self._ldap_ignore_attribute_create_modify_timestamp: + logger.info("auth.ldap_ignore_attribute_create_modify_timestamp applied (relevant for ldap3 only)") """Extend attributes to to be returned in the user query""" if self._ldap_groups_attr: self._ldap_attributes.append(self._ldap_groups_attr) @@ -258,9 +257,10 @@ class Auth(auth.BaseAuth): return "" def _login3(self, login: str, password: str) -> str: - """Connect the server""" if self._ldap_ignore_attribute_create_modify_timestamp: self.ldap3.utils.config._ATTRIBUTES_EXCLUDED_FROM_CHECK.extend(['createTimestamp', 'modifyTimestamp']) + + """Connect the server""" try: logger.debug(f"_login3 {self._ldap_uri}, {self._ldap_reader_dn}") if self._use_encryption: From d83f9fe29d592989350b20aebc81ef1c3ba9a4fa Mon Sep 17 00:00:00 2001 From: Peter Bieringer Date: Fri, 26 Sep 2025 07:53:28 +0200 Subject: [PATCH 061/290] extend copyright related to https://github.com/Kozea/Radicale/pull/1883 --- radicale/tests/test_expand.py | 1 + 1 file changed, 1 insertion(+) diff --git a/radicale/tests/test_expand.py b/radicale/tests/test_expand.py index d70af4b5..9783abeb 100644 --- a/radicale/tests/test_expand.py +++ b/radicale/tests/test_expand.py @@ -3,6 +3,7 @@ # Copyright © 2017-2019 Unrud # Copyright © 2024 Pieter Hijma # Copyright © 2025 David Greaves +# Copyright © 2025 Peter Bieringer # # This library is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by From c36fa29b14e101c54c490ce7586d8e9570ea4b8b Mon Sep 17 00:00:00 2001 From: Peter Bieringer Date: Fri, 26 Sep 2025 08:05:19 +0200 Subject: [PATCH 062/290] skip in case of coveralls --finish has an error --- .github/workflows/test.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 34774410..384f9e1e 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -34,6 +34,7 @@ jobs: needs: test if: github.event_name == 'push' runs-on: ubuntu-latest + continue-on-error: true steps: - uses: actions/setup-python@v5 with: From 29530ade4efd62cf2edc78fc51b802c694ddaf89 Mon Sep 17 00:00:00 2001 From: Peter Bieringer Date: Fri, 26 Sep 2025 15:21:21 +0200 Subject: [PATCH 063/290] carveout dedicated coveralls job --- .github/workflows/test.yml | 19 ++++++++++++++++++- 1 file changed, 18 insertions(+), 1 deletion(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 384f9e1e..4625ac76 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -11,6 +11,23 @@ jobs: - os: windows-latest python-version: pypy-3.9 runs-on: ${{ matrix.os }} + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: ${{ matrix.python-version }} + - name: Install Test dependencies + run: pip install tox + - name: Test + run: tox -c pyproject.toml -e py + + coveralls-test: + strategy: + matrix: + os: [ubuntu-latest] + python-version: ['3.13.0'] + runs-on: ${{ matrix.os }} + continue-on-error: true steps: - uses: actions/checkout@v4 - uses: actions/setup-python@v5 @@ -31,7 +48,7 @@ jobs: run: coveralls --service=github coveralls-finish: - needs: test + needs: coveralls-test if: github.event_name == 'push' runs-on: ubuntu-latest continue-on-error: true From be848d1937317259eaf5ee6544b7194e9d6e9839 Mon Sep 17 00:00:00 2001 From: Peter Bieringer Date: Fri, 26 Sep 2025 15:27:03 +0200 Subject: [PATCH 064/290] update python version for lint job --- .github/workflows/test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 4625ac76..0df546a9 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -69,7 +69,7 @@ jobs: - uses: actions/checkout@v4 - uses: actions/setup-python@v5 with: - python-version: '3.12' + python-version: '3.13.0' - name: Install tox run: pip install tox - name: Lint From c84b94c24550ea0b4193d5de8cffe70def3fd789 Mon Sep 17 00:00:00 2001 From: Peter Bieringer Date: Sat, 27 Sep 2025 07:41:08 +0200 Subject: [PATCH 065/290] do not continue on coveralls-test problem --- .github/workflows/test.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 0df546a9..ca137590 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -27,7 +27,6 @@ jobs: os: [ubuntu-latest] python-version: ['3.13.0'] runs-on: ${{ matrix.os }} - continue-on-error: true steps: - uses: actions/checkout@v4 - uses: actions/setup-python@v5 From accd65d94a943e4effbc26f1f76c22e0e3380a4c Mon Sep 17 00:00:00 2001 From: Peter Bieringer Date: Sat, 27 Sep 2025 07:46:46 +0200 Subject: [PATCH 066/290] adjust python versions --- .github/workflows/test.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index ca137590..3c7a1819 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -6,10 +6,10 @@ jobs: strategy: matrix: os: [ubuntu-latest, macos-latest, windows-latest] - python-version: ['3.9', '3.10', '3.11', '3.12.3', '3.13.0', pypy-3.9] + python-version: ['3.9', '3.10', '3.11', '3.12', '3.13', 'pypy-3.9', 'pypy-3.10', 'pypy-3.11'] exclude: - os: windows-latest - python-version: pypy-3.9 + python-version: pypy-3.11 runs-on: ${{ matrix.os }} steps: - uses: actions/checkout@v4 From c852070afe91656f80e19d1de34ba5ce5e56e280 Mon Sep 17 00:00:00 2001 From: Peter Bieringer Date: Sat, 27 Sep 2025 07:52:26 +0200 Subject: [PATCH 067/290] exclude all pypy on Windows, fix coveralls python version --- .github/workflows/test.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 3c7a1819..6ef03d8f 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -9,7 +9,7 @@ jobs: python-version: ['3.9', '3.10', '3.11', '3.12', '3.13', 'pypy-3.9', 'pypy-3.10', 'pypy-3.11'] exclude: - os: windows-latest - python-version: pypy-3.11 + python-version: [ 'pypy-3.9', 'pypy-3.10', 'pypy-3.11' ] runs-on: ${{ matrix.os }} steps: - uses: actions/checkout@v4 @@ -25,7 +25,7 @@ jobs: strategy: matrix: os: [ubuntu-latest] - python-version: ['3.13.0'] + python-version: ['3.13'] runs-on: ${{ matrix.os }} steps: - uses: actions/checkout@v4 From 61596ee5d7d05fba26352926212f3e12c16edff3 Mon Sep 17 00:00:00 2001 From: Peter Bieringer Date: Sat, 27 Sep 2025 07:58:35 +0200 Subject: [PATCH 068/290] fix exclude for windows-latest --- .github/workflows/test.yml | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 6ef03d8f..3a183f9b 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -9,7 +9,11 @@ jobs: python-version: ['3.9', '3.10', '3.11', '3.12', '3.13', 'pypy-3.9', 'pypy-3.10', 'pypy-3.11'] exclude: - os: windows-latest - python-version: [ 'pypy-3.9', 'pypy-3.10', 'pypy-3.11' ] + python-version: 'pypy-3.9' + - os: windows-latest + python-version: 'pypy-3.10' + - os: windows-latest + python-version: 'pypy-3.11' runs-on: ${{ matrix.os }} steps: - uses: actions/checkout@v4 From 1c323197dea696b86d94bf9b657680beebb462f2 Mon Sep 17 00:00:00 2001 From: Peter Bieringer Date: Sat, 27 Sep 2025 08:16:00 +0200 Subject: [PATCH 069/290] update version --- CHANGELOG.md | 2 +- pyproject.toml | 2 +- setup.py.legacy | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 300ed425..319ddede 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,6 @@ # Changelog -## 3.5.7.dev +## 3.5.7 * Extend: [auth] dovecot: add support for version >= 2.4 * Fix: report/getetag with enabled expand * Adjust: use of option [auth] ldap_ignore_attribute_create_modify_timestamp for support of Authentik LDAP server diff --git a/pyproject.toml b/pyproject.toml index 6816745a..ae060858 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -3,7 +3,7 @@ name = "Radicale" # When the version is updated, a new section in the CHANGELOG.md file must be # added too. readme = "README.md" -version = "3.5.7.dev" +version = "3.5.7" authors = [{name = "Guillaume Ayoub", email = "guillaume.ayoub@kozea.fr"}, {name = "Unrud", email = "unrud@outlook.com"}, {name = "Peter Bieringer", email = "pb@bieringer.de"}] license = {text = "GNU GPL v3"} description = "CalDAV and CardDAV Server" diff --git a/setup.py.legacy b/setup.py.legacy index f8af3ead..18626b5d 100644 --- a/setup.py.legacy +++ b/setup.py.legacy @@ -20,7 +20,7 @@ from setuptools import find_packages, setup # When the version is updated, a new section in the CHANGELOG.md file must be # added too. -VERSION = "3.5.7.dev" +VERSION = "3.5.7" with open("README.md", encoding="utf-8") as f: long_description = f.read() From 0648f417b1189eab30cfeb84ccbca65b35524f2e Mon Sep 17 00:00:00 2001 From: Peter Bieringer Date: Sun, 28 Sep 2025 20:25:17 +0200 Subject: [PATCH 070/290] 3.5.8.dev --- CHANGELOG.md | 2 ++ pyproject.toml | 2 +- setup.py.legacy | 2 +- 3 files changed, 4 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 319ddede..20e96ab4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,7 @@ # Changelog +## 3.5.8.dev + ## 3.5.7 * Extend: [auth] dovecot: add support for version >= 2.4 * Fix: report/getetag with enabled expand diff --git a/pyproject.toml b/pyproject.toml index ae060858..505b295a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -3,7 +3,7 @@ name = "Radicale" # When the version is updated, a new section in the CHANGELOG.md file must be # added too. readme = "README.md" -version = "3.5.7" +version = "3.5.8.dev" authors = [{name = "Guillaume Ayoub", email = "guillaume.ayoub@kozea.fr"}, {name = "Unrud", email = "unrud@outlook.com"}, {name = "Peter Bieringer", email = "pb@bieringer.de"}] license = {text = "GNU GPL v3"} description = "CalDAV and CardDAV Server" diff --git a/setup.py.legacy b/setup.py.legacy index 18626b5d..520be1bd 100644 --- a/setup.py.legacy +++ b/setup.py.legacy @@ -20,7 +20,7 @@ from setuptools import find_packages, setup # When the version is updated, a new section in the CHANGELOG.md file must be # added too. -VERSION = "3.5.7" +VERSION = "3.5.8.dev" with open("README.md", encoding="utf-8") as f: long_description = f.read() From caab7d371237cd50cb857af0031327292586221a Mon Sep 17 00:00:00 2001 From: Peter Marschall Date: Sun, 7 Sep 2025 19:03:56 +0200 Subject: [PATCH 071/290] LDAP auth: load SSL/TLS config unconditionally Currently it is not used by _login2(), but it does not hurt to have it available. It is a preparation for supporting encrypted connections in _login2(). --- radicale/auth/ldap.py | 28 +++++++++++++--------------- 1 file changed, 13 insertions(+), 15 deletions(-) diff --git a/radicale/auth/ldap.py b/radicale/auth/ldap.py index 94640f33..0974b024 100644 --- a/radicale/auth/ldap.py +++ b/radicale/auth/ldap.py @@ -102,21 +102,19 @@ class Auth(auth.BaseAuth): if ldap_secret_file_path: with open(ldap_secret_file_path, 'r') as file: self._ldap_secret = file.read().rstrip('\n') - if self._ldap_module_version == 3: - self._ldap_use_ssl = configuration.get("auth", "ldap_use_ssl") - self._ldap_security = configuration.get("auth", "ldap_security") - self._use_encryption = self._ldap_use_ssl or self._ldap_security in ("tls", "starttls") - if self._ldap_use_ssl and self._ldap_security == "starttls": - raise RuntimeError("Cannot set both 'ldap_use_ssl = True' and 'ldap_security' = 'starttls'") - if self._ldap_use_ssl: - logger.warning("Configuration uses soon to be deprecated 'ldap_use_ssl', use 'ldap_security' ('none', 'tls', 'starttls') instead.") - if self._use_encryption: - self._ldap_ssl_ca_file = configuration.get("auth", "ldap_ssl_ca_file") - tmp = configuration.get("auth", "ldap_ssl_verify_mode") - if tmp == "NONE": - self._ldap_ssl_verify_mode = ssl.CERT_NONE - elif tmp == "OPTIONAL": - self._ldap_ssl_verify_mode = ssl.CERT_OPTIONAL + self._ldap_use_ssl = configuration.get("auth", "ldap_use_ssl") + self._ldap_security = configuration.get("auth", "ldap_security") + self._use_encryption = self._ldap_use_ssl or self._ldap_security in ("tls", "starttls") + if self._ldap_use_ssl and self._ldap_security == "starttls": + raise RuntimeError("Cannot set both 'ldap_use_ssl = True' and 'ldap_security' = 'starttls'") + if self._ldap_use_ssl: + logger.warning("Configuration uses soon to be deprecated 'ldap_use_ssl', use 'ldap_security' ('none', 'tls', 'starttls') instead.") + self._ldap_ssl_ca_file = configuration.get("auth", "ldap_ssl_ca_file") + tmp = configuration.get("auth", "ldap_ssl_verify_mode") + if tmp == "NONE": + self._ldap_ssl_verify_mode = ssl.CERT_NONE + elif tmp == "OPTIONAL": + self._ldap_ssl_verify_mode = ssl.CERT_OPTIONAL logger.info("auth.ldap_uri : %r" % self._ldap_uri) logger.info("auth.ldap_base : %r" % self._ldap_base) From 7eb0c665127580aab6ab1ee1225f86fd59eaf765 Mon Sep 17 00:00:00 2001 From: Peter Marschall Date: Sun, 14 Sep 2025 09:50:46 +0200 Subject: [PATCH 072/290] LDAP auth: refactor dealing with 'ldap_use_ssl' * stop treating it as class property * refactor to consolidate logic into one big 'if' statement (for easier removal when the config option gets removed in the future) * make deprecation warning for 'ldap_use_ssl' more urgent * raise error if conflicting settings 'ldap_security' = "starttls" and 'ldap_use_ssl' = True are set together * if not set, infer 'ldap_security' = "tls" from 'ldap_use_ssl' = True, logging a warning for the admin to update the config --- radicale/auth/ldap.py | 20 +++++++++++--------- 1 file changed, 11 insertions(+), 9 deletions(-) diff --git a/radicale/auth/ldap.py b/radicale/auth/ldap.py index 0974b024..249c3b1a 100644 --- a/radicale/auth/ldap.py +++ b/radicale/auth/ldap.py @@ -68,7 +68,6 @@ class Auth(auth.BaseAuth): _ldap_group_members_attr: str _ldap_module_version: int = 3 _use_encryption: bool = False - _ldap_use_ssl: bool = False _ldap_security: str = "none" _ldap_ssl_verify_mode: int = ssl.CERT_REQUIRED _ldap_ssl_ca_file: str = "" @@ -102,13 +101,16 @@ class Auth(auth.BaseAuth): if ldap_secret_file_path: with open(ldap_secret_file_path, 'r') as file: self._ldap_secret = file.read().rstrip('\n') - self._ldap_use_ssl = configuration.get("auth", "ldap_use_ssl") self._ldap_security = configuration.get("auth", "ldap_security") - self._use_encryption = self._ldap_use_ssl or self._ldap_security in ("tls", "starttls") - if self._ldap_use_ssl and self._ldap_security == "starttls": - raise RuntimeError("Cannot set both 'ldap_use_ssl = True' and 'ldap_security' = 'starttls'") - if self._ldap_use_ssl: - logger.warning("Configuration uses soon to be deprecated 'ldap_use_ssl', use 'ldap_security' ('none', 'tls', 'starttls') instead.") + ldap_use_ssl = configuration.get("auth", "ldap_use_ssl") + self._use_encryption = ldap_use_ssl or self._ldap_security in ("tls", "starttls") + if ldap_use_ssl: + logger.warning("Configuration uses deprecated 'ldap_use_ssl': use 'ldap_security' ('none', 'tls', 'starttls') instead.") + if self._ldap_security == "starttls": + raise RuntimeError("Deprecated config setting 'ldap_use_ssl = True' conflicts with 'ldap_security' = 'starttls'") + elif self._ldap_security != "tls": + logger.warning("Update configuration: set 'ldap_security = tls' instead of deprecated 'ldap_use_ssl = True'") + self._ldap_security = "tls" self._ldap_ssl_ca_file = configuration.get("auth", "ldap_ssl_ca_file") tmp = configuration.get("auth", "ldap_ssl_verify_mode") if tmp == "NONE": @@ -152,7 +154,7 @@ class Auth(auth.BaseAuth): if self._ldap_reader_dn and not self._ldap_secret: logger.error("auth.ldap_secret : (not provided)") raise RuntimeError("LDAP authentication requires ldap_secret for ldap_reader_dn") - logger.info("auth.ldap_use_ssl : %s" % self._ldap_use_ssl) + logger.info("auth.ldap_use_ssl : %s" % ldap_use_ssl) logger.info("auth.ldap_security : %s" % self._ldap_security) if self._use_encryption: logger.info("auth.ldap_ssl_verify_mode : %s" % self._ldap_ssl_verify_mode) @@ -269,7 +271,7 @@ class Auth(auth.BaseAuth): validate=self._ldap_ssl_verify_mode, ca_certs_file=self._ldap_ssl_ca_file ) - if self._ldap_use_ssl or self._ldap_security == "tls": + if self._ldap_security == "tls": logger.debug("_login3 using ssl (reader)") server = self.ldap3.Server(self._ldap_uri, use_ssl=True, tls=tls) else: From c58eef4bacc7f457782a174178d55b03781477bc Mon Sep 17 00:00:00 2001 From: Peter Marschall Date: Sun, 14 Sep 2025 10:04:22 +0200 Subject: [PATCH 073/290] LDAP auth: infer 'ldap_security = tls' from the URL prefix: ldaps:// => LDAPS LDAP URIs starting with the scheme 'ldaps' are - by definition - meant to use LDAPS instead of plain LDAP: infer 'ldap_security' = "tls" if it is not set. --- radicale/auth/ldap.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/radicale/auth/ldap.py b/radicale/auth/ldap.py index 249c3b1a..9df25b83 100644 --- a/radicale/auth/ldap.py +++ b/radicale/auth/ldap.py @@ -118,6 +118,10 @@ class Auth(auth.BaseAuth): elif tmp == "OPTIONAL": self._ldap_ssl_verify_mode = ssl.CERT_OPTIONAL + if self._ldap_uri.lower().startswith("ldaps://") and self._ldap_security not in ("tls", "starttls"): + logger.info("Inferring 'ldap_security' = tls from 'ldap_uri' starting with 'ldaps://'") + self._ldap_security = "tls" + logger.info("auth.ldap_uri : %r" % self._ldap_uri) logger.info("auth.ldap_base : %r" % self._ldap_base) logger.info("auth.ldap_reader_dn : %r" % self._ldap_reader_dn) From 73b77defe455207162968f567fac9c112516076b Mon Sep 17 00:00:00 2001 From: Peter Marschall Date: Sun, 14 Sep 2025 10:27:26 +0200 Subject: [PATCH 074/290] LDAP auth: warn on unset ldap_ssl_ca_file when certificate verification is wanted --- radicale/auth/ldap.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/radicale/auth/ldap.py b/radicale/auth/ldap.py index 9df25b83..0783cfcf 100644 --- a/radicale/auth/ldap.py +++ b/radicale/auth/ldap.py @@ -122,6 +122,9 @@ class Auth(auth.BaseAuth): logger.info("Inferring 'ldap_security' = tls from 'ldap_uri' starting with 'ldaps://'") self._ldap_security = "tls" + if self._ldap_ssl_ca_file == "" and self._ldap_ssl_verify_mode != ssl.CERT_NONE and self._ldap_security in ("tls", "starttls"): + logger.warning("Certificate verification not possible: 'ldap_ssl_ca_file' not set") + logger.info("auth.ldap_uri : %r" % self._ldap_uri) logger.info("auth.ldap_base : %r" % self._ldap_base) logger.info("auth.ldap_reader_dn : %r" % self._ldap_reader_dn) From b21549b998565b661aa474534475ad85daadda83 Mon Sep 17 00:00:00 2001 From: Peter Marschall Date: Sun, 14 Sep 2025 11:41:10 +0200 Subject: [PATCH 075/290] LDAP auth: warn if 'ldap_ssl_ca_file' is set without LDAP encryption --- radicale/auth/ldap.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/radicale/auth/ldap.py b/radicale/auth/ldap.py index 0783cfcf..bd9e851c 100644 --- a/radicale/auth/ldap.py +++ b/radicale/auth/ldap.py @@ -124,6 +124,8 @@ class Auth(auth.BaseAuth): if self._ldap_ssl_ca_file == "" and self._ldap_ssl_verify_mode != ssl.CERT_NONE and self._ldap_security in ("tls", "starttls"): logger.warning("Certificate verification not possible: 'ldap_ssl_ca_file' not set") + if self._ldap_ssl_ca_file and self._ldap_security not in ("tls", "starttls"): + logger.warning("Config setting 'ldap_ssl_ca_file' useless without encrypted LDAP connection") logger.info("auth.ldap_uri : %r" % self._ldap_uri) logger.info("auth.ldap_base : %r" % self._ldap_base) From f8b15eb122b1ab3ff4d06e37f95db8b6dd84739c Mon Sep 17 00:00:00 2001 From: Peter Marschall Date: Sun, 14 Sep 2025 12:22:18 +0200 Subject: [PATCH 076/290] LDAP auth: get rid of helper property '_use_encryption' Inferring 'ldap_security' in earlier commits, allows us to get rid of the helper property '_use_encryption', streamlining the code. --- radicale/auth/ldap.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/radicale/auth/ldap.py b/radicale/auth/ldap.py index bd9e851c..8c9d5b69 100644 --- a/radicale/auth/ldap.py +++ b/radicale/auth/ldap.py @@ -67,7 +67,6 @@ class Auth(auth.BaseAuth): _ldap_group_filter: str _ldap_group_members_attr: str _ldap_module_version: int = 3 - _use_encryption: bool = False _ldap_security: str = "none" _ldap_ssl_verify_mode: int = ssl.CERT_REQUIRED _ldap_ssl_ca_file: str = "" @@ -103,7 +102,6 @@ class Auth(auth.BaseAuth): self._ldap_secret = file.read().rstrip('\n') self._ldap_security = configuration.get("auth", "ldap_security") ldap_use_ssl = configuration.get("auth", "ldap_use_ssl") - self._use_encryption = ldap_use_ssl or self._ldap_security in ("tls", "starttls") if ldap_use_ssl: logger.warning("Configuration uses deprecated 'ldap_use_ssl': use 'ldap_security' ('none', 'tls', 'starttls') instead.") if self._ldap_security == "starttls": @@ -165,7 +163,7 @@ class Auth(auth.BaseAuth): raise RuntimeError("LDAP authentication requires ldap_secret for ldap_reader_dn") logger.info("auth.ldap_use_ssl : %s" % ldap_use_ssl) logger.info("auth.ldap_security : %s" % self._ldap_security) - if self._use_encryption: + if self._ldap_security in ("tls", "starttls"): logger.info("auth.ldap_ssl_verify_mode : %s" % self._ldap_ssl_verify_mode) if self._ldap_ssl_ca_file: logger.info("auth.ldap_ssl_ca_file : %r" % self._ldap_ssl_ca_file) @@ -272,7 +270,7 @@ class Auth(auth.BaseAuth): """Connect the server""" try: logger.debug(f"_login3 {self._ldap_uri}, {self._ldap_reader_dn}") - if self._use_encryption: + if self._ldap_security in ("tls", "starttls"): logger.debug("_login3 using encryption (reader)") tls = self.ldap3.Tls(validate=self._ldap_ssl_verify_mode) if self._ldap_ssl_ca_file != "": From 2d7a9b001c1512fee5d0e023cf7802d3c7ee0490 Mon Sep 17 00:00:00 2001 From: Peter Marschall Date: Sun, 14 Sep 2025 13:57:36 +0200 Subject: [PATCH 077/290] LDAP auth: support TLS & start_tls also with python-ldap Until now, every connection to the LDAP server was silently unencryptedr when using Python's ldap module instead of the ldap3 module. I.e. using Python's ldap module was inherently insecure, as there was not even a hint that the config settings for encryption were ignored. This commit changes this and brings LDAP authentication based on the ldap module feature-wise on par with the one based on the ldap3 module. --- radicale/auth/ldap.py | 42 +++++++++++++++++++++++++++++++++++++++--- 1 file changed, 39 insertions(+), 3 deletions(-) diff --git a/radicale/auth/ldap.py b/radicale/auth/ldap.py index 8c9d5b69..9d41e5aa 100644 --- a/radicale/auth/ldap.py +++ b/radicale/auth/ldap.py @@ -83,7 +83,7 @@ class Auth(auth.BaseAuth): self._ldap_module_version = 2 self.ldap = ldap except ImportError as e: - raise RuntimeError("LDAP authentication requires the ldap3 module") from e + raise RuntimeError("LDAP authentication requires the ldap3 or ldap module") from e self._ldap_ignore_attribute_create_modify_timestamp = configuration.get("auth", "ldap_ignore_attribute_create_modify_timestamp") self._ldap_uri = configuration.get("auth", "ldap_uri") @@ -183,8 +183,26 @@ class Auth(auth.BaseAuth): """Bind as reader dn""" logger.debug(f"_login2 {self._ldap_uri}, {self._ldap_reader_dn}") conn = self.ldap.initialize(self._ldap_uri) - conn.protocol_version = 3 + conn.protocol_version = self.ldap.VERSION3 conn.set_option(self.ldap.OPT_REFERRALS, 0) + + if self._ldap_security in ("tls", "starttls"): + """certificate validation mode""" + if self._ldap_ssl_verify_mode == ssl.CERT_REQUIRED: + conn.set_option(self.ldap.OPT_X_TLS_REQUIRE_CERT, self.ldap.OPT_X_TLS_DEMAND) + elif self._ldap_ssl_verify_mode == ssl.CERT_OPTIONAL: + conn.set_option(self.ldap.OPT_X_TLS_REQUIRE_CERT, self.ldap.OPT_X_TLS_ALLOW) + else: + conn.set_option(self.ldap.OPT_X_TLS_REQUIRE_CERT, self.ldap.OPT_X_TLS_NONE) + """CA file to validate certificate against""" + if self._ldap_ssl_ca_file: + conn.set_option(self.ldap.OPT_X_TLS_CACERTFILE, self._ldap_ssl_ca_file) + """create TLS context- this must be the last TLS setting""" + conn.set_option(self.ldap.OPT_X_TLS_NEWCTX, self.ldap.OPT_ON) + + if self._ldap_security == "starttls": + conn.start_tls_s() + conn.simple_bind_s(self._ldap_reader_dn, self._ldap_secret) """Search for the dn of user to authenticate""" escaped_login = self.ldap.filter.escape_filter_chars(login) @@ -234,8 +252,26 @@ class Auth(auth.BaseAuth): try: """Bind as user to authenticate""" conn = self.ldap.initialize(self._ldap_uri) - conn.protocol_version = 3 + conn.protocol_version = self.ldap.VERSION3 conn.set_option(self.ldap.OPT_REFERRALS, 0) + + if self._ldap_security in ("tls", "starttls"): + """certificate validation mode""" + if self._ldap_ssl_verify_mode == ssl.CERT_REQUIRED: + conn.set_option(self.ldap.OPT_X_TLS_REQUIRE_CERT, self.ldap.OPT_X_TLS_DEMAND) + elif self._ldap_ssl_verify_mode == ssl.CERT_OPTIONAL: + conn.set_option(self.ldap.OPT_X_TLS_REQUIRE_CERT, self.ldap.OPT_X_TLS_ALLOW) + else: + conn.set_option(self.ldap.OPT_X_TLS_REQUIRE_CERT, self.ldap.OPT_X_TLS_NONE) + """CA file to validate certificate against""" + if self._ldap_ssl_ca_file: + conn.set_option(self.ldap.OPT_X_TLS_CACERTFILE, self._ldap_ssl_ca_file) + """create TLS context- this must be the last TLS setting""" + conn.set_option(self.ldap.OPT_X_TLS_NEWCTX, self.ldap.OPT_ON) + + if self._ldap_security == "starttls": + conn.start_tls_s() + conn.simple_bind_s(user_dn, password) if self._ldap_user_attr: if user_entry[1][self._ldap_user_attr]: From 44c64d70f51c3158b866fc33c7340c860967551f Mon Sep 17 00:00:00 2001 From: Peter Marschall Date: Sat, 27 Sep 2025 20:31:57 +0200 Subject: [PATCH 078/290] LDAP auth: _login2: re-bind as user within same connection Python's ldap module, which is modelled along OpenLDAP's API, allows us to keep the connection and doing a new bind as a different user, superseding the previous bind. Use this to simplify the code and avoid duplication. --- radicale/auth/ldap.py | 23 ----------------------- 1 file changed, 23 deletions(-) diff --git a/radicale/auth/ldap.py b/radicale/auth/ldap.py index 9d41e5aa..f9041993 100644 --- a/radicale/auth/ldap.py +++ b/radicale/auth/ldap.py @@ -244,34 +244,11 @@ class Auth(auth.BaseAuth): for dn, entry in res: groupDNs.append(dn) - """Close LDAP connection""" - conn.unbind() except Exception as e: raise RuntimeError(f"Invalid LDAP configuration:{e}") try: """Bind as user to authenticate""" - conn = self.ldap.initialize(self._ldap_uri) - conn.protocol_version = self.ldap.VERSION3 - conn.set_option(self.ldap.OPT_REFERRALS, 0) - - if self._ldap_security in ("tls", "starttls"): - """certificate validation mode""" - if self._ldap_ssl_verify_mode == ssl.CERT_REQUIRED: - conn.set_option(self.ldap.OPT_X_TLS_REQUIRE_CERT, self.ldap.OPT_X_TLS_DEMAND) - elif self._ldap_ssl_verify_mode == ssl.CERT_OPTIONAL: - conn.set_option(self.ldap.OPT_X_TLS_REQUIRE_CERT, self.ldap.OPT_X_TLS_ALLOW) - else: - conn.set_option(self.ldap.OPT_X_TLS_REQUIRE_CERT, self.ldap.OPT_X_TLS_NONE) - """CA file to validate certificate against""" - if self._ldap_ssl_ca_file: - conn.set_option(self.ldap.OPT_X_TLS_CACERTFILE, self._ldap_ssl_ca_file) - """create TLS context- this must be the last TLS setting""" - conn.set_option(self.ldap.OPT_X_TLS_NEWCTX, self.ldap.OPT_ON) - - if self._ldap_security == "starttls": - conn.start_tls_s() - conn.simple_bind_s(user_dn, password) if self._ldap_user_attr: if user_entry[1][self._ldap_user_attr]: From b6ee3b6991e1d4365a081edbcf3f756ad455ab07 Mon Sep 17 00:00:00 2001 From: Peter Marschall Date: Sun, 28 Sep 2025 10:24:20 +0200 Subject: [PATCH 079/290] LDAP auth: align values when logging config options In addition, log 'ldap_ssl_verify_mode' and 'ldap_ssl_ca_file' unconditionally. --- radicale/auth/ldap.py | 51 +++++++++++++++++++++---------------------- 1 file changed, 25 insertions(+), 26 deletions(-) diff --git a/radicale/auth/ldap.py b/radicale/auth/ldap.py index f9041993..65eb2c02 100644 --- a/radicale/auth/ldap.py +++ b/radicale/auth/ldap.py @@ -125,50 +125,49 @@ class Auth(auth.BaseAuth): if self._ldap_ssl_ca_file and self._ldap_security not in ("tls", "starttls"): logger.warning("Config setting 'ldap_ssl_ca_file' useless without encrypted LDAP connection") - logger.info("auth.ldap_uri : %r" % self._ldap_uri) - logger.info("auth.ldap_base : %r" % self._ldap_base) - logger.info("auth.ldap_reader_dn : %r" % self._ldap_reader_dn) - logger.info("auth.ldap_filter : %r" % self._ldap_filter) + logger.info("auth.ldap_uri : %r" % self._ldap_uri) + logger.info("auth.ldap_base : %r" % self._ldap_base) + logger.info("auth.ldap_reader_dn : %r" % self._ldap_reader_dn) + logger.info("auth.ldap_filter : %r" % self._ldap_filter) if self._ldap_user_attr: - logger.info("auth.ldap_user_attribute : %r" % self._ldap_user_attr) + logger.info("auth.ldap_user_attribute : %r" % self._ldap_user_attr) else: - logger.info("auth.ldap_user_attribute : (not provided)") + logger.info("auth.ldap_user_attribute : (not provided)") if self._ldap_groups_attr: - logger.info("auth.ldap_groups_attribute: %r" % self._ldap_groups_attr) + logger.info("auth.ldap_groups_attribute : %r" % self._ldap_groups_attr) else: - logger.info("auth.ldap_groups_attribute: (not provided)") + logger.info("auth.ldap_groups_attribute : (not provided)") if self._ldap_group_base: - logger.info("auth.ldap_group_base : %r" % self._ldap_group_base) + logger.info("auth.ldap_group_base : %r" % self._ldap_group_base) else: - logger.info("auth.ldap_group_base : (not provided, using ldap_base)") + logger.info("auth.ldap_group_base : (not provided, using ldap_base)") self._ldap_group_base = self._ldap_base if self._ldap_group_filter: - logger.info("auth.ldap_group_filter: %r" % self._ldap_group_filter) + logger.info("auth.ldap_group_filter : %r" % self._ldap_group_filter) else: - logger.info("auth.ldap_group_filter: (not provided)") + logger.info("auth.ldap_group_filter : (not provided)") if self._ldap_group_members_attr: logger.info("auth.ldap_group_members_attr: %r" % self._ldap_group_members_attr) else: logger.info("auth.ldap_group_members_attr: (not provided)") if ldap_secret_file_path: - logger.info("auth.ldap_secret_file_path: %r" % ldap_secret_file_path) + logger.info("auth.ldap_secret_file_path : %r" % ldap_secret_file_path) if self._ldap_secret: - logger.info("auth.ldap_secret : (from file)") + logger.info("auth.ldap_secret : (from file)") else: - logger.info("auth.ldap_secret_file_path: (not provided)") + logger.info("auth.ldap_secret_file_path : (not provided)") if self._ldap_secret: - logger.info("auth.ldap_secret : (from config)") + logger.info("auth.ldap_secret : (from config)") if self._ldap_reader_dn and not self._ldap_secret: - logger.error("auth.ldap_secret : (not provided)") + logger.error("auth.ldap_secret : (not provided)") raise RuntimeError("LDAP authentication requires ldap_secret for ldap_reader_dn") - logger.info("auth.ldap_use_ssl : %s" % ldap_use_ssl) - logger.info("auth.ldap_security : %s" % self._ldap_security) - if self._ldap_security in ("tls", "starttls"): - logger.info("auth.ldap_ssl_verify_mode : %s" % self._ldap_ssl_verify_mode) - if self._ldap_ssl_ca_file: - logger.info("auth.ldap_ssl_ca_file : %r" % self._ldap_ssl_ca_file) - else: - logger.info("auth.ldap_ssl_ca_file : (not provided)") + logger.info("auth.ldap_use_ssl : %s" % ldap_use_ssl) + logger.info("auth.ldap_security : %s" % self._ldap_security) + logger.info("auth.ldap_ssl_verify_mode : %s" % self._ldap_ssl_verify_mode) + if self._ldap_ssl_ca_file: + logger.info("auth.ldap_ssl_ca_file : %r" % self._ldap_ssl_ca_file) + else: + logger.info("auth.ldap_ssl_ca_file : (not provided)") if self._ldap_ignore_attribute_create_modify_timestamp: logger.info("auth.ldap_ignore_attribute_create_modify_timestamp applied (relevant for ldap3 only)") """Extend attributes to to be returned in the user query""" @@ -176,7 +175,7 @@ class Auth(auth.BaseAuth): self._ldap_attributes.append(self._ldap_groups_attr) if self._ldap_user_attr: self._ldap_attributes.append(self._ldap_user_attr) - logger.info("ldap_attributes : %r" % self._ldap_attributes) + logger.info("ldap_attributes : %r" % self._ldap_attributes) def _login2(self, login: str, password: str) -> str: try: From 7df4c070e1749beacec3321bb291c60d9d38bc54 Mon Sep 17 00:00:00 2001 From: Peter Marschall Date: Sun, 28 Sep 2025 10:44:33 +0200 Subject: [PATCH 080/290] LDAP auth: fail on illegal values for config settings Thr config settings 'ldap_security' and 'ldap_ssl_verify_mode' only accept a specific set of values: fail if other values are provided. --- radicale/auth/ldap.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/radicale/auth/ldap.py b/radicale/auth/ldap.py index 65eb2c02..5fbe2684 100644 --- a/radicale/auth/ldap.py +++ b/radicale/auth/ldap.py @@ -101,6 +101,8 @@ class Auth(auth.BaseAuth): with open(ldap_secret_file_path, 'r') as file: self._ldap_secret = file.read().rstrip('\n') self._ldap_security = configuration.get("auth", "ldap_security") + if self._ldap_security not in ("none", "tls", "starttls"): + raise RuntimeError("Illegal value for config setting ´ldap_security'") ldap_use_ssl = configuration.get("auth", "ldap_use_ssl") if ldap_use_ssl: logger.warning("Configuration uses deprecated 'ldap_use_ssl': use 'ldap_security' ('none', 'tls', 'starttls') instead.") @@ -115,6 +117,8 @@ class Auth(auth.BaseAuth): self._ldap_ssl_verify_mode = ssl.CERT_NONE elif tmp == "OPTIONAL": self._ldap_ssl_verify_mode = ssl.CERT_OPTIONAL + elif tmp != "REQUIRED": + raise RuntimeError("Illegal value for config setting ´ldap_ssl_verify_mode'") if self._ldap_uri.lower().startswith("ldaps://") and self._ldap_security not in ("tls", "starttls"): logger.info("Inferring 'ldap_security' = tls from 'ldap_uri' starting with 'ldaps://'") From bcba53ed8dbf53da20d77466a949bd1a45305ac5 Mon Sep 17 00:00:00 2001 From: Peter Marschall Date: Sun, 28 Sep 2025 12:31:10 +0200 Subject: [PATCH 081/290] LDAP auth: re-factor handling of 'ldap_ssl_verify_mode' * treat 'ldap_ssl_verify_mode' as string * perform check for accepted values; fail on illegal ones * translate to the values nbeeded by the respective LDAP module when doing the login, based on a module specific dictionary --- radicale/auth/ldap.py | 32 +++++++++++++------------------- 1 file changed, 13 insertions(+), 19 deletions(-) diff --git a/radicale/auth/ldap.py b/radicale/auth/ldap.py index 5fbe2684..a627e132 100644 --- a/radicale/auth/ldap.py +++ b/radicale/auth/ldap.py @@ -68,7 +68,7 @@ class Auth(auth.BaseAuth): _ldap_group_members_attr: str _ldap_module_version: int = 3 _ldap_security: str = "none" - _ldap_ssl_verify_mode: int = ssl.CERT_REQUIRED + _ldap_ssl_verify_mode: str = "REQUIRED" _ldap_ssl_ca_file: str = "" def __init__(self, configuration: config.Configuration) -> None: @@ -112,19 +112,15 @@ class Auth(auth.BaseAuth): logger.warning("Update configuration: set 'ldap_security = tls' instead of deprecated 'ldap_use_ssl = True'") self._ldap_security = "tls" self._ldap_ssl_ca_file = configuration.get("auth", "ldap_ssl_ca_file") - tmp = configuration.get("auth", "ldap_ssl_verify_mode") - if tmp == "NONE": - self._ldap_ssl_verify_mode = ssl.CERT_NONE - elif tmp == "OPTIONAL": - self._ldap_ssl_verify_mode = ssl.CERT_OPTIONAL - elif tmp != "REQUIRED": + self._ldap_ssl_verify_mode = configuration.get("auth", "ldap_ssl_verify_mode") + if self._ldap_ssl_verify_mode not in ("NONE", "OPTIONAL", "REQUIRED"): raise RuntimeError("Illegal value for config setting ´ldap_ssl_verify_mode'") if self._ldap_uri.lower().startswith("ldaps://") and self._ldap_security not in ("tls", "starttls"): logger.info("Inferring 'ldap_security' = tls from 'ldap_uri' starting with 'ldaps://'") self._ldap_security = "tls" - if self._ldap_ssl_ca_file == "" and self._ldap_ssl_verify_mode != ssl.CERT_NONE and self._ldap_security in ("tls", "starttls"): + if self._ldap_ssl_ca_file == "" and self._ldap_ssl_verify_mode != "NONE" and self._ldap_security in ("tls", "starttls"): logger.warning("Certificate verification not possible: 'ldap_ssl_ca_file' not set") if self._ldap_ssl_ca_file and self._ldap_security not in ("tls", "starttls"): logger.warning("Config setting 'ldap_ssl_ca_file' useless without encrypted LDAP connection") @@ -191,12 +187,10 @@ class Auth(auth.BaseAuth): if self._ldap_security in ("tls", "starttls"): """certificate validation mode""" - if self._ldap_ssl_verify_mode == ssl.CERT_REQUIRED: - conn.set_option(self.ldap.OPT_X_TLS_REQUIRE_CERT, self.ldap.OPT_X_TLS_DEMAND) - elif self._ldap_ssl_verify_mode == ssl.CERT_OPTIONAL: - conn.set_option(self.ldap.OPT_X_TLS_REQUIRE_CERT, self.ldap.OPT_X_TLS_ALLOW) - else: - conn.set_option(self.ldap.OPT_X_TLS_REQUIRE_CERT, self.ldap.OPT_X_TLS_NONE) + verifyMode = {"NONE": self.ldap.OPT_X_TLS_NEVER, + "OPTIONAL": self.ldap.OPT_X_TLS_ALLOW, + "REQUIRED": self.ldap.OPT_X_TLS_DEMAND} + conn.set_option(self.ldap.OPT_X_TLS_REQUIRE_CERT, verifyMode[self._ldap_ssl_verify_mode]) """CA file to validate certificate against""" if self._ldap_ssl_ca_file: conn.set_option(self.ldap.OPT_X_TLS_CACERTFILE, self._ldap_ssl_ca_file) @@ -288,12 +282,12 @@ class Auth(auth.BaseAuth): logger.debug(f"_login3 {self._ldap_uri}, {self._ldap_reader_dn}") if self._ldap_security in ("tls", "starttls"): logger.debug("_login3 using encryption (reader)") - tls = self.ldap3.Tls(validate=self._ldap_ssl_verify_mode) + verifyMode = {"NONE": ssl.CERT_NONE, + "OPTIONAL": ssl.CERT_OPTIONAL, + "REQUIRED": ssl.CERT_REQUIRED} + tls = self.ldap3.Tls(validate=verifyMode[self._ldap_ssl_verify_mode]) if self._ldap_ssl_ca_file != "": - tls = self.ldap3.Tls( - validate=self._ldap_ssl_verify_mode, - ca_certs_file=self._ldap_ssl_ca_file - ) + tls = self.ldap3.Tls(validate=verifyMode[self._ldap_ssl_verify_mode], ca_certs_file=self._ldap_ssl_ca_file) if self._ldap_security == "tls": logger.debug("_login3 using ssl (reader)") server = self.ldap3.Server(self._ldap_uri, use_ssl=True, tls=tls) From f0626a8dde005d351f61b93fa9f861d20a7562a2 Mon Sep 17 00:00:00 2001 From: Peter Marschall Date: Sun, 28 Sep 2025 13:17:29 +0200 Subject: [PATCH 082/290] LDAP auth: change 'ldap_ssl_verify_mode' to NONE for ldapi:// For ldapi:// connections, which connect - by definition - to a local UNIX socket, lower the value of config setting 'ldap_ssl_verify_mode' to "NONE" to avoid certificate validation failures. The UNIX socket address can NEVER match any DNS name from a certificate, making the whole certificate validation moot. This is a workaround for a limitation of Python's LDAP modules, that do not consider this edge case. --- radicale/auth/ldap.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/radicale/auth/ldap.py b/radicale/auth/ldap.py index a627e132..48634327 100644 --- a/radicale/auth/ldap.py +++ b/radicale/auth/ldap.py @@ -119,6 +119,9 @@ class Auth(auth.BaseAuth): if self._ldap_uri.lower().startswith("ldaps://") and self._ldap_security not in ("tls", "starttls"): logger.info("Inferring 'ldap_security' = tls from 'ldap_uri' starting with 'ldaps://'") self._ldap_security = "tls" + if self._ldap_uri.lower().startswith("ldapi://") and self._ldap_ssl_verify_mode != "NONE": + logger.info("Lowering 'ldap_'ldap_ssl_verify_mode' to NONE for 'ldap_uri' starting with 'ldapi://'") + self._ldap_ssl_verify_mode = "NONE" if self._ldap_ssl_ca_file == "" and self._ldap_ssl_verify_mode != "NONE" and self._ldap_security in ("tls", "starttls"): logger.warning("Certificate verification not possible: 'ldap_ssl_ca_file' not set") From 2d9830fb6a2e50c30fab18be9f83c6d2162fc658 Mon Sep 17 00:00:00 2001 From: Peter Marschall Date: Mon, 29 Sep 2025 20:17:16 +0200 Subject: [PATCH 083/290] LDAP auth: add my Copyright to radicale/auth/ldap.py --- radicale/auth/ldap.py | 1 + 1 file changed, 1 insertion(+) diff --git a/radicale/auth/ldap.py b/radicale/auth/ldap.py index 48634327..aadbbf64 100644 --- a/radicale/auth/ldap.py +++ b/radicale/auth/ldap.py @@ -1,6 +1,7 @@ # This file is part of Radicale - CalDAV and CardDAV server # Copyright © 2022-2024 Peter Varkoly # Copyright © 2024-2024 Peter Bieringer +# Copyright © 2024-2025 Peter Marschall # # This library is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by From 8ae5831e9c7aafbbf719949a05ef7ae3425a5576 Mon Sep 17 00:00:00 2001 From: Peter Marschall Date: Mon, 29 Sep 2025 20:21:37 +0200 Subject: [PATCH 084/290] LDAP auth: update CHANGELOG.md --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 20e96ab4..8314ad0e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,7 @@ # Changelog ## 3.5.8.dev +* Extend [auth]: re-factor & overhaul LDPA autrhentication, especially for Python's ldap module ## 3.5.7 * Extend: [auth] dovecot: add support for version >= 2.4 From f2afec719081ff065349e847bb094484cc164c12 Mon Sep 17 00:00:00 2001 From: Peter Bieringer Date: Wed, 1 Oct 2025 07:40:48 +0200 Subject: [PATCH 085/290] only publish final releases to PyPI --- .github/workflows/pypi-publish.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/pypi-publish.yml b/.github/workflows/pypi-publish.yml index be98f3bb..57cee4a0 100644 --- a/.github/workflows/pypi-publish.yml +++ b/.github/workflows/pypi-publish.yml @@ -1,7 +1,7 @@ name: PyPI publish on: release: - types: [published] + types: [released] jobs: publish: From 4080d2caddf3d7c0919b86773326e5ec257f049a Mon Sep 17 00:00:00 2001 From: Peter Bieringer Date: Wed, 1 Oct 2025 07:41:41 +0200 Subject: [PATCH 086/290] only publish final releases to docker container --- .github/workflows/docker-publish.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/docker-publish.yml b/.github/workflows/docker-publish.yml index 7537b5b6..3bee7778 100644 --- a/.github/workflows/docker-publish.yml +++ b/.github/workflows/docker-publish.yml @@ -2,7 +2,7 @@ name: Build and publish Docker image on: release: - types: [published] + types: [released] schedule: - cron: '0 0 * * *' workflow_dispatch: From 120fbb7328b0fa4081b8cc9356768611daa856e8 Mon Sep 17 00:00:00 2001 From: Peter Bieringer Date: Wed, 1 Oct 2025 20:36:12 +0200 Subject: [PATCH 087/290] Fix: out-of-range timestamp on 32-bit systems --- radicale/utils.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/radicale/utils.py b/radicale/utils.py index ed6c4ab2..f70e38e7 100644 --- a/radicale/utils.py +++ b/radicale/utils.py @@ -280,10 +280,7 @@ def format_ut(unixtime: int) -> str: # TODO check how to support this better return str(unixtime) if unixtime < DATETIME_MAX_UNIXTIME: - if sys.version_info < (3, 11): - dt = datetime.datetime.utcfromtimestamp(unixtime) - else: - dt = datetime.datetime.fromtimestamp(unixtime, datetime.UTC) + dt = datetime.datetime(1970, 1, 1, tzinfo=datetime.timezone.utc) + datetime.timedelta(seconds=unixtime) r = str(unixtime) + "(" + dt.strftime('%Y-%m-%dT%H:%M:%SZ') + ")" else: r = str(unixtime) + "(>MAX:" + str(DATETIME_MAX_UNIXTIME) + ")" From 2d36346d966177407d44093e540551af96373e9e Mon Sep 17 00:00:00 2001 From: Peter Bieringer Date: Wed, 1 Oct 2025 20:36:19 +0200 Subject: [PATCH 088/290] changelog for Fix: out-of-range timestamp on 32-bit systems --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 8314ad0e..b91a17ab 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,7 @@ ## 3.5.8.dev * Extend [auth]: re-factor & overhaul LDPA autrhentication, especially for Python's ldap module +* Fix: out-of-range timestamp on 32-bit systems ## 3.5.7 * Extend: [auth] dovecot: add support for version >= 2.4 From 2a07c5ab7fb1f0ffbf2a19f3547b4f19147f65a2 Mon Sep 17 00:00:00 2001 From: Peter Bieringer Date: Thu, 2 Oct 2025 07:27:19 +0200 Subject: [PATCH 089/290] coveralls-finish has no further parent, adjustment related to https://github.com/Kozea/Radicale/issues/1881 --- .github/workflows/test.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 3a183f9b..ad728e89 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -54,7 +54,6 @@ jobs: needs: coveralls-test if: github.event_name == 'push' runs-on: ubuntu-latest - continue-on-error: true steps: - uses: actions/setup-python@v5 with: From 8e8f98d2c699d6b815457dc5ad0ba9311d57d671 Mon Sep 17 00:00:00 2001 From: Peter Bieringer Date: Thu, 2 Oct 2025 07:28:50 +0200 Subject: [PATCH 090/290] coveralls-test: do not run on PR, related to https://github.com/Kozea/Radicale/issues/1881 --- .github/workflows/test.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index ad728e89..0492dbca 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -26,6 +26,7 @@ jobs: run: tox -c pyproject.toml -e py coveralls-test: + if: github.event_name == 'push' strategy: matrix: os: [ubuntu-latest] From 5e8d5e81df0519c38f4eae9f1a6998bff1f18403 Mon Sep 17 00:00:00 2001 From: Peter Marschall Date: Fri, 3 Oct 2025 14:07:02 +0200 Subject: [PATCH 091/290] fix typo in changelog entry In commit 8ae5831e9c7aafbbf719949a05ef7ae3425a5576, I updated CHANGELOG.md a bit to hastily, and overlooked 2 typos --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index b91a17ab..7ab0aac3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,7 +1,7 @@ # Changelog ## 3.5.8.dev -* Extend [auth]: re-factor & overhaul LDPA autrhentication, especially for Python's ldap module +* Extend [auth]: re-factor & overhaul LDAP authentication, especially for Python's ldap module * Fix: out-of-range timestamp on 32-bit systems ## 3.5.7 From fbbd116caae0073395810b2340abb7cbd1942735 Mon Sep 17 00:00:00 2001 From: Peter Bieringer Date: Sat, 4 Oct 2025 11:07:49 +0200 Subject: [PATCH 092/290] add missing o (onetime pattern compilation) --- contrib/logwatch/radicale | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/contrib/logwatch/radicale b/contrib/logwatch/radicale index 45298ad4..c882634d 100644 --- a/contrib/logwatch/radicale +++ b/contrib/logwatch/radicale @@ -75,7 +75,7 @@ while (defined($ThisLine = )) { if ( $ThisLine =~ / \S+ response status for .* with depth '(\d)' in ([0-9.]+) seconds: (\d+)/o ) { $req .= ":D=" . $1 . ":R=" . $3; ResponseTimesMinMaxSum($req, $2) if ($Detail >= 10); - } elsif ( $ThisLine =~ / \S+ response status for .* in ([0-9.]+) seconds: (\d+)/ ) { + } elsif ( $ThisLine =~ / \S+ response status for .* in ([0-9.]+) seconds: (\d+)/o ) { $req .= ":R=" . $2; ResponseTimesMinMaxSum($req, $1) if ($Detail >= 10); } From 17b67e899885c6b2b4240ad8c5d8d5e6172debac Mon Sep 17 00:00:00 2001 From: Peter Bieringer Date: Sat, 4 Oct 2025 11:08:12 +0200 Subject: [PATCH 093/290] extend copyright --- contrib/logwatch/radicale | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/contrib/logwatch/radicale b/contrib/logwatch/radicale index c882634d..9634cbdd 100644 --- a/contrib/logwatch/radicale +++ b/contrib/logwatch/radicale @@ -1,6 +1,6 @@ # This file is related to Radicale - CalDAV and CardDAV server # for logwatch (script) -# Copyright © 2024-2024 Peter Bieringer +# Copyright © 2024-2025 Peter Bieringer # # Detail levels # >= 5: Logins From 50a6b8462d908f3feff1d58c995386944aa77883 Mon Sep 17 00:00:00 2001 From: Peter Bieringer Date: Sat, 4 Oct 2025 11:48:08 +0200 Subject: [PATCH 094/290] log size of answer if given --- radicale/app/__init__.py | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/radicale/app/__init__.py b/radicale/app/__init__.py index 0fcbd328..ce948e82 100644 --- a/radicale/app/__init__.py +++ b/radicale/app/__init__.py @@ -164,6 +164,7 @@ class Application(ApplicationPartDelete, ApplicationPartHead, answer: Union[None, str, bytes]) -> _IntermediateResponse: """Helper to create response from internal types.WSGIResponse""" headers = dict(headers) + content_encoding = "plain" # Set content length answers = [] if answer is not None: @@ -183,6 +184,7 @@ class Application(ApplicationPartDelete, ApplicationPartHead, zcomp = zlib.compressobj(wbits=16 + zlib.MAX_WBITS) answer = zcomp.compress(answer) + zcomp.flush() headers["Content-Encoding"] = "gzip" + content_encoding = "gzip" headers["Content-Length"] = str(len(answer)) answers.append(answer) @@ -194,9 +196,14 @@ class Application(ApplicationPartDelete, ApplicationPartHead, time_end = datetime.datetime.now() status_text = "%d %s" % ( status, client.responses.get(status, "Unknown")) - logger.info("%s response status for %r%s in %.3f seconds: %s", - request_method, unsafe_path, depthinfo, - (time_end - time_begin).total_seconds(), status_text) + if answer is not None: + logger.info("%s response status for %r%s in %.3f seconds %s %s bytes: %s", + request_method, unsafe_path, depthinfo, + (time_end - time_begin).total_seconds(), content_encoding, str(len(answer)), status_text) + else: + logger.info("%s response status for %r%s in %.3f seconds: %s", + request_method, unsafe_path, depthinfo, + (time_end - time_begin).total_seconds(), status_text) # Return response content return status_text, list(headers.items()), answers From bb6e9171c737f27de1fd7c2135f9c60ec2c07544 Mon Sep 17 00:00:00 2001 From: Peter Bieringer Date: Sat, 4 Oct 2025 11:48:34 +0200 Subject: [PATCH 095/290] add Response size statisitics --- contrib/logwatch/radicale | 47 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 47 insertions(+) diff --git a/contrib/logwatch/radicale b/contrib/logwatch/radicale index 9634cbdd..7cc1b1b8 100644 --- a/contrib/logwatch/radicale +++ b/contrib/logwatch/radicale @@ -9,6 +9,7 @@ $Detail = $ENV{'LOGWATCH_DETAIL_LEVEL'} || 0; my %ResponseTimes; +my %ResponseSizes; my %Responses; my %Requests; my %Logins; @@ -39,6 +40,28 @@ sub ResponseTimesMinMaxSum($$) { $ResponseTimes{$req}->{'sum'} += $time; } +sub ResponseSizesMinMaxSum($$$) { + my $req = $_[0]; + my $type = $_[1]; + my $size = $_[2]; + + $ResponseSizes{$type}->{$req}->{'cnt'}++; + + if (! defined $ResponseSizes{$type}->{$req}->{'min'}) { + $ResponseSizes{$type}->{$req}->{'min'} = $size; + } elsif ($ResponseSizes{$type}->{$req}->{'min'} > $size) { + $ResponseSizes{$type}->{$req}->{'min'} = $size; + } + + if (! defined $ResponseSizes{$type}->{$req}->{'max'}) { + $ResponseSizes{$type}->{$req}{'max'} = $size; + } elsif ($ResponseSizes{$type}->{$req}->{'max'} < $size) { + $ResponseSizes{$type}->{$req}{'max'} = $size; + } + + $ResponseSizes{$type}->{$req}->{'sum'} += $size; +} + sub Sum($) { my $phash = $_[0]; my $sum = 0; @@ -78,6 +101,14 @@ while (defined($ThisLine = )) { } elsif ( $ThisLine =~ / \S+ response status for .* in ([0-9.]+) seconds: (\d+)/o ) { $req .= ":R=" . $2; ResponseTimesMinMaxSum($req, $1) if ($Detail >= 10); + } elsif ( $ThisLine =~ / \S+ response status for .* with depth '(\d)' in ([0-9.]+) seconds (\S+) (\d+) bytes: (\d+)/o ) { + $req .= ":D=" . $1 . ":R=" . $5; + ResponseTimesMinMaxSum($req, $2) if ($Detail >= 10); + ResponseSizesMinMaxSum($req, $3, $4) if ($Detail >= 10); + } elsif ( $ThisLine =~ / \S+ response status for .* in ([0-9.]+) seconds (\S+) (\d+) bytes: (\d+)/o ) { + $req .= ":R=" . $4; + ResponseTimesMinMaxSum($req, $1) if ($Detail >= 10); + ResponseSizesMinMaxSum($req, $2, $3) if ($Detail >= 10); } $Responses{$req}++; } @@ -174,6 +205,22 @@ if (keys %ResponseTimes) { print "-" x60 . "\n"; } +if (keys %ResponseSizes) { + for my $type (sort keys %ResponseSizes) { + print "\n**Response sizes (counts, bytes: $type) (D= R=)**\n"; + printf "%-18s | %7s | %9s | %9s | %9s |\n", "Response", "cnt", "min", "max", "avg"; + print "-" x66 . "\n"; + foreach my $req (sort keys %{$ResponseSizes{$type}}) { + printf "%-18s | %7d | %9d | %9d | %9d |\n", $req + , $ResponseSizes{$type}->{$req}->{'cnt'} + , $ResponseSizes{$type}->{$req}->{'min'} + , $ResponseSizes{$type}->{$req}->{'max'} + , $ResponseSizes{$type}->{$req}->{'sum'} / $ResponseSizes{$type}->{$req}->{'cnt'}; + } + print "-" x66 . "\n"; + } +} + if (keys %OtherEvents) { print "\n**Other Events**\n"; foreach $ThisOne (sort keys %OtherEvents) { From 126f4a862d417ca8cb2e4b8a76d1e561b797b093 Mon Sep 17 00:00:00 2001 From: Peter Bieringer Date: Sat, 4 Oct 2025 18:29:24 +0200 Subject: [PATCH 096/290] changelog for https://github.com/Kozea/Radicale/pull/1894 --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 7ab0aac3..8f62112d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,6 +3,7 @@ ## 3.5.8.dev * Extend [auth]: re-factor & overhaul LDAP authentication, especially for Python's ldap module * Fix: out-of-range timestamp on 32-bit systems +* Feature: extend logging with response size in bytes and flag served as plain or gzip ## 3.5.7 * Extend: [auth] dovecot: add support for version >= 2.4 From 253d7b365fc81733baf1284c172b2cd2d32494c7 Mon Sep 17 00:00:00 2001 From: Peter Bieringer Date: Sun, 19 Oct 2025 17:52:16 +0200 Subject: [PATCH 097/290] fix improper section, overseen in f0aa588638d04605757db62fae34b335053c5406 --- config | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/config b/config index 79bb275a..70c2343e 100644 --- a/config +++ b/config @@ -181,6 +181,9 @@ # Strip domain name from username #strip_domain = False +# URL Decode the given username (when URL-encoded by the client - useful for iOS devices when using email address) +#urldecode_username = False + [rights] @@ -197,8 +200,6 @@ # Permit overwrite of a collection (global) #permit_overwrite_collection = True -# URL Decode the given username (when URL-encoded by the client - useful for iOS devices when using email address) -# urldecode_username = False [storage] From 8a53939fae94845b540ca681653e1215f05be602 Mon Sep 17 00:00:00 2001 From: Peter Bieringer Date: Sun, 19 Oct 2025 18:54:37 +0200 Subject: [PATCH 098/290] strict_preconditions: new config option --- radicale/config.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/radicale/config.py b/radicale/config.py index 7693e9e6..a4ba6610 100644 --- a/radicale/config.py +++ b/radicale/config.py @@ -430,6 +430,10 @@ DEFAULT_CONFIG_SCHEMA: types.CONFIG_SCHEMA = OrderedDict([ "value": "", "help": "command that is run after changes to storage", "type": str}), + ("strict_preconditions", { + "value": "False", + "help": "strict preconditions check on PUT", + "type": bool}), ("_filesystem_fsync", { "value": "True", "help": "sync all changes to filesystem during requests", From c5d64b84ed6d76434bc29ff9a86e4afa617b7ece Mon Sep 17 00:00:00 2001 From: Peter Bieringer Date: Sun, 19 Oct 2025 18:55:05 +0200 Subject: [PATCH 099/290] strict_preconditions: new config option / doc --- DOCUMENTATION.md | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/DOCUMENTATION.md b/DOCUMENTATION.md index 42aa64b6..828c74c7 100644 --- a/DOCUMENTATION.md +++ b/DOCUMENTATION.md @@ -1523,6 +1523,14 @@ Skip broken item instead of triggering an exception Default: `True` +##### strict_preconditions + +_(>= 3.5.8)_ + +Strict preconditions check on PUT. + +Default: `False` + ##### hook Command that is run after changes to storage. See the From ea1df00161e569829b57da4489d9840ed263c419 Mon Sep 17 00:00:00 2001 From: Peter Bieringer Date: Sun, 19 Oct 2025 18:55:16 +0200 Subject: [PATCH 100/290] strict_preconditions: new config option / example config --- config | 3 +++ 1 file changed, 3 insertions(+) diff --git a/config b/config index 70c2343e..77df29b9 100644 --- a/config +++ b/config @@ -242,6 +242,9 @@ # Skip broken item instead of triggering an exception #skip_broken_item = True +# Strict preconditions check on PUT +#strict_preconditions = False + # Command that is run after changes to storage, default is emtpy # Supported placeholders: # %(user)s: logged-in user From c503542c6c0e2012f7c8bf7772c66b814d14093d Mon Sep 17 00:00:00 2001 From: Peter Bieringer Date: Sun, 19 Oct 2025 18:55:36 +0200 Subject: [PATCH 101/290] strict_preconditions: new config option / handling --- radicale/app/__init__.py | 3 +++ radicale/app/put.py | 3 +++ 2 files changed, 6 insertions(+) diff --git a/radicale/app/__init__.py b/radicale/app/__init__.py index ce948e82..fb2e8e82 100644 --- a/radicale/app/__init__.py +++ b/radicale/app/__init__.py @@ -75,6 +75,7 @@ class Application(ApplicationPartDelete, ApplicationPartHead, _extra_headers: Mapping[str, str] _permit_delete_collection: bool _permit_overwrite_collection: bool + _strict_preconditions: bool def __init__(self, configuration: config.Configuration) -> None: """Initialize Application. @@ -116,6 +117,8 @@ class Application(ApplicationPartDelete, ApplicationPartHead, self._extra_headers = dict() for key in self.configuration.options("headers"): self._extra_headers[key] = configuration.get("headers", key) + self._strict_preconditions = configuration.get("storage", "strict_preconditions") + logger.info("strict preconditions check: %s", self._strict_preconditions) def _scrub_headers(self, environ: types.WSGIEnviron) -> types.WSGIEnviron: """Mask passwords and cookies.""" diff --git a/radicale/app/put.py b/radicale/app/put.py index d7818eaa..6cfed1eb 100644 --- a/radicale/app/put.py +++ b/radicale/app/put.py @@ -207,6 +207,9 @@ class ApplicationPartPut(ApplicationBase): return httputils.NOT_ALLOWED etag = environ.get("HTTP_IF_MATCH", "") + if item and not etag and self._strict_preconditions: + logger.warning("Precondition failed for %r: existing item, no If-Match header, strict mode enabled", path) + return httputils.PRECONDITION_FAILED if not item and etag: # Etag asked but no item found: item has been removed logger.warning("Precondition failed on PUT request for %r (HTTP_IF_MATCH: %s, item not existing)", path, etag) From 6df86987c25984ae616114aaad99acffdc684e27 Mon Sep 17 00:00:00 2001 From: Peter Bieringer Date: Sun, 19 Oct 2025 18:55:58 +0200 Subject: [PATCH 102/290] test: add support for optional HTTP_IF_MATCH header --- radicale/tests/__init__.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/radicale/tests/__init__.py b/radicale/tests/__init__.py index e5ecb1f9..c1a2aab2 100644 --- a/radicale/tests/__init__.py +++ b/radicale/tests/__init__.py @@ -75,6 +75,10 @@ class BaseTest: if login is not None and not isinstance(login, str): raise TypeError("login argument must be %r, not %r" % (str, type(login))) + http_if_match = kwargs.pop("http_if_match", None) + if http_if_match is not None and not isinstance(http_if_match, str): + raise TypeError("http_if_match argument must be %r, not %r" % + (str, type(http_if_match))) environ: Dict[str, Any] = {k.upper(): v for k, v in kwargs.items()} for k, v in environ.items(): if not isinstance(v, str): @@ -84,6 +88,8 @@ class BaseTest: if login: environ["HTTP_AUTHORIZATION"] = "Basic " + base64.b64encode( login.encode(encoding)).decode() + if http_if_match: + environ["HTTP_IF_MATCH"] = http_if_match environ["REQUEST_METHOD"] = method.upper() environ["PATH_INFO"] = path if data is not None: From 8ef8b767f37e099cb0e15106ec4c1ffbb67e3e2a Mon Sep 17 00:00:00 2001 From: Peter Bieringer Date: Sun, 19 Oct 2025 18:56:23 +0200 Subject: [PATCH 103/290] strict_preconditions: new config option / test cases --- radicale/tests/test_base.py | 34 ++++++++++++++++++++++++++++++++++ 1 file changed, 34 insertions(+) diff --git a/radicale/tests/test_base.py b/radicale/tests/test_base.py index a9d0acc7..cf9b87d8 100644 --- a/radicale/tests/test_base.py +++ b/radicale/tests/test_base.py @@ -229,6 +229,40 @@ permissions: RrWw""") _, answer = self.get(path) assert "DTSTAMP:20130902T150159Z" in answer + def test_update_event_no_etag_strict_preconditions_true(self) -> None: + """Update an event without serving etag.""" + self.configure({"storage": {"strict_preconditions": True}}) + self.mkcalendar("/calendar.ics/") + event = get_file_content("event1.ics") + event_modified = get_file_content("event1_modified.ics") + path = "/calendar.ics/event1.ics" + self.put(path, event, check=201) + self.put(path, event_modified, check=412) + + def test_update_event_with_etag_strict_preconditions_true(self) -> None: + """Update an event with serving etag.""" + self.configure({"storage": {"strict_preconditions": True}}) + self.configure({"logging": {"response_content_on_debug": True}}) + self.mkcalendar("/calendar.ics/") + event = get_file_content("event1.ics") + event_modified = get_file_content("event1_modified.ics") + path = "/calendar.ics/event1.ics" + self.put(path, event, check=201) + # get etag + _, responses = self.report("/calendar.ics/", """\ + + + + + +""") + assert len(responses) == 1 + response = responses["/calendar.ics/event1.ics"] + assert not isinstance(response, int) + status, prop = response["D:getetag"] + assert status == 200 and prop.text + self.put(path, event_modified, check=204, http_if_match=prop.text) + def test_update_event_uid_event(self) -> None: """Update an event with a different UID.""" self.mkcalendar("/calendar.ics/") From 661206c7afa02fe4f03eb60e229eec9c40acd342 Mon Sep 17 00:00:00 2001 From: Peter Bieringer Date: Sun, 19 Oct 2025 18:57:02 +0200 Subject: [PATCH 104/290] strict_preconditions: new config option / changelog --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 8f62112d..bc54d8eb 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,7 @@ * Extend [auth]: re-factor & overhaul LDAP authentication, especially for Python's ldap module * Fix: out-of-range timestamp on 32-bit systems * Feature: extend logging with response size in bytes and flag served as plain or gzip +* Feature: [storage] strict_preconditions: new config option to enforce strict precondition check ## 3.5.7 * Extend: [auth] dovecot: add support for version >= 2.4 From c5633b83255fb7f10b366f45f33d2e5c3893aacc Mon Sep 17 00:00:00 2001 From: Peter Bieringer Date: Sun, 19 Oct 2025 20:54:00 +0200 Subject: [PATCH 105/290] fix lint --- radicale/app/__init__.py | 1 - radicale/app/base.py | 1 + 2 files changed, 1 insertion(+), 1 deletion(-) diff --git a/radicale/app/__init__.py b/radicale/app/__init__.py index fb2e8e82..aeb4daf2 100644 --- a/radicale/app/__init__.py +++ b/radicale/app/__init__.py @@ -75,7 +75,6 @@ class Application(ApplicationPartDelete, ApplicationPartHead, _extra_headers: Mapping[str, str] _permit_delete_collection: bool _permit_overwrite_collection: bool - _strict_preconditions: bool def __init__(self, configuration: config.Configuration) -> None: """Initialize Application. diff --git a/radicale/app/base.py b/radicale/app/base.py index 28b6f262..6e3a7cd3 100644 --- a/radicale/app/base.py +++ b/radicale/app/base.py @@ -41,6 +41,7 @@ class ApplicationBase: _encoding: str _permit_delete_collection: bool _permit_overwrite_collection: bool + _strict_preconditions: bool _hook: hook.BaseHook def __init__(self, configuration: config.Configuration) -> None: From a51e6ff65e26833ba9936657c15939329adf22ed Mon Sep 17 00:00:00 2001 From: Peter Bieringer Date: Sun, 19 Oct 2025 20:55:10 +0200 Subject: [PATCH 106/290] remove duplicate definition --- radicale/app/__init__.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/radicale/app/__init__.py b/radicale/app/__init__.py index aeb4daf2..bb418431 100644 --- a/radicale/app/__init__.py +++ b/radicale/app/__init__.py @@ -73,8 +73,6 @@ class Application(ApplicationPartDelete, ApplicationPartHead, _web_type: str _script_name: str _extra_headers: Mapping[str, str] - _permit_delete_collection: bool - _permit_overwrite_collection: bool def __init__(self, configuration: config.Configuration) -> None: """Initialize Application. From 8ace457428f787b55cd875865416356ef784443a Mon Sep 17 00:00:00 2001 From: Peter Bieringer Date: Sun, 19 Oct 2025 21:10:00 +0200 Subject: [PATCH 107/290] strict_preconditions: changelog extension --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index bc54d8eb..12e5c97e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,7 +4,7 @@ * Extend [auth]: re-factor & overhaul LDAP authentication, especially for Python's ldap module * Fix: out-of-range timestamp on 32-bit systems * Feature: extend logging with response size in bytes and flag served as plain or gzip -* Feature: [storage] strict_preconditions: new config option to enforce strict precondition check +* Feature: [storage] strict_preconditions: new config option to enforce strict preconditions check on PUT in case item already exists [RFC6352#9.2] ## 3.5.7 * Extend: [auth] dovecot: add support for version >= 2.4 From 15a6655036a9cc79d921c527bb155aebbcc0c416 Mon Sep 17 00:00:00 2001 From: Peter Bieringer Date: Sun, 19 Oct 2025 21:10:15 +0200 Subject: [PATCH 108/290] strict_preconditions: doc extension --- DOCUMENTATION.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/DOCUMENTATION.md b/DOCUMENTATION.md index 828c74c7..4fb59d48 100644 --- a/DOCUMENTATION.md +++ b/DOCUMENTATION.md @@ -1527,7 +1527,7 @@ Default: `True` _(>= 3.5.8)_ -Strict preconditions check on PUT. +Strict preconditions check on PUT in case item already exists [RFC6352#9.2](https://datatracker.ietf.org/doc/html/rfc6352#section-9.2) Default: `False` From 4fdc78760914040d5f74ece8978013b8836a712e Mon Sep 17 00:00:00 2001 From: Peter Bieringer Date: Sun, 19 Oct 2025 21:13:40 +0200 Subject: [PATCH 109/290] align rfc url --- DOCUMENTATION.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/DOCUMENTATION.md b/DOCUMENTATION.md index 4fb59d48..2d92c7af 100644 --- a/DOCUMENTATION.md +++ b/DOCUMENTATION.md @@ -1527,7 +1527,7 @@ Default: `True` _(>= 3.5.8)_ -Strict preconditions check on PUT in case item already exists [RFC6352#9.2](https://datatracker.ietf.org/doc/html/rfc6352#section-9.2) +Strict preconditions check on PUT in case item already exists [RFC6352#9.2](https://www.rfc-editor.org/rfc/rfc6352#section-9.2) Default: `False` From bd0d5038b3138c3277c59835fdd822af97920c83 Mon Sep 17 00:00:00 2001 From: Peter Bieringer Date: Mon, 20 Oct 2025 07:41:12 +0200 Subject: [PATCH 110/290] cosmetics --- radicale/tests/test_base.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/radicale/tests/test_base.py b/radicale/tests/test_base.py index cf9b87d8..b94ae15e 100644 --- a/radicale/tests/test_base.py +++ b/radicale/tests/test_base.py @@ -230,7 +230,7 @@ permissions: RrWw""") assert "DTSTAMP:20130902T150159Z" in answer def test_update_event_no_etag_strict_preconditions_true(self) -> None: - """Update an event without serving etag.""" + """Update an event without serving etag having strict_preconditions enabled (Precondition Failed).""" self.configure({"storage": {"strict_preconditions": True}}) self.mkcalendar("/calendar.ics/") event = get_file_content("event1.ics") @@ -240,7 +240,7 @@ permissions: RrWw""") self.put(path, event_modified, check=412) def test_update_event_with_etag_strict_preconditions_true(self) -> None: - """Update an event with serving etag.""" + """Update an event with serving equal etag having strict_preconditions enabled (OK).""" self.configure({"storage": {"strict_preconditions": True}}) self.configure({"logging": {"response_content_on_debug": True}}) self.mkcalendar("/calendar.ics/") From 54bd8023b3ba87797712337b56dbd6c53a090ce8 Mon Sep 17 00:00:00 2001 From: Peter Bieringer Date: Mon, 20 Oct 2025 07:41:27 +0200 Subject: [PATCH 111/290] add additional etag related tests --- radicale/tests/test_base.py | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/radicale/tests/test_base.py b/radicale/tests/test_base.py index b94ae15e..c13c8725 100644 --- a/radicale/tests/test_base.py +++ b/radicale/tests/test_base.py @@ -263,6 +263,22 @@ permissions: RrWw""") assert status == 200 and prop.text self.put(path, event_modified, check=204, http_if_match=prop.text) + def test_update_event_with_etag_mismatch(self) -> None: + """Update an event with serving mismatch etag (Precondition Failed).""" + self.mkcalendar("/calendar.ics/") + event = get_file_content("event1.ics") + event_modified = get_file_content("event1_modified.ics") + path = "/calendar.ics/event1.ics" + self.put(path, event, check=201) + self.put(path, event_modified, check=412, http_if_match="0000") + + def test_add_event_with_etag(self) -> None: + """Add an event with serving etag (Precondition Failed).""" + self.mkcalendar("/calendar.ics/") + event = get_file_content("event1.ics") + path = "/calendar.ics/event1.ics" + self.put(path, event, check=412, http_if_match="0000") + def test_update_event_uid_event(self) -> None: """Update an event with a different UID.""" self.mkcalendar("/calendar.ics/") From d17fea063c9133103599cb69783788b313112c4a Mon Sep 17 00:00:00 2001 From: kalsi-avneet <4151485+kalsi-avneet@users.noreply.github.com> Date: Sat, 1 Nov 2025 10:48:38 +0000 Subject: [PATCH 112/290] Docker images : new workflow to cleanup nightly images Criteria for cleanup: Images having tag "nightly-*", and older than 30 days --- .github/workflows/docker-nightly-cleanup.yml | 34 ++++++++++++++++++++ 1 file changed, 34 insertions(+) create mode 100644 .github/workflows/docker-nightly-cleanup.yml diff --git a/.github/workflows/docker-nightly-cleanup.yml b/.github/workflows/docker-nightly-cleanup.yml new file mode 100644 index 00000000..87ce7363 --- /dev/null +++ b/.github/workflows/docker-nightly-cleanup.yml @@ -0,0 +1,34 @@ +on: + schedule: + - cron: '10 0 * * *' + workflow_dispatch: + + +jobs: + delete-package-versions: + name: Delete old nightly docker images + runs-on: ubuntu-latest + steps: + - name: Get list of all docker image versions in registry + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: | + gh api --paginate -X GET "/orgs/Kozea/packages/container/Radicale/versions" -F package_type=container -F per_page=200 > data.json + + - name: Delete each nightly image older than cutoff date + run: | + cutoff_date=$(date --date="30 days ago" --iso-8601) + echo "Cutoff date is: $cutoff_date" + + # Loop through each nightly container version (tag) older than the cutoff date + for tag in $(jq --arg cutoff_date $cutoff_date -r '.[] | select((.metadata.container.tags | any(. | contains("nightly"))) and (.created_at < $cutoff_date)) | .metadata.container.tags[]' data.json); do + echo "Tag - $tag" + + # Because of multi-platform, manifest for each tag would contain more than 1 image. Loop through all + all_digests=$(docker manifest inspect "ghcr.io/kozea/radicale:${tag}" | jq -r 'if .manifests then .manifests[]?.digest else empty end') + for digest in $all_digests; do + image_id=$(jq -r --arg digest "$digest" '.[] | select(.name == $digest) | .id' data.json) + echo "Deleting $image_id" + gh api -X DELETE "/orgs/Kozea/packages/container/Radicale/versions/$image_id" + done + done From 3a51642dde331b0697bea4ba81d94bb03d6c8749 Mon Sep 17 00:00:00 2001 From: goowtham1412-p Date: Sun, 2 Nov 2025 19:40:12 +0530 Subject: [PATCH 113/290] Add Telugu translation of documentation --- DOCUMENTATION.te.md | 284 ++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 284 insertions(+) create mode 100644 DOCUMENTATION.te.md diff --git a/DOCUMENTATION.te.md b/DOCUMENTATION.te.md new file mode 100644 index 00000000..edc17da4 --- /dev/null +++ b/DOCUMENTATION.te.md @@ -0,0 +1,284 @@ +\# డాక్యుమెంటేషన్ + + + +\## ప్రారంభించడం + + + +\#### రాడికేల్ గురించి + + + +రాడికేల్ అనేది ఒక చిన్న కానీ శక్తివంతమైన CalDAV (క్యాలెండర్లు, చేయవలసిన జాబితాలు) మరియు CardDAV + +(పరిచయాలు) సర్వర్, ఇది: + + + +\* CalDAV, CardDAV మరియు HTTP ద్వారా క్యాలెండర్లు మరియు పరిచయ జాబితాలను పంచుకుంటుంది. + +\* ఈవెంట్‌లు, టోడోలు, జర్నల్ ఎంట్రీలు మరియు వ్యాపార కార్డులకు మద్దతు ఇస్తుంది. + +\* బాక్స్ వెలుపల పనిచేస్తుంది, సంక్లిష్టమైన సెటప్ లేదా కాన్ఫిగరేషన్ అవసరం లేదు. + +\* సౌకర్యవంతమైన ప్రామాణీకరణ ఎంపికలను అందిస్తుంది. + +\* అధికారం ద్వారా యాక్సెస్‌ను పరిమితం చేయవచ్చు. + +\* TLSతో కనెక్షన్‌లను సురక్షితం చేయవచ్చు. + +\* చాలా మందితో పనిచేస్తుంది + +\[CalDAV మరియు CardDAV క్లయింట్లు](#సపోర్టెడ్-క్లయింట్లు). + +\* ఫైల్ సిస్టమ్‌లోని అన్ని డేటాను సాధారణ ఫోల్డర్ నిర్మాణంలో నిల్వ చేస్తుంది. + +\* ప్లగిన్‌లతో పొడిగించవచ్చు. + +\* GPLv3-లైసెన్స్ పొందిన ఉచిత సాఫ్ట్‌వేర్. + + + +\#### ఇన్‌స్టాలేషన్ + + + +తనిఖీ చేయండి + + + +\* \[ట్యుటోరియల్స్](#ట్యుటోరియల్స్) + +\* \[డాక్యుమెంటేషన్](#డాక్యుమెంటేషన్-1) + +\* \[GitHubలో వికీ](https://github.com/Kozea/Radicale/wiki) + +\* \[GitHubలో చర్చలు](https://github.com/Kozea/Radicale/discussions) + +\* \[GitHubలో తెరిచి ఉన్న మరియు ఇప్పటికే మూసివేయబడిన సమస్యలు](https://github.com/Kozea/Radicale/issues?q=is%3Aissue) + + + +\#### కొత్తగా ఏముంది? + + + +\[GitHubలో చేంజ్‌లాగ్](https://github.com/Kozea/Radicale/blob/master/CHANGELOG.md) చదవండి. + + + +\## ట్యుటోరియల్స్ + + + +\### 5 నిమిషాల సులభమైన సెటప్ + + + +మీరు Radicaleని ప్రయత్నించాలనుకుంటున్నారా కానీ మీ క్యాలెండర్‌లో 5 నిమిషాలు మాత్రమే ఖాళీగా ఉందా? + + + +ఇప్పుడే వెళ్లి Radicaleతో కొంచెం ఆడుదాం! + + + +ఈ విభాగం నుండి సెట్టింగ్‌లతో కాన్ఫిగర్ చేయబడిన సర్వర్, localhost + +కి మాత్రమే బైండ్ అవుతుంది (అంటే ఇది నెట్‌వర్క్ ద్వారా చేరుకోలేరు), మరియు మీరు ఏదైనా వినియోగదారు పేరు మరియు పాస్‌వర్డ్‌తో లాగిన్ అవ్వవచ్చు. + + + +ప్రతిదీ పనిచేసినప్పుడు, మీరు స్థానిక \[client](#supported-clients) + +ని పొందవచ్చు మరియు క్యాలెండర్‌లు మరియు చిరునామా పుస్తకాలను సృష్టించడం ప్రారంభించవచ్చు. + + + +Radicale మీ అవసరాలకు సరిపోతుంటే, రిమోట్ క్లయింట్‌లు మరియు కావలసిన ప్రామాణీకరణ రకానికి మద్దతు ఇవ్వడానికి కొంత \[ప్రాథమిక కాన్ఫిగరేషన్](#basic-configuration) + +కి సమయం కావచ్చు. + + + +మీ ఆపరేటింగ్ సిస్టమ్‌ను బట్టి దిగువన ఉన్న అధ్యాయాలలో ఒకదాన్ని అనుసరించండి. + + + +\#### Linux / \\\*BSD + + + +సూచన: PyPI నుండి డౌన్‌లోడ్ చేయడానికి బదులుగా, మీ \[distribution](#linux-distribution-packages) అందించిన ప్యాకేజీల కోసం చూడండి. + + + +అవి మీ పంపిణీలలో ఇంటిగ్రేట్ చేయబడిన స్టార్టప్ స్క్రిప్ట్‌లను కూడా కలిగి ఉంటాయి, ఇవి Radicaleని డెమోనైజ్ చేయడానికి అనుమతిస్తాయి. + + + +ముందుగా, \*\*python\*\* 3.9 లేదా తరువాత మరియు \*\*pip\*\* ఇన్‌స్టాల్ చేయబడిందని నిర్ధారించుకోండి. చాలా డిస్ట్రిబ్యూషన్లలో ``python3-pip`` ప్యాకేజీని ఇన్‌స్టాల్ చేయడానికి సరిపోతుంది. + + + +\##### సాధారణ వినియోగదారుగా + + + +పరీక్ష కోసం మాత్రమే సిఫార్సు చేయబడింది - కన్సోల్‌ను తెరిచి ఇలా టైప్ చేయండి: + + + +```bash + +\# ప్రస్తుత వినియోగదారు కోసం మాత్రమే ఇన్‌స్టాల్ చేయడానికి కింది ఆదేశాన్ని అమలు చేయండి + +python3 -m pip install --user --upgrade https://github.com/Kozea/Radicale/archive/master.tar.gz + +``` + + + +\_install\_ పని చేయకపోతే మరియు బదులుగా `error: externally-managed-environment` ప్రదర్శించబడితే, + +ముందుగానే వర్చువల్ వాతావరణాన్ని సృష్టించండి మరియు సక్రియం చేయండి. + + + +```bash + +python3 -m venv ~/venv + +source ~/venv/bin/activate + +``` + + + +మరియు దీనితో ఇన్‌స్టాల్ చేయడానికి ప్రయత్నించండి + + + +```bash + +python3 -m pip install --upgrade https://github.com/Kozea/Radicale/archive/master.tar.gz + +``` + + + +సేవను మాన్యువల్‌గా ప్రారంభించండి, డేటా ప్రస్తుత వినియోగదారు కోసం మాత్రమే నిల్వ చేయబడుతుంది + + + +```bash + +\# ప్రారంభించు, డేటా ప్రస్తుత వినియోగదారు కోసం మాత్రమే నిల్వ చేయబడుతుంది + +python3 -m radicale --storage-filesystem-folder=~/.var/lib/radicale/collections --auth-type none + +``` + + + +\#### సిస్టమ్ వినియోగదారుగా (లేదా రూట్‌గా) + + + +ప్రత్యామ్నాయంగా, మీరు సిస్టమ్ వినియోగదారుగా లేదా రూట్‌గా ఇన్‌స్టాల్ చేసి అమలు చేయవచ్చు (సిఫార్సు చేయబడలేదు): + + + +```bash + +\# కింది ఆదేశాన్ని రూట్ (సిఫార్సు చేయబడలేదు) లేదా రూట్ కాని వ్యవస్థ వినియోగదారుగా అమలు చేయండి + +\# (డిపెండెన్సీలు లేనప్పుడు తరువాతి వాటికి --user అవసరం కావచ్చు సిస్టమ్-వైడ్ మరియు/లేదా వర్చువల్ ఎన్విరాన్మెంట్ అందుబాటులో ఉంది) + +python3 -m pip install --upgrade https://github.com/Kozea/Radicale/archive/master.tar.gz + +``` + + + +`/var/lib/radicale/collections` కింద సిస్టమ్ ఫోల్డర్‌లో నిల్వ చేయబడిన డేటాతో సేవను మాన్యువల్‌గా ప్రారంభించండి: + + + +```bash + +\# Start, డేటా సిస్టమ్ ఫోల్డర్‌లో నిల్వ చేయబడుతుంది (/var/lib/radicale/collections కు వ్రాయడానికి అనుమతులు అవసరం) + +python3 -m radicale --storage-filesystem-folder=/var/lib/radicale/collections --auth-type none + +``` + + + +\#### Windows + + + +మొదటి దశ పైథాన్‌ను ఇన్‌స్టాల్ చేయడం. + +\[python.org](https://python.org) కు వెళ్లి పైథాన్ 3 యొక్క తాజా వెర్షన్‌ను డౌన్‌లోడ్ చేసుకోండి. + +తర్వాత ఇన్‌స్టాలర్‌ను అమలు చేయండి. + +ఇన్‌స్టాలర్ యొక్క మొదటి విండోలో, "PATH కు పైథాన్‌ను జోడించు" బాక్స్‌ను తనిఖీ చేసి, + +"ఇప్పుడే ఇన్‌స్టాల్ చేయి"పై క్లిక్ చేయండి. రెండు నిమిషాలు వేచి ఉండండి, పూర్తయింది! + + + +కమాండ్ ప్రాంప్ట్‌ను ప్రారంభించి ఇలా టైప్ చేయండి: + + + +```powershell + +python -m pip install --upgrade https://github.com/Kozea/Radicale/archive/master.tar.gz + +python -m radicale --storage-filesystem-folder=~/radicale/collections --auth-type none + +``` + + + +\##### Common + + + +విజయవంతం!!! మీ బ్రౌజర్‌లో తెరవండి! + +ఉదాహరణ ఎంపిక `--auth-type none` ద్వారా ప్రామాణీకరణ అవసరం లేనందున మీరు ఏదైనా వినియోగదారు పేరు మరియు పాస్‌వర్డ్‌తో లాగిన్ అవ్వవచ్చు. + +ఇది \*\*సురక్షితం\*\*, మరిన్ని వివరాల కోసం \[కాన్ఫిగరేషన్/ప్రామాణీకరణ](#auth) చూడండి. + + + +భద్రతా కారణాల దృష్ట్యా డిఫాల్ట్ కాన్ఫిగరేషన్ సర్వర్‌ను `localhost` (IPv4: `127.0.0.1`, IPv6: `::1`) కు బంధిస్తుందని గమనించండి. + + + +మరిన్ని వివరాల కోసం \[చిరునామాలు](#చిరునామాలు) మరియు \[కాన్ఫిగరేషన్/సర్వర్](#సర్వర్) చూడండి. + + + +\### ప్రాథమిక కాన్ఫిగరేషన్ + + + +ఇన్‌స్టాలేషన్ సూచనలను + +\[సరళమైన 5-నిమిషాల సెటప్](#సింపుల్-5-నిమిషాల-సెటప్) ట్యుటోరియల్‌లో చూడవచ్చు. + + + +రాడికేల్ `/etc/radicale/config` మరియు + +`~/.config/radicale/config` నుండి కాన్ఫిగరేషన్ ఫైల్‌లను లోడ్ చేయడానికి ప్రయత్నిస్తుంది. + +Cu + From 012e50e3edb41ca6eb58e7d5554f7b5e13747e99 Mon Sep 17 00:00:00 2001 From: Peter Bieringer Date: Sun, 2 Nov 2025 18:33:04 +0100 Subject: [PATCH 114/290] add min_unixtime constant --- radicale/utils.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/radicale/utils.py b/radicale/utils.py index f70e38e7..131e4891 100644 --- a/radicale/utils.py +++ b/radicale/utils.py @@ -47,8 +47,9 @@ ADDRESS_TYPE = Union[Tuple[Union[str, bytes, bytearray], int], Tuple[str, int, int, int]] -# Max YEAR in datetime in unixtime +# Max/Min YEAR in datetime in unixtime DATETIME_MAX_UNIXTIME: int = (datetime.MAXYEAR - 1970) * 365 * 24 * 60 * 60 +DATETIME_MIN_UNIXTIME: int = (datetime.MINYEAR - 1970) * 365 * 24 * 60 * 60 def load_plugin(internal_types: Sequence[str], module_name: str, From bd1497ca15786b4a78807e9ac27b8d13677610eb Mon Sep 17 00:00:00 2001 From: Peter Bieringer Date: Sun, 2 Nov 2025 18:33:28 +0100 Subject: [PATCH 115/290] catch min/max to avoid issues on 32-bit systems --- radicale/utils.py | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/radicale/utils.py b/radicale/utils.py index 131e4891..ba5c69ff 100644 --- a/radicale/utils.py +++ b/radicale/utils.py @@ -280,9 +280,14 @@ def format_ut(unixtime: int) -> str: if sys.platform == "win32": # TODO check how to support this better return str(unixtime) - if unixtime < DATETIME_MAX_UNIXTIME: - dt = datetime.datetime(1970, 1, 1, tzinfo=datetime.timezone.utc) + datetime.timedelta(seconds=unixtime) - r = str(unixtime) + "(" + dt.strftime('%Y-%m-%dT%H:%M:%SZ') + ")" + if unixtime <= DATETIME_MIN_UNIXTIME: + r = str(unixtime) + "(<=MIN:" + str(DATETIME_MIN_UNIXTIME) + ")" + elif unixtime >= DATETIME_MAX_UNIXTIME: + r = str(unixtime) + "(>=MAX:" + str(DATETIME_MAX_UNIXTIME) + ")" else: - r = str(unixtime) + "(>MAX:" + str(DATETIME_MAX_UNIXTIME) + ")" + if sys.version_info < (3, 11): + dt = datetime.datetime.utcfromtimestamp(unixtime) + else: + dt = datetime.datetime.fromtimestamp(unixtime, datetime.UTC) + r = str(unixtime) + "(" + dt.strftime('%Y-%m-%dT%H:%M:%SZ') + ")" return r From 91a92226ba4985a4ea686abc8da9f7949c147cda Mon Sep 17 00:00:00 2001 From: Peter Bieringer Date: Sun, 2 Nov 2025 18:33:54 +0100 Subject: [PATCH 116/290] changelog for fixing #1904 --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 12e5c97e..cde3ad4f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,7 @@ * Fix: out-of-range timestamp on 32-bit systems * Feature: extend logging with response size in bytes and flag served as plain or gzip * Feature: [storage] strict_preconditions: new config option to enforce strict preconditions check on PUT in case item already exists [RFC6352#9.2] +* Fix: format_ut problem on 32-bit systems ## 3.5.7 * Extend: [auth] dovecot: add support for version >= 2.4 From 72399d076e31af96da21b020b65d08018b62d2dd Mon Sep 17 00:00:00 2001 From: kalsi-avneet <4151485+kalsi-avneet@users.noreply.github.com> Date: Mon, 3 Nov 2025 16:41:13 +0000 Subject: [PATCH 117/290] Docker images : Nightly image cleanup - update workflow file --- .github/workflows/docker-nightly-cleanup.yml | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/.github/workflows/docker-nightly-cleanup.yml b/.github/workflows/docker-nightly-cleanup.yml index 87ce7363..77550c98 100644 --- a/.github/workflows/docker-nightly-cleanup.yml +++ b/.github/workflows/docker-nightly-cleanup.yml @@ -1,3 +1,5 @@ +name: Cleanup old nightly docker images + on: schedule: - cron: '10 0 * * *' @@ -6,7 +8,7 @@ on: jobs: delete-package-versions: - name: Delete old nightly docker images + name: Cleanup old nightly docker images runs-on: ubuntu-latest steps: - name: Get list of all docker image versions in registry @@ -16,12 +18,14 @@ jobs: gh api --paginate -X GET "/orgs/Kozea/packages/container/Radicale/versions" -F package_type=container -F per_page=200 > data.json - name: Delete each nightly image older than cutoff date + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} run: | cutoff_date=$(date --date="30 days ago" --iso-8601) echo "Cutoff date is: $cutoff_date" - # Loop through each nightly container version (tag) older than the cutoff date - for tag in $(jq --arg cutoff_date $cutoff_date -r '.[] | select((.metadata.container.tags | any(. | contains("nightly"))) and (.created_at < $cutoff_date)) | .metadata.container.tags[]' data.json); do + # Loop through each nightly image version (tag) older than the cutoff date + jq --arg cutoff_date "$cutoff_date" -r '.[] | select((.metadata.container.tags | any(. | contains("nightly"))) and (.created_at < $cutoff_date)) | [.metadata.container.tags[], .id] | @tsv' data.json | while IFS=$'\t' read -r tag nightly_image_id ; do echo "Tag - $tag" # Because of multi-platform, manifest for each tag would contain more than 1 image. Loop through all @@ -31,4 +35,7 @@ jobs: echo "Deleting $image_id" gh api -X DELETE "/orgs/Kozea/packages/container/Radicale/versions/$image_id" done + # Now that all dependents are deleted, delete this tag + echo "Deleting $tag with ID: $nightly_image_id" + gh api -X DELETE "/orgs/Kozea/packages/container/Radicale/versions/$nightly_image_id" done From 9fbb876631326f4d8383b89a4a607e6de4aa9ede Mon Sep 17 00:00:00 2001 From: goowtham1412-p Date: Wed, 5 Nov 2025 08:14:10 +0530 Subject: [PATCH 118/290] Move Telugu docs to docs/ folder and add version info --- DOCUMENTATION.te.md => docs/DOCUMENTATION.te.md | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename DOCUMENTATION.te.md => docs/DOCUMENTATION.te.md (100%) diff --git a/DOCUMENTATION.te.md b/docs/DOCUMENTATION.te.md similarity index 100% rename from DOCUMENTATION.te.md rename to docs/DOCUMENTATION.te.md From b707d143937f87f41b604ba8cb75696593476ba4 Mon Sep 17 00:00:00 2001 From: goowtham1412-p Date: Wed, 5 Nov 2025 08:18:20 +0530 Subject: [PATCH 119/290] Add version information header to Telugu documentation --- docs/DOCUMENTATION.te.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docs/DOCUMENTATION.te.md b/docs/DOCUMENTATION.te.md index edc17da4..802329ca 100644 --- a/docs/DOCUMENTATION.te.md +++ b/docs/DOCUMENTATION.te.md @@ -1,3 +1,5 @@ +> Last updated: 2024-11-02 +> Based on commit: [4fdc78760914040d5f74ece8978013b8836a712e] of DOCUMENTATION.md \# డాక్యుమెంటేషన్ From 354b572598aa21a8a714547142b961814ceaba42 Mon Sep 17 00:00:00 2001 From: goowtham1412-p Date: Wed, 5 Nov 2025 08:23:06 +0530 Subject: [PATCH 120/290] Add version information header to Telugu documentation. --- docs/DOCUMENTATION.te.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/docs/DOCUMENTATION.te.md b/docs/DOCUMENTATION.te.md index 802329ca..01970d69 100644 --- a/docs/DOCUMENTATION.te.md +++ b/docs/DOCUMENTATION.te.md @@ -1,5 +1,6 @@ -> Last updated: 2024-11-02 +> Last updated: 2025-10-20 > Based on commit: [4fdc78760914040d5f74ece8978013b8836a712e] of DOCUMENTATION.md + \# డాక్యుమెంటేషన్ From 2ded487ba1a5415c6fa38d0481d95920017b2bc7 Mon Sep 17 00:00:00 2001 From: Peter Bieringer Date: Wed, 5 Nov 2025 06:44:31 +0100 Subject: [PATCH 121/290] add link to Telugu translation, related to https://github.com/Kozea/Radicale/issues/1906 --- DOCUMENTATION.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/DOCUMENTATION.md b/DOCUMENTATION.md index 2d92c7af..13386eec 100644 --- a/DOCUMENTATION.md +++ b/DOCUMENTATION.md @@ -1,5 +1,9 @@ # Documentation +## Translations of this page + +* [Telugu](https://github.com/Kozea/Radicale/blob/master/docs/DOCUMENTATION.te.md) by [@gowtham1412-p](https://github.com/gowtham1412-p) + ## Getting started #### About Radicale From 975618a21ed25056229808494f1d83e594a24543 Mon Sep 17 00:00:00 2001 From: Peter Bieringer Date: Wed, 5 Nov 2025 06:47:25 +0100 Subject: [PATCH 122/290] move author of Telugu translation, related to https://github.com/Kozea/Radicale/issues/1906 --- CHANGELOG.md | 1 + DOCUMENTATION.md | 2 +- docs/DOCUMENTATION.te.md | 2 +- 3 files changed, 3 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index cde3ad4f..e306cd49 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,7 @@ * Feature: extend logging with response size in bytes and flag served as plain or gzip * Feature: [storage] strict_preconditions: new config option to enforce strict preconditions check on PUT in case item already exists [RFC6352#9.2] * Fix: format_ut problem on 32-bit systems +* Doc: Telugu translation ## 3.5.7 * Extend: [auth] dovecot: add support for version >= 2.4 diff --git a/DOCUMENTATION.md b/DOCUMENTATION.md index 13386eec..5909d9f6 100644 --- a/DOCUMENTATION.md +++ b/DOCUMENTATION.md @@ -2,7 +2,7 @@ ## Translations of this page -* [Telugu](https://github.com/Kozea/Radicale/blob/master/docs/DOCUMENTATION.te.md) by [@gowtham1412-p](https://github.com/gowtham1412-p) +* [Telugu](https://github.com/Kozea/Radicale/blob/master/docs/DOCUMENTATION.te.md) ## Getting started diff --git a/docs/DOCUMENTATION.te.md b/docs/DOCUMENTATION.te.md index 01970d69..e078d24e 100644 --- a/docs/DOCUMENTATION.te.md +++ b/docs/DOCUMENTATION.te.md @@ -1,4 +1,4 @@ -> Last updated: 2025-10-20 +> Last updated: 2025-10-20 by [@gowtham1412-p](https://github.com/gowtham1412-p) > Based on commit: [4fdc78760914040d5f74ece8978013b8836a712e] of DOCUMENTATION.md \# డాక్యుమెంటేషన్ From 2d7d29edd95a89352d67d1063ea6cc2aca35ca0c Mon Sep 17 00:00:00 2001 From: Peter Bieringer Date: Wed, 5 Nov 2025 06:48:51 +0100 Subject: [PATCH 123/290] add and change links, related to https://github.com/Kozea/Radicale/issues/1906 --- DOCUMENTATION.md | 2 +- docs/DOCUMENTATION.te.md | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/DOCUMENTATION.md b/DOCUMENTATION.md index 5909d9f6..536048de 100644 --- a/DOCUMENTATION.md +++ b/DOCUMENTATION.md @@ -2,7 +2,7 @@ ## Translations of this page -* [Telugu](https://github.com/Kozea/Radicale/blob/master/docs/DOCUMENTATION.te.md) +* [Telugu](docs/DOCUMENTATION.te.md) ## Getting started diff --git a/docs/DOCUMENTATION.te.md b/docs/DOCUMENTATION.te.md index e078d24e..95b0e5fd 100644 --- a/docs/DOCUMENTATION.te.md +++ b/docs/DOCUMENTATION.te.md @@ -1,5 +1,5 @@ > Last updated: 2025-10-20 by [@gowtham1412-p](https://github.com/gowtham1412-p) -> Based on commit: [4fdc78760914040d5f74ece8978013b8836a712e] of DOCUMENTATION.md +> Based on commit: [4fdc78760914040d5f74ece8978013b8836a712e] of [DOCUMENTATION.md](../DOCUMENTATION.md) \# డాక్యుమెంటేషన్ From 0c9af8c51c0fafe136ada3da2f6adf140198d6b5 Mon Sep 17 00:00:00 2001 From: Peter Bieringer Date: Wed, 5 Nov 2025 06:51:40 +0100 Subject: [PATCH 124/290] cosmetics, related to https://github.com/Kozea/Radicale/issues/1906 --- docs/DOCUMENTATION.te.md | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/DOCUMENTATION.te.md b/docs/DOCUMENTATION.te.md index 95b0e5fd..812b93f5 100644 --- a/docs/DOCUMENTATION.te.md +++ b/docs/DOCUMENTATION.te.md @@ -1,4 +1,5 @@ > Last updated: 2025-10-20 by [@gowtham1412-p](https://github.com/gowtham1412-p) + > Based on commit: [4fdc78760914040d5f74ece8978013b8836a712e] of [DOCUMENTATION.md](../DOCUMENTATION.md) \# డాక్యుమెంటేషన్ From 5254efa2237d26cdef72d49c3b47ae4270df46b6 Mon Sep 17 00:00:00 2001 From: Peter Bieringer Date: Wed, 5 Nov 2025 06:54:09 +0100 Subject: [PATCH 125/290] revert to absolut URLs, otherwise links are not working in generated web page, related to https://github.com/Kozea/Radicale/issues/1906 --- DOCUMENTATION.md | 2 +- docs/DOCUMENTATION.te.md | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/DOCUMENTATION.md b/DOCUMENTATION.md index 536048de..5909d9f6 100644 --- a/DOCUMENTATION.md +++ b/DOCUMENTATION.md @@ -2,7 +2,7 @@ ## Translations of this page -* [Telugu](docs/DOCUMENTATION.te.md) +* [Telugu](https://github.com/Kozea/Radicale/blob/master/docs/DOCUMENTATION.te.md) ## Getting started diff --git a/docs/DOCUMENTATION.te.md b/docs/DOCUMENTATION.te.md index 812b93f5..2187594e 100644 --- a/docs/DOCUMENTATION.te.md +++ b/docs/DOCUMENTATION.te.md @@ -1,6 +1,6 @@ > Last updated: 2025-10-20 by [@gowtham1412-p](https://github.com/gowtham1412-p) -> Based on commit: [4fdc78760914040d5f74ece8978013b8836a712e] of [DOCUMENTATION.md](../DOCUMENTATION.md) +> Based on commit: [4fdc78760914040d5f74ece8978013b8836a712e] of [DOCUMENTATION.md](https://github.com/Kozea/Radicale/blob/master/DOCUMENTATION.md) \# డాక్యుమెంటేషన్ From 045370571e8a1013b75cb9099d905bcd6edb2029 Mon Sep 17 00:00:00 2001 From: Peter Bieringer Date: Thu, 6 Nov 2025 06:29:21 +0100 Subject: [PATCH 126/290] release 3.5.8 --- CHANGELOG.md | 2 +- pyproject.toml | 2 +- setup.py.legacy | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index e306cd49..13449634 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,6 @@ # Changelog -## 3.5.8.dev +## 3.5.8 * Extend [auth]: re-factor & overhaul LDAP authentication, especially for Python's ldap module * Fix: out-of-range timestamp on 32-bit systems * Feature: extend logging with response size in bytes and flag served as plain or gzip diff --git a/pyproject.toml b/pyproject.toml index 505b295a..08fde8ed 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -3,7 +3,7 @@ name = "Radicale" # When the version is updated, a new section in the CHANGELOG.md file must be # added too. readme = "README.md" -version = "3.5.8.dev" +version = "3.5.8" authors = [{name = "Guillaume Ayoub", email = "guillaume.ayoub@kozea.fr"}, {name = "Unrud", email = "unrud@outlook.com"}, {name = "Peter Bieringer", email = "pb@bieringer.de"}] license = {text = "GNU GPL v3"} description = "CalDAV and CardDAV Server" diff --git a/setup.py.legacy b/setup.py.legacy index 520be1bd..8349ddaa 100644 --- a/setup.py.legacy +++ b/setup.py.legacy @@ -20,7 +20,7 @@ from setuptools import find_packages, setup # When the version is updated, a new section in the CHANGELOG.md file must be # added too. -VERSION = "3.5.8.dev" +VERSION = "3.5.8" with open("README.md", encoding="utf-8") as f: long_description = f.read() From e9cef4e265e8df5cae1b6a5ac3ff0d3f277a4127 Mon Sep 17 00:00:00 2001 From: Peter Bieringer Date: Sun, 9 Nov 2025 06:13:33 +0100 Subject: [PATCH 127/290] prep 3.5.9.dev --- CHANGELOG.md | 2 ++ pyproject.toml | 2 +- setup.py.legacy | 2 +- 3 files changed, 4 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 13449634..e2863fb3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,7 @@ # Changelog +## 3.5.9.dev + ## 3.5.8 * Extend [auth]: re-factor & overhaul LDAP authentication, especially for Python's ldap module * Fix: out-of-range timestamp on 32-bit systems diff --git a/pyproject.toml b/pyproject.toml index 08fde8ed..034e5446 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -3,7 +3,7 @@ name = "Radicale" # When the version is updated, a new section in the CHANGELOG.md file must be # added too. readme = "README.md" -version = "3.5.8" +version = "3.5.9.dev" authors = [{name = "Guillaume Ayoub", email = "guillaume.ayoub@kozea.fr"}, {name = "Unrud", email = "unrud@outlook.com"}, {name = "Peter Bieringer", email = "pb@bieringer.de"}] license = {text = "GNU GPL v3"} description = "CalDAV and CardDAV Server" diff --git a/setup.py.legacy b/setup.py.legacy index 8349ddaa..83545da7 100644 --- a/setup.py.legacy +++ b/setup.py.legacy @@ -20,7 +20,7 @@ from setuptools import find_packages, setup # When the version is updated, a new section in the CHANGELOG.md file must be # added too. -VERSION = "3.5.8" +VERSION = "3.5.9.dev" with open("README.md", encoding="utf-8") as f: long_description = f.read() From e47920b6ff0420e15c0b55aae766bf5b2714249a Mon Sep 17 00:00:00 2001 From: Peter Bieringer Date: Tue, 11 Nov 2025 07:58:04 +0100 Subject: [PATCH 128/290] extend with Python 3.14 --- .github/workflows/test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 0492dbca..25afd109 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -6,7 +6,7 @@ jobs: strategy: matrix: os: [ubuntu-latest, macos-latest, windows-latest] - python-version: ['3.9', '3.10', '3.11', '3.12', '3.13', 'pypy-3.9', 'pypy-3.10', 'pypy-3.11'] + python-version: ['3.9', '3.10', '3.11', '3.12', '3.13', '3.14', 'pypy-3.9', 'pypy-3.10', 'pypy-3.11'] exclude: - os: windows-latest python-version: 'pypy-3.9' From f095d296967174d77a4a12d7f82c542219259f66 Mon Sep 17 00:00:00 2001 From: Peter Bieringer Date: Tue, 11 Nov 2025 07:58:19 +0100 Subject: [PATCH 129/290] use 3.13.latest instead of 3.13.0 --- .github/workflows/test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 25afd109..03dcd687 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -72,7 +72,7 @@ jobs: - uses: actions/checkout@v4 - uses: actions/setup-python@v5 with: - python-version: '3.13.0' + python-version: '3.13' - name: Install tox run: pip install tox - name: Lint From 7bd49475fe85e725dc7d030c0c7f39b8965894aa Mon Sep 17 00:00:00 2001 From: Wei-Luan Wang Date: Wed, 12 Nov 2025 21:47:07 +0800 Subject: [PATCH 130/290] doc: Add a missing parenthesis --- DOCUMENTATION.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/DOCUMENTATION.md b/DOCUMENTATION.md index 5909d9f6..69405101 100644 --- a/DOCUMENTATION.md +++ b/DOCUMENTATION.md @@ -272,7 +272,7 @@ requirements. Recommendation: check support by [Linux Distribution Packages](#linux-distribution-packages) instead of manual setup / initial configuration. -Create the **radicale** user and group for the Radicale service by running (as `root`: +Create the **radicale** user and group for the Radicale service by running (as `root`): ```bash useradd --system --user-group --home-dir / --shell /sbin/nologin radicale ``` From a94028979854663475e006a89bf626c6ab77b841 Mon Sep 17 00:00:00 2001 From: Peter Bieringer Date: Thu, 13 Nov 2025 06:24:34 +0100 Subject: [PATCH 131/290] add hints related to auth and move, related to https://github.com/Kozea/Radicale/issues/1912 --- contrib/caddy/radicale.caddyfile | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/contrib/caddy/radicale.caddyfile b/contrib/caddy/radicale.caddyfile index 6739283b..b578b383 100644 --- a/contrib/caddy/radicale.caddyfile +++ b/contrib/caddy/radicale.caddyfile @@ -16,11 +16,17 @@ caldav.example.com { not path /.web/* } + # disable this in case authentication is handled by Radicale basic_auth @not-webui { USER HASH } reverse_proxy localhost:5232 { + # disable this in case authentication is handled by Radicale header_up X-Remote-User {http.auth.user.id} + # replace "HOST" with configured hostname of URL (FQDN) in client + header_up Host HOST + # replace "PORT" with configured port of URL in client + header_up X-Forwarded-Port PORT } } From bf8619a41cc6723cfc0565808b420afc7b806eb7 Mon Sep 17 00:00:00 2001 From: Peter Bieringer Date: Sat, 15 Nov 2025 15:32:13 +0100 Subject: [PATCH 132/290] add support for http_remote_user --- DOCUMENTATION.md | 9 +++++++- config | 2 +- radicale/app/__init__.py | 2 +- radicale/auth/__init__.py | 4 +++- radicale/auth/http_remote_user.py | 36 +++++++++++++++++++++++++++++++ radicale/tests/test_auth.py | 17 +++++++++++++++ 6 files changed, 66 insertions(+), 4 deletions(-) create mode 100644 radicale/auth/http_remote_user.py diff --git a/DOCUMENTATION.md b/DOCUMENTATION.md index 69405101..855c671a 100644 --- a/DOCUMENTATION.md +++ b/DOCUMENTATION.md @@ -902,8 +902,15 @@ Available types are: Requires validation, otherwise clients can supply the header themselves, which then is unconditionally trusted. +* `http_remote_user` _(>= 3.5.9)_ + Takes the username from the Remote-User HTTP header `HTTP_REMOTE_USER` and disables + Radicale's internal HTTP authentication. This can be used to provide the + username from a reverse proxy which authenticated the client upfront. + Requires validation, otherwise clients can supply the header themselves, + which then is unconditionally trusted. + * `http_x_remote_user` - Takes the username from the `X-Remote-User` HTTP header and disables + Takes the username from the X-Remote-User HTTP header `HTTP_X_REMOTE_USER` and disables Radicale's internal HTTP authentication. This can be used to provide the username from a reverse proxy which authenticated the client upfront. Requires validation, otherwise clients can supply the header themselves, diff --git a/config b/config index 77df29b9..93291650 100644 --- a/config +++ b/config @@ -63,7 +63,7 @@ [auth] # Authentication method -# Value: none | htpasswd | remote_user | http_x_remote_user | dovecot | ldap | oauth2 | pam | denyall +# Value: none | htpasswd | remote_user | http_remote_user | http_x_remote_user | dovecot | ldap | oauth2 | pam | denyall #type = denyall # Cache logins for until expiration time diff --git a/radicale/app/__init__.py b/radicale/app/__init__.py index bb418431..0b895820 100644 --- a/radicale/app/__init__.py +++ b/radicale/app/__init__.py @@ -275,7 +275,7 @@ class Application(ApplicationPartDelete, ApplicationPartHead, logger.debug("Called by reverse proxy, remove base prefix %r from path: %r => %r", base_prefix, path, path_new) path = path_new else: - if self._auth_type in ['remote_user', 'http_x_remote_user'] and self._web_type == 'internal': + if self._auth_type in ['remote_user', 'http_remote_user', 'http_x_remote_user'] and self._web_type == 'internal': logger.warning("Called by reverse proxy, cannot remove base prefix %r from path: %r as not matching (may cause authentication issues using internal WebUI)", base_prefix, path) else: logger.debug("Called by reverse proxy, cannot remove base prefix %r from path: %r as not matching", base_prefix, path) diff --git a/radicale/auth/__init__.py b/radicale/auth/__init__.py index c32a8306..7eadb18a 100644 --- a/radicale/auth/__init__.py +++ b/radicale/auth/__init__.py @@ -23,7 +23,7 @@ Authentication module. Authentication is based on usernames and passwords. If something more advanced is needed an external WSGI server or reverse proxy can be used -(see ``remote_user`` or ``http_x_remote_user`` backend). +(see ``remote_user``, ``http_remote_user`` or ``http_x_remote_user`` backend). Take a look at the class ``BaseAuth`` if you want to implement your own. @@ -40,6 +40,7 @@ from radicale import config, types, utils from radicale.log import logger INTERNAL_TYPES: Sequence[str] = ("none", "remote_user", "http_x_remote_user", + "http_remote_user", "denyall", "htpasswd", "ldap", @@ -59,6 +60,7 @@ CACHE_LOGIN_TYPES: Sequence[str] = ( INSECURE_IF_NO_LOOPBACK_TYPES: Sequence[str] = ( "remote_user", + "http_remote_user", "http_x_remote_user", ) diff --git a/radicale/auth/http_remote_user.py b/radicale/auth/http_remote_user.py new file mode 100644 index 00000000..40695f7b --- /dev/null +++ b/radicale/auth/http_remote_user.py @@ -0,0 +1,36 @@ +# This file is part of Radicale - CalDAV and CardDAV server +# Copyright © 2025-2025 Peter Bieringer +# +# This library is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This library is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with Radicale. If not, see . + +""" +Authentication backend that takes the username from the +``HTTP_REMOTE_USER`` header. + +It's intended for use with a reverse proxy. Be aware as this will be insecure +if the reverse proxy is not configured properly. + +""" + +from typing import Tuple, Union + +from radicale import types +from radicale.auth import none + + +class Auth(none.Auth): + + def get_external_login(self, environ: types.WSGIEnviron) -> Union[ + Tuple[()], Tuple[str, str]]: + return environ.get("HTTP_REMOTE_USER", ""), "" diff --git a/radicale/tests/test_auth.py b/radicale/tests/test_auth.py index 5ffc540d..86b61062 100644 --- a/radicale/tests/test_auth.py +++ b/radicale/tests/test_auth.py @@ -263,6 +263,23 @@ class TestBaseAuthRequests(BaseTest): href_element = prop.find(xmlutils.make_clark("D:href")) assert href_element is not None and href_element.text == "/test/" + def test_http_remote_user(self) -> None: + self.configure({"auth": {"type": "http_remote_user"}}) + _, responses = self.propfind("/", """\ + + + + + +""", HTTP_REMOTE_USER="test") + assert responses is not None + response = responses["/"] + assert not isinstance(response, int) + status, prop = response["D:current-user-principal"] + assert status == 200 + href_element = prop.find(xmlutils.make_clark("D:href")) + assert href_element is not None and href_element.text == "/test/" + def test_http_x_remote_user(self) -> None: self.configure({"auth": {"type": "http_x_remote_user"}}) _, responses = self.propfind("/", """\ From ae8211ad42c0badcb06dd1998556ed06c3dba3f6 Mon Sep 17 00:00:00 2001 From: Peter Bieringer Date: Sat, 15 Nov 2025 15:33:17 +0100 Subject: [PATCH 133/290] extend changelog for http_remote_user --- CHANGELOG.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index e2863fb3..5a1f0037 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,9 +1,10 @@ # Changelog ## 3.5.9.dev +* Extend: [auth] add support for type http_remote_use ## 3.5.8 -* Extend [auth]: re-factor & overhaul LDAP authentication, especially for Python's ldap module +* Extend: [auth] re-factor & overhaul LDAP authentication, especially for Python's ldap module * Fix: out-of-range timestamp on 32-bit systems * Feature: extend logging with response size in bytes and flag served as plain or gzip * Feature: [storage] strict_preconditions: new config option to enforce strict preconditions check on PUT in case item already exists [RFC6352#9.2] From 46dc5db61f6fe622c10f5d9b7cae91e044dd7779 Mon Sep 17 00:00:00 2001 From: Peter Bieringer Date: Sat, 15 Nov 2025 15:35:21 +0100 Subject: [PATCH 134/290] fix typo --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 5a1f0037..ff94443b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,7 +1,7 @@ # Changelog ## 3.5.9.dev -* Extend: [auth] add support for type http_remote_use +* Extend: [auth] add support for type http_remote_user ## 3.5.8 * Extend: [auth] re-factor & overhaul LDAP authentication, especially for Python's ldap module From 3f74ab1cd3e5d2d14beb2ccc3c6645079ff6ffda Mon Sep 17 00:00:00 2001 From: Peter Bieringer Date: Sun, 16 Nov 2025 15:46:50 +0100 Subject: [PATCH 135/290] do not log problematic sync token twice --- radicale/app/report.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/radicale/app/report.py b/radicale/app/report.py index b63681f7..023e6fe4 100644 --- a/radicale/app/report.py +++ b/radicale/app/report.py @@ -213,8 +213,8 @@ def xml_report(base_prefix: str, path: str, xml_request: Optional[ET.Element], sync_token, names = collection.sync(old_sync_token) except ValueError as e: # Invalid sync token - logger.warning("Client provided invalid sync token %r: %s", - old_sync_token, e, exc_info=True) + logger.warning("Client provided invalid sync token: %s", + e, exc_info=True) # client.CONFLICT doesn't work with some clients (e.g. InfCloud) return (client.FORBIDDEN, xmlutils.webdav_error("D:valid-sync-token")) From 4c0d216cb970865ce206a60a29f1eafb6d190d56 Mon Sep 17 00:00:00 2001 From: Peter Bieringer Date: Sun, 16 Nov 2025 15:50:05 +0100 Subject: [PATCH 136/290] log path on invalid sync token --- radicale/app/report.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/radicale/app/report.py b/radicale/app/report.py index 023e6fe4..42ce1ae0 100644 --- a/radicale/app/report.py +++ b/radicale/app/report.py @@ -213,8 +213,8 @@ def xml_report(base_prefix: str, path: str, xml_request: Optional[ET.Element], sync_token, names = collection.sync(old_sync_token) except ValueError as e: # Invalid sync token - logger.warning("Client provided invalid sync token: %s", - e, exc_info=True) + logger.warning("Client provided invalid sync token for path %r: %s", + path, e, exc_info=True) # client.CONFLICT doesn't work with some clients (e.g. InfCloud) return (client.FORBIDDEN, xmlutils.webdav_error("D:valid-sync-token")) From ed849a727136ea29269c5dfff65dce749dc8cee0 Mon Sep 17 00:00:00 2001 From: Peter Bieringer Date: Sun, 16 Nov 2025 16:06:26 +0100 Subject: [PATCH 137/290] add support for optional arguments --- radicale/tests/test_base.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/radicale/tests/test_base.py b/radicale/tests/test_base.py index c13c8725..944243e2 100644 --- a/radicale/tests/test_base.py +++ b/radicale/tests/test_base.py @@ -1687,7 +1687,7 @@ permissions: RrWw""") """, 400, is_xml=False) def _report_sync_token( - self, calendar_path: str, sync_token: Optional[str] = None + self, calendar_path: str, sync_token: Optional[str] = None, **kwargs ) -> Tuple[str, RESPONSES]: sync_token_xml = ( "" % sync_token @@ -1699,7 +1699,7 @@ permissions: RrWw""") %s -""" % sync_token_xml) +""" % sync_token_xml, **kwargs) xml = DefusedET.fromstring(answer) if status in (403, 409): assert xml.tag == xmlutils.make_clark("D:error") From f9697eeda1d1295aa36b7528638fdb6ecc121d7e Mon Sep 17 00:00:00 2001 From: Peter Bieringer Date: Sun, 16 Nov 2025 16:06:53 +0100 Subject: [PATCH 138/290] log user on invalid sync token --- radicale/app/report.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/radicale/app/report.py b/radicale/app/report.py index 42ce1ae0..e6dc45c0 100644 --- a/radicale/app/report.py +++ b/radicale/app/report.py @@ -149,7 +149,7 @@ def free_busy_report(base_prefix: str, path: str, xml_request: Optional[ET.Eleme def xml_report(base_prefix: str, path: str, xml_request: Optional[ET.Element], collection: storage.BaseCollection, encoding: str, unlock_storage_fn: Callable[[], None], - max_occurrence: int = 0, + max_occurrence: int = 0, user: str = "" ) -> Tuple[int, ET.Element]: """Read and answer REPORT requests that return XML. @@ -213,8 +213,8 @@ def xml_report(base_prefix: str, path: str, xml_request: Optional[ET.Element], sync_token, names = collection.sync(old_sync_token) except ValueError as e: # Invalid sync token - logger.warning("Client provided invalid sync token for path %r: %s", - path, e, exc_info=True) + logger.warning("Client provided invalid sync token for path %r (user %r): %s", + path, user, e, exc_info=True) # client.CONFLICT doesn't work with some clients (e.g. InfCloud) return (client.FORBIDDEN, xmlutils.webdav_error("D:valid-sync-token")) @@ -820,7 +820,7 @@ class ApplicationPartReport(ApplicationBase): try: status, xml_answer = xml_report( base_prefix, path, xml_content, collection, self._encoding, - lock_stack.close, max_occurrence) + lock_stack.close, max_occurrence, user) except ValueError as e: logger.warning( "Bad REPORT request on %r: %s", path, e, exc_info=True) From b2320c607d77faabc04be770e2c74998dce22bde Mon Sep 17 00:00:00 2001 From: Peter Bieringer Date: Sun, 16 Nov 2025 16:07:13 +0100 Subject: [PATCH 139/290] test case for invalid-sync token with user --- radicale/tests/test_base.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/radicale/tests/test_base.py b/radicale/tests/test_base.py index 944243e2..453bf045 100644 --- a/radicale/tests/test_base.py +++ b/radicale/tests/test_base.py @@ -1847,6 +1847,15 @@ permissions: RrWw""") calendar_path, "http://radicale.org/ns/sync/INVALID") assert not sync_token + def test_report_sync_collection_invalid_sync_token_with_user(self) -> None: + """Test sync-collection report with an invalid sync token and user+client""" + self.configure({"auth": {"type": "none"}}) + calendar_path = "/calendar.ics/" + self.mkcalendar(calendar_path) + sync_token, _ = self._report_sync_token( + calendar_path, "http://radicale.org/ns/sync/INVALID", login="testuser:") + assert not sync_token + def test_propfind_sync_token(self) -> None: """Retrieve the sync-token with a propfind request""" calendar_path = "/calendar.ics/" From 3fecba62e7f7c166a5dd0d6dc8e831385709bfc0 Mon Sep 17 00:00:00 2001 From: Peter Bieringer Date: Sun, 16 Nov 2025 16:22:20 +0100 Subject: [PATCH 140/290] fix comment --- CHANGELOG.md | 1 + radicale/tests/test_base.py | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index ff94443b..99cdc931 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,7 @@ ## 3.5.9.dev * Extend: [auth] add support for type http_remote_user +* Extend: logging of invalid sync-token with user and path ## 3.5.8 * Extend: [auth] re-factor & overhaul LDAP authentication, especially for Python's ldap module diff --git a/radicale/tests/test_base.py b/radicale/tests/test_base.py index 453bf045..5512ef9d 100644 --- a/radicale/tests/test_base.py +++ b/radicale/tests/test_base.py @@ -1848,7 +1848,7 @@ permissions: RrWw""") assert not sync_token def test_report_sync_collection_invalid_sync_token_with_user(self) -> None: - """Test sync-collection report with an invalid sync token and user+client""" + """Test sync-collection report with an invalid sync token and user""" self.configure({"auth": {"type": "none"}}) calendar_path = "/calendar.ics/" self.mkcalendar(calendar_path) From 4980172defef4977c9bcd48477aa32814ccf1b94 Mon Sep 17 00:00:00 2001 From: Peter Bieringer Date: Sun, 16 Nov 2025 16:57:31 +0100 Subject: [PATCH 141/290] extend copyright --- radicale/app/get.py | 3 ++- radicale/app/head.py | 3 ++- radicale/app/options.py | 3 ++- radicale/app/post.py | 5 +++-- radicale/app/propfind.py | 3 ++- radicale/tests/__init__.py | 3 ++- 6 files changed, 13 insertions(+), 7 deletions(-) diff --git a/radicale/app/get.py b/radicale/app/get.py index edd29b75..d170d380 100644 --- a/radicale/app/get.py +++ b/radicale/app/get.py @@ -2,7 +2,8 @@ # Copyright © 2008 Nicolas Kandel # Copyright © 2008 Pascal Halter # Copyright © 2008-2017 Guillaume Ayoub -# Copyright © 2017-2018 Unrud +# Copyright © 2017-2023 Unrud +# Copyright © 2025-2025 Peter Bieringer # # This library is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by diff --git a/radicale/app/head.py b/radicale/app/head.py index 5166db2d..718861e6 100644 --- a/radicale/app/head.py +++ b/radicale/app/head.py @@ -2,7 +2,8 @@ # Copyright © 2008 Nicolas Kandel # Copyright © 2008 Pascal Halter # Copyright © 2008-2017 Guillaume Ayoub -# Copyright © 2017-2018 Unrud +# Copyright © 2017-2022 Unrud +# Copyright © 2025-2025 Peter Bieringer # # This library is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by diff --git a/radicale/app/options.py b/radicale/app/options.py index 6e9053a3..159584bf 100644 --- a/radicale/app/options.py +++ b/radicale/app/options.py @@ -2,7 +2,8 @@ # Copyright © 2008 Nicolas Kandel # Copyright © 2008 Pascal Halter # Copyright © 2008-2017 Guillaume Ayoub -# Copyright © 2017-2018 Unrud +# Copyright © 2017-2021 Unrud +# Copyright © 2025-2025 Peter Bieringer # # This library is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by diff --git a/radicale/app/post.py b/radicale/app/post.py index f5367b86..1bb68a12 100644 --- a/radicale/app/post.py +++ b/radicale/app/post.py @@ -2,8 +2,9 @@ # Copyright © 2008 Nicolas Kandel # Copyright © 2008 Pascal Halter # Copyright © 2008-2017 Guillaume Ayoub -# Copyright © 2017-2018 Unrud -# Copyright © 2020 Tom Hacohen +# Copyright © 2017-2021 Unrud +# Copyright © 2020-2020 Tom Hacohen +# Copyright © 2025-2025 Peter Bieringer # # This library is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by diff --git a/radicale/app/propfind.py b/radicale/app/propfind.py index 6a3cea6d..2ba4b5d1 100644 --- a/radicale/app/propfind.py +++ b/radicale/app/propfind.py @@ -2,7 +2,8 @@ # Copyright © 2008 Nicolas Kandel # Copyright © 2008 Pascal Halter # Copyright © 2008-2017 Guillaume Ayoub -# Copyright © 2017-2018 Unrud +# Copyright © 2017-2021 Unrud +# Copyright © 2025-2025 Peter Bieringer # # This library is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by diff --git a/radicale/tests/__init__.py b/radicale/tests/__init__.py index c1a2aab2..73179efe 100644 --- a/radicale/tests/__init__.py +++ b/radicale/tests/__init__.py @@ -1,6 +1,7 @@ # This file is part of Radicale - CalDAV and CardDAV server # Copyright © 2012-2017 Guillaume Ayoub -# Copyright © 2017-2018 Unrud +# Copyright © 2017-2023 Unrud +# Copyright © 2024-2025 Peter Bieringer # # This library is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by From 03481a11840d57e6e8d089099b70a1bd5446c8b4 Mon Sep 17 00:00:00 2001 From: Peter Bieringer Date: Sun, 16 Nov 2025 17:02:04 +0100 Subject: [PATCH 142/290] add support for remote host+useragent --- radicale/tests/__init__.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/radicale/tests/__init__.py b/radicale/tests/__init__.py index 73179efe..9a57e6fd 100644 --- a/radicale/tests/__init__.py +++ b/radicale/tests/__init__.py @@ -80,6 +80,8 @@ class BaseTest: if http_if_match is not None and not isinstance(http_if_match, str): raise TypeError("http_if_match argument must be %r, not %r" % (str, type(http_if_match))) + remote_useragent = kwargs.pop("remote_useragent", None) + remote_host = kwargs.pop("remote_host", None) environ: Dict[str, Any] = {k.upper(): v for k, v in kwargs.items()} for k, v in environ.items(): if not isinstance(v, str): @@ -91,6 +93,10 @@ class BaseTest: login.encode(encoding)).decode() if http_if_match: environ["HTTP_IF_MATCH"] = http_if_match + if remote_useragent: + environ["HTTP_USER_AGENT"] = remote_useragent + if remote_host: + environ["REMOTE_ADDR"] = remote_host environ["REQUEST_METHOD"] = method.upper() environ["PATH_INFO"] = path if data is not None: From 91cee5e5146b43723bf547f959211a52b3df6419 Mon Sep 17 00:00:00 2001 From: Peter Bieringer Date: Sun, 16 Nov 2025 17:02:31 +0100 Subject: [PATCH 143/290] extend test with remote host+useragent --- radicale/tests/test_base.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/radicale/tests/test_base.py b/radicale/tests/test_base.py index 5512ef9d..8eff78d5 100644 --- a/radicale/tests/test_base.py +++ b/radicale/tests/test_base.py @@ -1848,12 +1848,12 @@ permissions: RrWw""") assert not sync_token def test_report_sync_collection_invalid_sync_token_with_user(self) -> None: - """Test sync-collection report with an invalid sync token and user""" + """Test sync-collection report with an invalid sync token and user+host+useragent""" self.configure({"auth": {"type": "none"}}) calendar_path = "/calendar.ics/" self.mkcalendar(calendar_path) sync_token, _ = self._report_sync_token( - calendar_path, "http://radicale.org/ns/sync/INVALID", login="testuser:") + calendar_path, "http://radicale.org/ns/sync/INVALID", login="testuser:", remote_host = "192.0.2.1", remote_useragent = "Testclient/1.0") assert not sync_token def test_propfind_sync_token(self) -> None: From 85f1850b198ec790bdbff1289a3d572fd772e225 Mon Sep 17 00:00:00 2001 From: Peter Bieringer Date: Sun, 16 Nov 2025 17:03:59 +0100 Subject: [PATCH 144/290] make flake8 happy --- radicale/tests/test_base.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/radicale/tests/test_base.py b/radicale/tests/test_base.py index 8eff78d5..0f33a4fc 100644 --- a/radicale/tests/test_base.py +++ b/radicale/tests/test_base.py @@ -1853,7 +1853,7 @@ permissions: RrWw""") calendar_path = "/calendar.ics/" self.mkcalendar(calendar_path) sync_token, _ = self._report_sync_token( - calendar_path, "http://radicale.org/ns/sync/INVALID", login="testuser:", remote_host = "192.0.2.1", remote_useragent = "Testclient/1.0") + calendar_path, "http://radicale.org/ns/sync/INVALID", login="testuser:", remote_host="192.0.2.1", remote_useragent="Testclient/1.0") assert not sync_token def test_propfind_sync_token(self) -> None: From 1c7ff414a96a6024616f1ece1d473bebd0455206 Mon Sep 17 00:00:00 2001 From: Peter Bieringer Date: Sun, 16 Nov 2025 17:06:00 +0100 Subject: [PATCH 145/290] extend app calls with remote host+useragent --- radicale/app/__init__.py | 2 +- radicale/app/delete.py | 2 +- radicale/app/get.py | 2 +- radicale/app/head.py | 4 ++-- radicale/app/mkcalendar.py | 2 +- radicale/app/mkcol.py | 2 +- radicale/app/move.py | 2 +- radicale/app/options.py | 2 +- radicale/app/post.py | 2 +- radicale/app/propfind.py | 2 +- radicale/app/proppatch.py | 2 +- radicale/app/put.py | 2 +- radicale/app/report.py | 6 +++--- 13 files changed, 16 insertions(+), 16 deletions(-) diff --git a/radicale/app/__init__.py b/radicale/app/__init__.py index 0b895820..940d15b5 100644 --- a/radicale/app/__init__.py +++ b/radicale/app/__init__.py @@ -371,7 +371,7 @@ class Application(ApplicationPartDelete, ApplicationPartHead, if not login or user: status, headers, answer = function( - environ, base_prefix, path, user) + environ, base_prefix, path, user, remote_host, remote_useragent) if (status, headers, answer) == httputils.NOT_ALLOWED: logger.info("Access to %r denied for %s", path, repr(user) if user else "anonymous user") diff --git a/radicale/app/delete.py b/radicale/app/delete.py index 060abb18..695de45f 100644 --- a/radicale/app/delete.py +++ b/radicale/app/delete.py @@ -55,7 +55,7 @@ def xml_delete(base_prefix: str, path: str, collection: storage.BaseCollection, class ApplicationPartDelete(ApplicationBase): def do_DELETE(self, environ: types.WSGIEnviron, base_prefix: str, - path: str, user: str) -> types.WSGIResponse: + path: str, user: str, remote_host: str, remote_useragent: str) -> types.WSGIResponse: """Manage DELETE request.""" access = Access(self._rights, user, path) if not access.check("w"): diff --git a/radicale/app/get.py b/radicale/app/get.py index d170d380..b8adb39a 100644 --- a/radicale/app/get.py +++ b/radicale/app/get.py @@ -59,7 +59,7 @@ class ApplicationPartGet(ApplicationBase): return value def do_GET(self, environ: types.WSGIEnviron, base_prefix: str, path: str, - user: str) -> types.WSGIResponse: + user: str, remote_host: str, remote_useragent: str) -> types.WSGIResponse: """Manage GET request.""" # Redirect to /.web if the root path is requested if not pathutils.strip_path(path): diff --git a/radicale/app/head.py b/radicale/app/head.py index 718861e6..eec68bb5 100644 --- a/radicale/app/head.py +++ b/radicale/app/head.py @@ -26,7 +26,7 @@ from radicale.app.get import ApplicationPartGet class ApplicationPartHead(ApplicationPartGet, ApplicationBase): def do_HEAD(self, environ: types.WSGIEnviron, base_prefix: str, path: str, - user: str) -> types.WSGIResponse: + user: str, remote_host: str, remote_useragent: str) -> types.WSGIResponse: """Manage HEAD request.""" # Body is dropped in `Application.__call__` for HEAD requests - return self.do_GET(environ, base_prefix, path, user) + return self.do_GET(environ, base_prefix, path, user, remote_host, remote_useragent) diff --git a/radicale/app/mkcalendar.py b/radicale/app/mkcalendar.py index 632d3c38..db14bfdc 100644 --- a/radicale/app/mkcalendar.py +++ b/radicale/app/mkcalendar.py @@ -33,7 +33,7 @@ from radicale.log import logger class ApplicationPartMkcalendar(ApplicationBase): def do_MKCALENDAR(self, environ: types.WSGIEnviron, base_prefix: str, - path: str, user: str) -> types.WSGIResponse: + path: str, user: str, remote_host: str, remote_useragent: str) -> types.WSGIResponse: """Manage MKCALENDAR request.""" if "w" not in self._rights.authorization(user, path): return httputils.NOT_ALLOWED diff --git a/radicale/app/mkcol.py b/radicale/app/mkcol.py index 169cb62c..72d5aa2b 100644 --- a/radicale/app/mkcol.py +++ b/radicale/app/mkcol.py @@ -33,7 +33,7 @@ from radicale.log import logger class ApplicationPartMkcol(ApplicationBase): def do_MKCOL(self, environ: types.WSGIEnviron, base_prefix: str, - path: str, user: str) -> types.WSGIResponse: + path: str, user: str, remote_host: str, remote_useragent: str) -> types.WSGIResponse: """Manage MKCOL request.""" permissions = self._rights.authorization(user, path) if not rights.intersect(permissions, "Ww"): diff --git a/radicale/app/move.py b/radicale/app/move.py index 77e56f3e..ba346762 100644 --- a/radicale/app/move.py +++ b/radicale/app/move.py @@ -48,7 +48,7 @@ def get_server_netloc(environ: types.WSGIEnviron, force_port: bool = False): class ApplicationPartMove(ApplicationBase): def do_MOVE(self, environ: types.WSGIEnviron, base_prefix: str, - path: str, user: str) -> types.WSGIResponse: + path: str, user: str, remote_host: str, remote_useragent: str) -> types.WSGIResponse: """Manage MOVE request.""" raw_dest = environ.get("HTTP_DESTINATION", "") to_url = urlparse(raw_dest) diff --git a/radicale/app/options.py b/radicale/app/options.py index 159584bf..a869e2ab 100644 --- a/radicale/app/options.py +++ b/radicale/app/options.py @@ -27,7 +27,7 @@ from radicale.app.base import ApplicationBase class ApplicationPartOptions(ApplicationBase): def do_OPTIONS(self, environ: types.WSGIEnviron, base_prefix: str, - path: str, user: str) -> types.WSGIResponse: + path: str, user: str, remote_host: str, remote_useragent: str) -> types.WSGIResponse: """Manage OPTIONS request.""" headers = { "Allow": ", ".join( diff --git a/radicale/app/post.py b/radicale/app/post.py index 1bb68a12..df944499 100644 --- a/radicale/app/post.py +++ b/radicale/app/post.py @@ -26,7 +26,7 @@ from radicale.app.base import ApplicationBase class ApplicationPartPost(ApplicationBase): def do_POST(self, environ: types.WSGIEnviron, base_prefix: str, - path: str, user: str) -> types.WSGIResponse: + path: str, user: str, remote_host: str, remote_useragent: str) -> types.WSGIResponse: """Manage POST request.""" if path == "/.web" or path.startswith("/.web/"): return self._web.post(environ, base_prefix, path, user) diff --git a/radicale/app/propfind.py b/radicale/app/propfind.py index 2ba4b5d1..b546c5e1 100644 --- a/radicale/app/propfind.py +++ b/radicale/app/propfind.py @@ -377,7 +377,7 @@ class ApplicationPartPropfind(ApplicationBase): yield item, permission def do_PROPFIND(self, environ: types.WSGIEnviron, base_prefix: str, - path: str, user: str) -> types.WSGIResponse: + path: str, user: str, remote_host: str, remote_useragent: str) -> types.WSGIResponse: """Manage PROPFIND request.""" access = Access(self._rights, user, path) if not access.check("r"): diff --git a/radicale/app/proppatch.py b/radicale/app/proppatch.py index 2e8eed47..9d6dc221 100644 --- a/radicale/app/proppatch.py +++ b/radicale/app/proppatch.py @@ -73,7 +73,7 @@ def xml_proppatch(base_prefix: str, path: str, class ApplicationPartProppatch(ApplicationBase): def do_PROPPATCH(self, environ: types.WSGIEnviron, base_prefix: str, - path: str, user: str) -> types.WSGIResponse: + path: str, user: str, remote_host: str, remote_useragent: str) -> types.WSGIResponse: """Manage PROPPATCH request.""" access = Access(self._rights, user, path) if not access.check("w"): diff --git a/radicale/app/put.py b/radicale/app/put.py index 6cfed1eb..de11589b 100644 --- a/radicale/app/put.py +++ b/radicale/app/put.py @@ -142,7 +142,7 @@ def prepare(vobject_items: List[vobject.base.Component], path: str, class ApplicationPartPut(ApplicationBase): def do_PUT(self, environ: types.WSGIEnviron, base_prefix: str, - path: str, user: str) -> types.WSGIResponse: + path: str, user: str, remote_host: str, remote_useragent: str) -> types.WSGIResponse: """Manage PUT request.""" access = Access(self._rights, user, path) if not access.check("w"): diff --git a/radicale/app/report.py b/radicale/app/report.py index e6dc45c0..dd59c373 100644 --- a/radicale/app/report.py +++ b/radicale/app/report.py @@ -149,7 +149,7 @@ def free_busy_report(base_prefix: str, path: str, xml_request: Optional[ET.Eleme def xml_report(base_prefix: str, path: str, xml_request: Optional[ET.Element], collection: storage.BaseCollection, encoding: str, unlock_storage_fn: Callable[[], None], - max_occurrence: int = 0, user: str = "" + max_occurrence: int = 0, user: str = "", remote_addr: str = "", remote_useragent: str = "" ) -> Tuple[int, ET.Element]: """Read and answer REPORT requests that return XML. @@ -776,7 +776,7 @@ def test_filter(collection_tag: str, item: radicale_item.Item, class ApplicationPartReport(ApplicationBase): def do_REPORT(self, environ: types.WSGIEnviron, base_prefix: str, - path: str, user: str) -> types.WSGIResponse: + path: str, user: str, remote_host: str, remote_useragent: str) -> types.WSGIResponse: """Manage REPORT request.""" access = Access(self._rights, user, path) if not access.check("r"): @@ -820,7 +820,7 @@ class ApplicationPartReport(ApplicationBase): try: status, xml_answer = xml_report( base_prefix, path, xml_content, collection, self._encoding, - lock_stack.close, max_occurrence, user) + lock_stack.close, max_occurrence, user, remote_host, remote_useragent) except ValueError as e: logger.warning( "Bad REPORT request on %r: %s", path, e, exc_info=True) From 7ebe5703acf07714ba6a5f51ef888feb53615449 Mon Sep 17 00:00:00 2001 From: Peter Bieringer Date: Sun, 16 Nov 2025 17:06:21 +0100 Subject: [PATCH 146/290] log remote host+useragent on invalid sync token --- radicale/app/report.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/radicale/app/report.py b/radicale/app/report.py index dd59c373..dda4ba10 100644 --- a/radicale/app/report.py +++ b/radicale/app/report.py @@ -213,8 +213,8 @@ def xml_report(base_prefix: str, path: str, xml_request: Optional[ET.Element], sync_token, names = collection.sync(old_sync_token) except ValueError as e: # Invalid sync token - logger.warning("Client provided invalid sync token for path %r (user %r): %s", - path, user, e, exc_info=True) + logger.warning("Client provided invalid sync token for path %r (user %r from %s%s): %s", + path, user, remote_addr, remote_useragent, e, exc_info=True) # client.CONFLICT doesn't work with some clients (e.g. InfCloud) return (client.FORBIDDEN, xmlutils.webdav_error("D:valid-sync-token")) From fb991798e843600cf52bbd2d186f8658d1c36fe5 Mon Sep 17 00:00:00 2001 From: Peter Bieringer Date: Sun, 16 Nov 2025 17:10:45 +0100 Subject: [PATCH 147/290] extend changelog --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 99cdc931..25129ef1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,7 +2,7 @@ ## 3.5.9.dev * Extend: [auth] add support for type http_remote_user -* Extend: logging of invalid sync-token with user and path +* Extend: logging of invalid sync-token with user, path, remote host and useragent ## 3.5.8 * Extend: [auth] re-factor & overhaul LDAP authentication, especially for Python's ldap module From 62166eb2cce771f1a29c875913717641b0cb7956 Mon Sep 17 00:00:00 2001 From: Peter Bieringer Date: Tue, 25 Nov 2025 20:02:50 +0100 Subject: [PATCH 148/290] fix typo related to collection delete hook, fixes https://github.com/Kozea/Radicale/issues/1920 --- radicale/app/delete.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/radicale/app/delete.py b/radicale/app/delete.py index 695de45f..61fe6c43 100644 --- a/radicale/app/delete.py +++ b/radicale/app/delete.py @@ -87,7 +87,7 @@ class ApplicationPartDelete(ApplicationBase): path=access.path, content=i.uid, uid=i.uid, - old_content=item.serialize(), # type: ignore + old_content=i.serialize(), # type: ignore new_content=None ) ) From 62353b42ee0ef556815c086a025dc2f0a2fedbc0 Mon Sep 17 00:00:00 2001 From: Peter Bieringer Date: Tue, 25 Nov 2025 20:03:27 +0100 Subject: [PATCH 149/290] changelog for: fix typo related to collection delete hook --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 25129ef1..9d5cca20 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,6 +3,7 @@ ## 3.5.9.dev * Extend: [auth] add support for type http_remote_user * Extend: logging of invalid sync-token with user, path, remote host and useragent +* Fix: typo related to collection delete hook ## 3.5.8 * Extend: [auth] re-factor & overhaul LDAP authentication, especially for Python's ldap module From ebe95ec2ffcf466996011ece2c2726435a467fcc Mon Sep 17 00:00:00 2001 From: Peter Bieringer Date: Sat, 29 Nov 2025 15:33:06 +0100 Subject: [PATCH 150/290] Release 3.5.9 --- CHANGELOG.md | 2 +- pyproject.toml | 2 +- setup.py.legacy | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 9d5cca20..4b255ddd 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,6 @@ # Changelog -## 3.5.9.dev +## 3.5.9 * Extend: [auth] add support for type http_remote_user * Extend: logging of invalid sync-token with user, path, remote host and useragent * Fix: typo related to collection delete hook diff --git a/pyproject.toml b/pyproject.toml index 034e5446..1e45143f 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -3,7 +3,7 @@ name = "Radicale" # When the version is updated, a new section in the CHANGELOG.md file must be # added too. readme = "README.md" -version = "3.5.9.dev" +version = "3.5.9" authors = [{name = "Guillaume Ayoub", email = "guillaume.ayoub@kozea.fr"}, {name = "Unrud", email = "unrud@outlook.com"}, {name = "Peter Bieringer", email = "pb@bieringer.de"}] license = {text = "GNU GPL v3"} description = "CalDAV and CardDAV Server" diff --git a/setup.py.legacy b/setup.py.legacy index 83545da7..f2e7e429 100644 --- a/setup.py.legacy +++ b/setup.py.legacy @@ -20,7 +20,7 @@ from setuptools import find_packages, setup # When the version is updated, a new section in the CHANGELOG.md file must be # added too. -VERSION = "3.5.9.dev" +VERSION = "3.5.9" with open("README.md", encoding="utf-8") as f: long_description = f.read() From 6c496b8021ba83ea645757d639bd4e2f19177f7c Mon Sep 17 00:00:00 2001 From: Peter Bieringer Date: Mon, 1 Dec 2025 18:47:00 +0100 Subject: [PATCH 151/290] 3.5.10.dev --- CHANGELOG.md | 2 ++ pyproject.toml | 2 +- setup.py.legacy | 2 +- 3 files changed, 4 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 4b255ddd..7be52c88 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,7 @@ # Changelog +## 3.5.10.dev + ## 3.5.9 * Extend: [auth] add support for type http_remote_user * Extend: logging of invalid sync-token with user, path, remote host and useragent diff --git a/pyproject.toml b/pyproject.toml index 1e45143f..688ce0a6 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -3,7 +3,7 @@ name = "Radicale" # When the version is updated, a new section in the CHANGELOG.md file must be # added too. readme = "README.md" -version = "3.5.9" +version = "3.5.10.dev" authors = [{name = "Guillaume Ayoub", email = "guillaume.ayoub@kozea.fr"}, {name = "Unrud", email = "unrud@outlook.com"}, {name = "Peter Bieringer", email = "pb@bieringer.de"}] license = {text = "GNU GPL v3"} description = "CalDAV and CardDAV Server" diff --git a/setup.py.legacy b/setup.py.legacy index f2e7e429..0797fdd0 100644 --- a/setup.py.legacy +++ b/setup.py.legacy @@ -20,7 +20,7 @@ from setuptools import find_packages, setup # When the version is updated, a new section in the CHANGELOG.md file must be # added too. -VERSION = "3.5.9" +VERSION = "3.5.10.dev" with open("README.md", encoding="utf-8") as f: long_description = f.read() From 5b2cb5c0bfe22398c48aec7d0941b26e90539d18 Mon Sep 17 00:00:00 2001 From: Peter Bieringer Date: Mon, 1 Dec 2025 18:50:46 +0100 Subject: [PATCH 152/290] add new test cases for broken items --- radicale/tests/static/broken-vcards.vcf | 16 +++++++ .../tests/static/broken-vcards2-no_uid.vcf | 16 +++++++ radicale/tests/static/broken-vcards2.vcf | 17 ++++++++ radicale/tests/static/broken-vevents.ics | 25 +++++++++++ radicale/tests/test_base.py | 42 +++++++++++++++++++ 5 files changed, 116 insertions(+) create mode 100644 radicale/tests/static/broken-vcards.vcf create mode 100644 radicale/tests/static/broken-vcards2-no_uid.vcf create mode 100644 radicale/tests/static/broken-vcards2.vcf create mode 100644 radicale/tests/static/broken-vevents.ics diff --git a/radicale/tests/static/broken-vcards.vcf b/radicale/tests/static/broken-vcards.vcf new file mode 100644 index 00000000..19bd670b --- /dev/null +++ b/radicale/tests/static/broken-vcards.vcf @@ -0,0 +1,16 @@ +BEGIN:VCARD +VERSION:3.0 +PRODID:-//Inverse inc.//SOGo Connector 1.0//EN +UID:C68582D2-2E60-0001-C2C0-000000000000.vcf +X-MOZILLA-HTML:FALSE +EMAIL;TYPE=work:test-misses-N-or-FN@example.com +X-RADICALE-NAME:C68582D2-2E60-0001-C2C0-000000000000.vcf +END:VCARD +BEGIN:VCARD +VERSION:3.0 +PRODID:-//Inverse inc.//SOGo Connector 1.0//EN +UID:C68582D2-2E60-0001-C2C0-000000000001.vcf +X-MOZILLA-HTML:FALSE +EMAIL;TYPE=work:test-misses-N-or-FN@example1.com +X-RADICALE-NAME:C68582D2-2E60-0001-C2C0-000000000001.vcf +END:VCARD diff --git a/radicale/tests/static/broken-vcards2-no_uid.vcf b/radicale/tests/static/broken-vcards2-no_uid.vcf new file mode 100644 index 00000000..76a3dfb6 --- /dev/null +++ b/radicale/tests/static/broken-vcards2-no_uid.vcf @@ -0,0 +1,16 @@ +BEGIN:VCARD +VERSION:3.0 +PRODID:-//Inverse inc.//SOGo Connector 1.0//EN +UID:C68582D2-2E60-0001-C2C0-000000000000.vcf +X-MOZILLA-HTML:FALSE +EMAIL;TYPE=work:test-misses-N-or-FN@example.com +FN:Test Example +X-RADICALE-NAME:C68582D2-2E60-0001-C2C0-000000000000.vcf +END:VCARD +BEGIN:VCARD +VERSION:3.0 +PRODID:-//Inverse inc.//SOGo Connector 1.0//EN +X-MOZILLA-HTML:FALSE +EMAIL;TYPE=work:test-misses-N-or-FN@example1.com +X-RADICALE-NAME:C68582D2-2E60-0001-C2C0-000000000001.vcf +END:VCARD diff --git a/radicale/tests/static/broken-vcards2.vcf b/radicale/tests/static/broken-vcards2.vcf new file mode 100644 index 00000000..ca5fa3ac --- /dev/null +++ b/radicale/tests/static/broken-vcards2.vcf @@ -0,0 +1,17 @@ +BEGIN:VCARD +VERSION:3.0 +PRODID:-//Inverse inc.//SOGo Connector 1.0//EN +UID:C68582D2-2E60-0001-C2C0-000000000000.vcf +X-MOZILLA-HTML:FALSE +EMAIL;TYPE=work:test-misses-N-or-FN@example.com +FN:Test Example +X-RADICALE-NAME:C68582D2-2E60-0001-C2C0-000000000000.vcf +END:VCARD +BEGIN:VCARD +VERSION:3.0 +PRODID:-//Inverse inc.//SOGo Connector 1.0//EN +UID:C68582D2-2E60-0001-C2C0-000000000001.vcf +X-MOZILLA-HTML:FALSE +EMAIL;TYPE=work:test-misses-N-or-FN@example1.com +X-RADICALE-NAME:C68582D2-2E60-0001-C2C0-000000000001.vcf +END:VCARD diff --git a/radicale/tests/static/broken-vevents.ics b/radicale/tests/static/broken-vevents.ics new file mode 100644 index 00000000..bbfbcd0b --- /dev/null +++ b/radicale/tests/static/broken-vevents.ics @@ -0,0 +1,25 @@ +BEGIN:VCALENDAR +PRODID:-//Radicale//NONSGML Radicale Server//EN +VERSION:2.0 +BEGIN:VEVENT +CREATED:20160725T060147Z +LAST-MODIFIED:20160727T193435Z +DTSTAMP:20160727T193435Z +UID:040000008200E00074C5B7101A82E00800000000 +SUMMARY:Good ICS +STATUS:CONFIRMED +X-MOZ-LASTACK:20160727T193435Z +DTSTART;TZID=Europe/Budapest:20160727T170000 +DTEND;TZID=Europe/Budapest:20160727T223000 +CLASS:PUBLIC +X-LIC-ERROR:No value for LOCATION property. Removing entire property: +END:VEVENT +BEGIN:VEVENT +CREATED:20160725T060147Z +LAST-MODIFIED:20160727T193435Z +DTSTAMP:20160727T193435Z +UID:040000008200E00074C5B7101A82E00800000001 +CLASS:PUBLIC +X-LIC-ERROR:No value for LOCATION property. Removing entire property: +END:VEVENT +END:VCALENDAR diff --git a/radicale/tests/test_base.py b/radicale/tests/test_base.py index 0f33a4fc..afcca1b2 100644 --- a/radicale/tests/test_base.py +++ b/radicale/tests/test_base.py @@ -142,6 +142,20 @@ permissions: RrWw""") assert "Event" in answer assert "UID:event" in answer + def test_add_event_broken(self) -> None: + """Add a broken event.""" + self.mkcalendar("/calendar.ics/") + event = get_file_content("broken-vevent.ics") + path = "/calendar.ics/broken-vevent.ics" + self.put(path, event, check=400) + + def test_add_events_broken2(self) -> None: + """Add a broken event (2nd one is broken).""" + self.mkcalendar("/calendar.ics/") + event = get_file_content("broken-vevents.ics") + path = "/calendar.ics/" + self.put(path, event, check=400) + def test_add_event_without_uid(self) -> None: """Add an event without UID.""" self.mkcalendar("/calendar.ics/") @@ -201,6 +215,34 @@ permissions: RrWw""") _, answer = self.get(path) assert "UID:contact1" in answer + def test_add_contact_broken(self) -> None: + """Add a broken contact.""" + self.create_addressbook("/contacts.vcf/") + contact = get_file_content("broken-vcard.vcf") + path = "/contacts.vcf/broken-vcards.vcf" + self.put(path, contact, check=400) + + def test_add_contacts_broken(self) -> None: + """Add broken contacts.""" + self.create_addressbook("/contacts.vcf/") + contact = get_file_content("broken-vcards.vcf") + path = "/contacts.vcf/" + self.put(path, contact, check=400) + + def test_add_contacts_broken2(self) -> None: + """Add broken contacts (only 2nd one is broken).""" + self.create_addressbook("/contacts.vcf/") + contact = get_file_content("broken-vcards2.vcf") + path = "/contacts.vcf/" + self.put(path, contact, check=400) + + def test_add_contacts_broken2_no_uid(self) -> None: + """Add broken contacts (only 2nd one is broken and has no UID).""" + self.create_addressbook("/contacts.vcf/") + contact = get_file_content("broken-vcards2-no_uid.vcf") + path = "/contacts.vcf/" + self.put(path, contact, check=400) + def test_add_contact_photo_with_data_uri(self) -> None: """Test workaround for broken PHOTO data from InfCloud""" self.create_addressbook("/contacts.vcf/") From 56e0362f59efcdaf4224542933e380c952d98636 Mon Sep 17 00:00:00 2001 From: Peter Bieringer Date: Mon, 1 Dec 2025 18:51:17 +0100 Subject: [PATCH 153/290] extend broken vcalendar item log --- radicale/app/put.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/radicale/app/put.py b/radicale/app/put.py index de11589b..0a72d00a 100644 --- a/radicale/app/put.py +++ b/radicale/app/put.py @@ -93,9 +93,13 @@ def prepare(vobject_items: List[vobject.base.Component], path: str, logger.debug("Prepare item with UID '%s'", item.uid) try: item.prepare() - except ValueError as e: + except (RuntimeError, ValueError, AttributeError) as e: if logger.isEnabledFor(logging.DEBUG): - logger.warning("Problem during prepare item with UID '%s' (content below): %s\n%s", item.uid, e, item._text) + if item._text is None: + content = vobject_item + else: + content = item._text + logger.warning("Problem during prepare item with UID '%s' (content below): %s\n%s", item.uid, e, content) else: logger.warning("Problem during prepare item with UID '%s' (content suppressed in this loglevel): %s", item.uid, e) raise From da612cfccb57c3e4b5677c88f817dc02148ed723 Mon Sep 17 00:00:00 2001 From: Peter Bieringer Date: Mon, 1 Dec 2025 18:51:41 +0100 Subject: [PATCH 154/290] add broken vaddressbook logging similar to vcalendar --- radicale/app/put.py | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/radicale/app/put.py b/radicale/app/put.py index 0a72d00a..daec6339 100644 --- a/radicale/app/put.py +++ b/radicale/app/put.py @@ -108,7 +108,19 @@ def prepare(vobject_items: List[vobject.base.Component], path: str, for vobject_item in vobject_items: item = radicale_item.Item(collection_path=collection_path, vobject_item=vobject_item) - item.prepare() + logger.debug("Prepare item with UID '%s'", item.uid) + try: + item.prepare() + except (RuntimeError, ValueError, AttributeError) as e: + if logger.isEnabledFor(logging.DEBUG): + if item._text is None: + content = vobject_item + else: + content = item._text + logger.warning("Problem during prepare item with UID '%s' (content below): %s\n%s", item.uid, e, content) + else: + logger.warning("Problem during prepare item with UID '%s' (content suppressed in this loglevel): %s", item.uid, e) + raise items.append(item) elif not write_whole_collection: vobject_item, = vobject_items From 9b43b8e3de4d310325ebffe5a7e0a8459dc7b520 Mon Sep 17 00:00:00 2001 From: Peter Bieringer Date: Mon, 1 Dec 2025 18:52:05 +0100 Subject: [PATCH 155/290] catch broken event --- radicale/item/filter.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/radicale/item/filter.py b/radicale/item/filter.py index 94cdc015..a1329988 100644 --- a/radicale/item/filter.py +++ b/radicale/item/filter.py @@ -354,7 +354,10 @@ def visit_time_ranges(vobject_item: vobject.base.Component, child_name: str, for child, is_recurrence, recurrences in get_children( vobject_item.vevent_list): # TODO: check if there's a timezone - dtstart = child.dtstart.value + try: + dtstart = child.dtstart.value + except AttributeError: + raise AttributeError("missing DTSTART") if child.rruleset: dtstarts, infinity = getrruleset(child, recurrences) From 5fa370fa1ced5e8b28377e2c52d7cd432de0137e Mon Sep 17 00:00:00 2001 From: Peter Bieringer Date: Mon, 1 Dec 2025 18:59:20 +0100 Subject: [PATCH 156/290] extend changelog --- CHANGELOG.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 7be52c88..8d36b54d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,8 @@ # Changelog ## 3.5.10.dev +* Improve: logging of broken calendar items during PUT +* Add: logging of broken contact items during PUT ## 3.5.9 * Extend: [auth] add support for type http_remote_user From f3f73ece15dba3c36428404c7271a4eea246006c Mon Sep 17 00:00:00 2001 From: Peter Bieringer Date: Wed, 3 Dec 2025 08:24:25 +0100 Subject: [PATCH 157/290] reactivate IMAP AUTH=LOGIN as fallback replaced by 25402ab641246fedad7be1b7f1b38731a89356ed supporting https://github.com/Kozea/Radicale/issues/1929 --- radicale/auth/imap.py | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/radicale/auth/imap.py b/radicale/auth/imap.py index 18ec527b..f0d52b47 100644 --- a/radicale/auth/imap.py +++ b/radicale/auth/imap.py @@ -64,10 +64,18 @@ class Auth(auth.BaseAuth): if self._security == "starttls": connection.starttls(ssl.create_default_context()) try: - connection.authenticate( - "PLAIN", - lambda _: "{0}\x00{0}\x00{1}".format(login, password).encode(), - ) + if "AUTH=PLAIN" in connection.capabilities: + logger.debug("IMAP authentication PLAIN selected for user %r via %s:%d (security: %s)", login, self._host, self._port, self._security) + connection.authenticate( + "PLAIN", + lambda _: "{0}\x00{0}\x00{1}".format(login, password).encode(), + ) + elif "AUTH=LOGIN" in connection.capabilities: + logger.debug("IMAP authentication LOGIN selected for user %r via %s:%d (security: %s)", login, self._host, self._port, self._security) + connection.login(login, password) + else: + logger.error("IMAP server is neither supporting AUTH=PLAIN or AUTH=LOGIN: %s:%d (security: %s)", self._host, self._port, self._security) + return "" except imaplib.IMAP4.error as e: logger.warning("IMAP authentication failed for user %r: %s", login, e, exc_info=False) return "" From 0f664dcdb1b5d7503f2318ef7676c871e977b5a0 Mon Sep 17 00:00:00 2001 From: Peter Bieringer Date: Wed, 3 Dec 2025 08:31:52 +0100 Subject: [PATCH 158/290] Changelog for IMAP/LOGIN fallback --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 8d36b54d..1338b2bf 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,6 +3,7 @@ ## 3.5.10.dev * Improve: logging of broken calendar items during PUT * Add: logging of broken contact items during PUT +* Extend: [auth] imap: add fallback support for LOGIN towards remote IMAP server (replaced in 3.5.0) ## 3.5.9 * Extend: [auth] add support for type http_remote_user From 3abab9cd8e6c55b7a1d8deda63190b40c8432147 Mon Sep 17 00:00:00 2001 From: Peter Bieringer Date: Fri, 5 Dec 2025 16:24:19 +0100 Subject: [PATCH 159/290] move: fix detection of HTTP_X_FORWARDED_PORT --- radicale/app/move.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/radicale/app/move.py b/radicale/app/move.py index ba346762..ac4fcab4 100644 --- a/radicale/app/move.py +++ b/radicale/app/move.py @@ -34,7 +34,7 @@ def get_server_netloc(environ: types.WSGIEnviron, force_port: bool = False): host = environ["HTTP_X_FORWARDED_HOST"] proto = environ.get("HTTP_X_FORWARDED_PROTO") or "http" port = "443" if proto == "https" else "80" - port = environ["HTTP_X_FORWARDED_PORT"] or port + port = environ.get("HTTP_X_FORWARDED_PORT") or port else: host = environ.get("HTTP_HOST") or environ["SERVER_NAME"] proto = environ["wsgi.url_scheme"] From dd471d59f73e40e8101db733cdb7d8585d3ed975 Mon Sep 17 00:00:00 2001 From: Peter Bieringer Date: Fri, 5 Dec 2025 16:25:32 +0100 Subject: [PATCH 160/290] changelog for 3abab9cd8e6c55b7a1d8deda63190b40c8432147 --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 1338b2bf..920df43b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,7 @@ * Improve: logging of broken calendar items during PUT * Add: logging of broken contact items during PUT * Extend: [auth] imap: add fallback support for LOGIN towards remote IMAP server (replaced in 3.5.0) +* Fix: improper detection of HTTP_X_FORWARDED_PORT on MOVE ## 3.5.9 * Extend: [auth] add support for type http_remote_user From 3c0267c98a21175f4fc8edea0ff8be391e3a1c2b Mon Sep 17 00:00:00 2001 From: Peter Bieringer Date: Mon, 8 Dec 2025 12:29:43 +0100 Subject: [PATCH 161/290] profiling: add options --- radicale/config.py | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/radicale/config.py b/radicale/config.py index a4ba6610..5ec5908f 100644 --- a/radicale/config.py +++ b/radicale/config.py @@ -41,6 +41,8 @@ from radicale import auth, hook, rights, storage, types, web from radicale.hook import email from radicale.item import check_and_sanitize_props +from radicale import app # isort:skip (circular import issue) + DEFAULT_CONFIG_PATH: str = os.pathsep.join([ "?/etc/radicale/config", "?~/.config/radicale/config"]) @@ -581,6 +583,23 @@ This is an automated message. Please do not reply.""", "value": "False", "help": "log storage cache action on level=debug", "type": bool}), + ("profiling", { + "value": "per_request_method", + "help": "log profiling data level=info", + "type": str, + "internal": app.PROFILING}), + ("profiling_per_request_min_duration", { + "value": "3", + "help": "log profiling data per request minimum duration (seconds)", + "type": int}), + ("profiling_per_request_method_interval", { + "value": "600", + "help": "log profiling data per request method interval (seconds)", + "type": int}), + ("profiling_top_x_functions", { + "value": "10", + "help": "log profiling top X functions (limit)", + "type": int}), ("mask_passwords", { "value": "True", "help": "mask passwords in logs", From edc88fbed8463eae87d8a2a6e45b08326dc86ffc Mon Sep 17 00:00:00 2001 From: Peter Bieringer Date: Mon, 8 Dec 2025 12:31:09 +0100 Subject: [PATCH 162/290] profiling: add option default --- config | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/config b/config index 93291650..46b2f689 100644 --- a/config +++ b/config @@ -321,6 +321,20 @@ # Log storage cache actions on level=debug #storage_cache_actions_on_debug = False +# Log profiling data on level=info +# Value: per_request | per_request_method +#profiling = per_request_method + +# Log profiling data per request minium duration (seconds) +#profiling_per_request_min_duration = 3 + +# Log profiling data per request method interval +#profiling_per_request_method_interval = 600 + +# Log profiling top X functions (limit) +#profiling_top_x_functions = 10 + + [headers] # Additional HTTP headers From 462fa931846fa4bbda36935ab406fc37e95a4faf Mon Sep 17 00:00:00 2001 From: Peter Bieringer Date: Mon, 8 Dec 2025 12:31:49 +0100 Subject: [PATCH 163/290] profiling: document new options --- DOCUMENTATION.md | 37 +++++++++++++++++++++++++++++++++++++ 1 file changed, 37 insertions(+) diff --git a/DOCUMENTATION.md b/DOCUMENTATION.md index 855c671a..420345b6 100644 --- a/DOCUMENTATION.md +++ b/DOCUMENTATION.md @@ -1690,6 +1690,43 @@ Log storage cache actions on `level = debug` Default: `False` +##### profiling_per_request + +_(>= 3.5.10)_ + +Log profiling data on level=info + +Default: `per_request` + +One of +* `per_request` (above minimum duration) +* `per_request_method` (regular interval) + +##### profiling_per_request_min_duration + +_(>= 3.5.10)_ + +Log profiling data per request minimum duration (seconds) before logging, otherwise skip + +Default: `3` + +##### profiling_per_request_method_interval + +_(>= 3.5.10)_ + +Log profiling data per method interval (seconds) +Triggered by request, not active on idle systems + +Default: `600` + +##### profiling_top_x_functions + +_(>= 3.5.10)_ + +Log profiling top X functions (limit) + +Default: `10` + #### [headers] This section can be used to specify additional HTTP headers that will be sent to clients. From 3f4d43443927794499f3301f443df6142fa45e1d Mon Sep 17 00:00:00 2001 From: Peter Bieringer Date: Mon, 8 Dec 2025 12:33:17 +0100 Subject: [PATCH 164/290] profiling: add support --- radicale/app/__init__.py | 102 ++++++++++++++++++++++++++++++++++++++- 1 file changed, 100 insertions(+), 2 deletions(-) diff --git a/radicale/app/__init__.py b/radicale/app/__init__.py index 940d15b5..882787ec 100644 --- a/radicale/app/__init__.py +++ b/radicale/app/__init__.py @@ -27,13 +27,16 @@ the built-in server (see ``radicale.server`` module). """ import base64 +import cProfile import datetime +import io import pprint +import pstats import random import time import zlib from http import client -from typing import Iterable, List, Mapping, Tuple, Union +from typing import Iterable, List, Mapping, Sequence, Tuple, Union from radicale import config, httputils, log, pathutils, types from radicale.app.base import ApplicationBase @@ -55,6 +58,10 @@ from radicale.log import logger # Combination of types.WSGIStartResponse and WSGI application return value _IntermediateResponse = Tuple[str, List[Tuple[str, str]], Iterable[bytes]] +REQUEST_METHODS = ["DELETE", "GET", "HEAD", "MKCALENDAR", "MKCOL", "MOVE", "OPTIONS", "POST", "PROPFIND", "PROPPATCH", "PUT", "REPORT"] + +PROFILING: Sequence[str] = ("per_request", "per_request_method") + class Application(ApplicationPartDelete, ApplicationPartHead, ApplicationPartGet, ApplicationPartMkcalendar, @@ -73,6 +80,12 @@ class Application(ApplicationPartDelete, ApplicationPartHead, _web_type: str _script_name: str _extra_headers: Mapping[str, str] + _profiling_per_request: bool = False + _profiling_per_request_method: bool = False + profiler_per_request_method: dict[str, cProfile.Profile] = {} + profiler_per_request_method_counter: dict[str, int] = {} + profiler_per_request_method_starttime: datetime.datetime + profiler_per_request_method_logtime: datetime.datetime def __init__(self, configuration: config.Configuration) -> None: """Initialize Application. @@ -116,6 +129,52 @@ class Application(ApplicationPartDelete, ApplicationPartHead, self._extra_headers[key] = configuration.get("headers", key) self._strict_preconditions = configuration.get("storage", "strict_preconditions") logger.info("strict preconditions check: %s", self._strict_preconditions) + # Profiling options + self._profiling = configuration.get("logging", "profiling") + self._profiling_per_request_min_duration = configuration.get("logging", "profiling_per_request_min_duration") + self._profiling_per_request_method_interval = configuration.get("logging", "profiling_per_request_method_interval") + self._profiling_top_x_functions = configuration.get("logging", "profiling_top_x_functions") + if self._profiling == "per_request": + self._profiling_per_request = True + elif self._profiling == "per_request_method": + self._profiling_per_request_method = True + else: + logger.warning("profiling: %s (not supported, disabled)", self._profiling) + if self._profiling_per_request or self._profiling_per_request_method: + logger.info("profiling: %s", self._profiling) + logger.info("profiling top X functions: %d", self._profiling_top_x_functions) + if self._profiling_per_request: + logger.info("profiling per request minimum duration: %d (below are skipped)", self._profiling_per_request_min_duration) + if self._profiling_per_request_method: + logger.info("profiling per request method interval: %d seconds", self._profiling_per_request_method_interval) + # Profiling per request method initialization + if self._profiling_per_request_method: + for method in REQUEST_METHODS: + self.profiler_per_request_method[method] = cProfile.Profile() + self.profiler_per_request_method_counter[method] = False + self.profiler_per_request_method_starttime = datetime.datetime.now() + self.profiler_per_request_method_logtime = self.profiler_per_request_method_starttime + + def __del__(self) -> None: + """Shutdown application.""" + if self._profiling_per_request_method: + # Profiling since startup + self._profiler_per_request_method(True) + + def _profiler_per_request_method(self, shutdown: bool = False) -> None: + """Display profiler data per method.""" + profiler_timedelta_start = (datetime.datetime.now() - self.profiler_per_request_method_starttime).total_seconds() + for method in REQUEST_METHODS: + if self.profiler_per_request_method_counter[method] > 0: + s = io.StringIO() + stats = pstats.Stats(self.profiler_per_request_method[method], stream=s).sort_stats('cumulative') + stats.print_stats(self._profiling_top_x_functions) # Print top X functions + logger.info("Profiling data per request method after %d seconds and %d requests: %s: %s", profiler_timedelta_start, self.profiler_per_request_method_counter[method], method, s.getvalue()) + else: + if shutdown: + logger.info("Profiling data per request method after %d seconds: %s: (no requests seen so far)", profiler_timedelta_start, method) + else: + logger.debug("Profiling data per request method after %d seconds: %s: (no requests seen so far)", profiler_timedelta_start, method) def _scrub_headers(self, environ: types.WSGIEnviron) -> types.WSGIEnviron: """Mask passwords and cookies.""" @@ -156,6 +215,7 @@ class Application(ApplicationPartDelete, ApplicationPartHead, request_method = environ["REQUEST_METHOD"].upper() unsafe_path = environ.get("PATH_INFO", "") https = environ.get("HTTPS", "") + profiler = None context = AuthContext() @@ -194,6 +254,7 @@ class Application(ApplicationPartDelete, ApplicationPartHead, # Start response time_end = datetime.datetime.now() + time_delta_seconds = (time_end - time_begin).total_seconds() status_text = "%d %s" % ( status, client.responses.get(status, "Unknown")) if answer is not None: @@ -203,7 +264,29 @@ class Application(ApplicationPartDelete, ApplicationPartHead, else: logger.info("%s response status for %r%s in %.3f seconds: %s", request_method, unsafe_path, depthinfo, - (time_end - time_begin).total_seconds(), status_text) + time_delta_seconds, status_text) + + # Profiling end + if self._profiling_per_request: + if profiler is not None: + # Profiling per request + if time_delta_seconds < self._profiling_per_request_min_duration: + logger.debug("Profiling data %s response for %r%s: (supressed because duration below minimum %.3f < %.3f)", request_method, unsafe_path, depthinfo, time_delta_seconds, self._profiling_per_request_min_duration) + else: + s = io.StringIO() + stats = pstats.Stats(profiler, stream=s).sort_stats('cumulative') + stats.print_stats(self._profiling_top_x_functions) # Print top X functions + logger.info("Profiling data %s response for %r%s: %s", request_method, unsafe_path, depthinfo, s.getvalue()) + else: + logger.debug("Profiling data %s response for %r%s: (supressed because of no data)", request_method, unsafe_path, depthinfo) + elif self._profiling_per_request_method: + self.profiler_per_request_method[request_method].disable() + self.profiler_per_request_method_counter[request_method] += 1 + profiler_timedelta = (datetime.datetime.now() - self.profiler_per_request_method_logtime).total_seconds() + if profiler_timedelta > self._profiling_per_request_method_interval: + self._profiler_per_request_method() + self.profiler_per_request_method_logtime = datetime.datetime.now() + # Return response content return status_text, list(headers.items()), answers @@ -370,8 +453,23 @@ class Application(ApplicationPartDelete, ApplicationPartHead, return response(*httputils.REQUEST_ENTITY_TOO_LARGE) if not login or user: + # Profiling + if self._profiling_per_request: + profiler = cProfile.Profile() + profiler.enable() + elif self._profiling_per_request_method: + self.profiler_per_request_method[request_method].enable() + status, headers, answer = function( environ, base_prefix, path, user, remote_host, remote_useragent) + + # Profiling + if self._profiling_per_request: + if profiler is not None: + profiler.disable() + elif self._profiling_per_request_method: + self.profiler_per_request_method[request_method].disable() + if (status, headers, answer) == httputils.NOT_ALLOWED: logger.info("Access to %r denied for %s", path, repr(user) if user else "anonymous user") From 0776b1bdd0daf2649cc433635a9b4c76a80ec465 Mon Sep 17 00:00:00 2001 From: Peter Bieringer Date: Mon, 8 Dec 2025 12:33:30 +0100 Subject: [PATCH 165/290] profiling: extend changelog --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 920df43b..599a108e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,7 @@ * Add: logging of broken contact items during PUT * Extend: [auth] imap: add fallback support for LOGIN towards remote IMAP server (replaced in 3.5.0) * Fix: improper detection of HTTP_X_FORWARDED_PORT on MOVE +* Extend: [logging] with profiling log per reqest or regular per request method ## 3.5.9 * Extend: [auth] add support for type http_remote_user From 15ebb1e647deaea2e8a0d637d3aefceb2b095266 Mon Sep 17 00:00:00 2001 From: Peter Bieringer Date: Mon, 8 Dec 2025 17:54:18 +0100 Subject: [PATCH 166/290] profiling: default is now 'none' and config option will be checked instantly --- DOCUMENTATION.md | 3 ++- config | 4 ++-- radicale/config.py | 15 ++++++++++----- 3 files changed, 14 insertions(+), 8 deletions(-) diff --git a/DOCUMENTATION.md b/DOCUMENTATION.md index 420345b6..71670c68 100644 --- a/DOCUMENTATION.md +++ b/DOCUMENTATION.md @@ -1696,9 +1696,10 @@ _(>= 3.5.10)_ Log profiling data on level=info -Default: `per_request` +Default: `none` One of +* `none` (disabled) * `per_request` (above minimum duration) * `per_request_method` (regular interval) diff --git a/config b/config index 46b2f689..290fa368 100644 --- a/config +++ b/config @@ -322,8 +322,8 @@ #storage_cache_actions_on_debug = False # Log profiling data on level=info -# Value: per_request | per_request_method -#profiling = per_request_method +# Value: per_request | per_request_method | none +#profiling = none # Log profiling data per request minium duration (seconds) #profiling_per_request_min_duration = 3 diff --git a/radicale/config.py b/radicale/config.py index 5ec5908f..cf59bcd1 100644 --- a/radicale/config.py +++ b/radicale/config.py @@ -41,12 +41,12 @@ from radicale import auth, hook, rights, storage, types, web from radicale.hook import email from radicale.item import check_and_sanitize_props -from radicale import app # isort:skip (circular import issue) - DEFAULT_CONFIG_PATH: str = os.pathsep.join([ "?/etc/radicale/config", "?~/.config/radicale/config"]) +PROFILING: Sequence[str] = ("per_request", "per_request_method", "none") + def positive_int(value: Any) -> int: value = int(value) @@ -72,6 +72,12 @@ def logging_level(value: Any) -> str: return value +def profiling(value: Any) -> str: + if value not in PROFILING: + raise ValueError("unsupported profiling: %r" % value) + return value + + def filepath(value: Any) -> str: if not value: return "" @@ -584,10 +590,9 @@ This is an automated message. Please do not reply.""", "help": "log storage cache action on level=debug", "type": bool}), ("profiling", { - "value": "per_request_method", + "value": "none", "help": "log profiling data level=info", - "type": str, - "internal": app.PROFILING}), + "type": profiling}), ("profiling_per_request_min_duration", { "value": "3", "help": "log profiling data per request minimum duration (seconds)", From 2df265617b2cdab1f40d316d7e887cef61b34e42 Mon Sep 17 00:00:00 2001 From: Peter Bieringer Date: Mon, 8 Dec 2025 17:55:11 +0100 Subject: [PATCH 167/290] profiling: fix for 'none' --- radicale/app/__init__.py | 17 +++++++---------- 1 file changed, 7 insertions(+), 10 deletions(-) diff --git a/radicale/app/__init__.py b/radicale/app/__init__.py index 882787ec..d5d480e2 100644 --- a/radicale/app/__init__.py +++ b/radicale/app/__init__.py @@ -36,7 +36,7 @@ import random import time import zlib from http import client -from typing import Iterable, List, Mapping, Sequence, Tuple, Union +from typing import Iterable, List, Mapping, Tuple, Union from radicale import config, httputils, log, pathutils, types from radicale.app.base import ApplicationBase @@ -60,8 +60,6 @@ _IntermediateResponse = Tuple[str, List[Tuple[str, str]], Iterable[bytes]] REQUEST_METHODS = ["DELETE", "GET", "HEAD", "MKCALENDAR", "MKCOL", "MOVE", "OPTIONS", "POST", "PROPFIND", "PROPPATCH", "PUT", "REPORT"] -PROFILING: Sequence[str] = ("per_request", "per_request_method") - class Application(ApplicationPartDelete, ApplicationPartHead, ApplicationPartGet, ApplicationPartMkcalendar, @@ -134,14 +132,13 @@ class Application(ApplicationPartDelete, ApplicationPartHead, self._profiling_per_request_min_duration = configuration.get("logging", "profiling_per_request_min_duration") self._profiling_per_request_method_interval = configuration.get("logging", "profiling_per_request_method_interval") self._profiling_top_x_functions = configuration.get("logging", "profiling_top_x_functions") - if self._profiling == "per_request": - self._profiling_per_request = True - elif self._profiling == "per_request_method": - self._profiling_per_request_method = True - else: - logger.warning("profiling: %s (not supported, disabled)", self._profiling) - if self._profiling_per_request or self._profiling_per_request_method: + if self._profiling in config.PROFILING: logger.info("profiling: %s", self._profiling) + if self._profiling == "per_request": + self._profiling_per_request = True + elif self._profiling == "per_request_method": + self._profiling_per_request_method = True + if self._profiling_per_request or self._profiling_per_request_method: logger.info("profiling top X functions: %d", self._profiling_top_x_functions) if self._profiling_per_request: logger.info("profiling per request minimum duration: %d (below are skipped)", self._profiling_per_request_min_duration) From 5fbd838eab0e762867d29f29307738d4dfc7e300 Mon Sep 17 00:00:00 2001 From: Peter Bieringer Date: Mon, 8 Dec 2025 18:41:43 +0100 Subject: [PATCH 168/290] profiling: cosmetics/alignment --- radicale/app/__init__.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/radicale/app/__init__.py b/radicale/app/__init__.py index d5d480e2..9b3c9ba3 100644 --- a/radicale/app/__init__.py +++ b/radicale/app/__init__.py @@ -166,12 +166,12 @@ class Application(ApplicationPartDelete, ApplicationPartHead, s = io.StringIO() stats = pstats.Stats(self.profiler_per_request_method[method], stream=s).sort_stats('cumulative') stats.print_stats(self._profiling_top_x_functions) # Print top X functions - logger.info("Profiling data per request method after %d seconds and %d requests: %s: %s", profiler_timedelta_start, self.profiler_per_request_method_counter[method], method, s.getvalue()) + logger.info("Profiling data per request method %s after %d seconds and %d requests: %s", method, profiler_timedelta_start, self.profiler_per_request_method_counter[method], s.getvalue()) else: if shutdown: - logger.info("Profiling data per request method after %d seconds: %s: (no requests seen so far)", profiler_timedelta_start, method) + logger.info("Profiling data per request method %s after %d seconds: (no request seen so far)", method, profiler_timedelta_start) else: - logger.debug("Profiling data per request method after %d seconds: %s: (no requests seen so far)", profiler_timedelta_start, method) + logger.debug("Profiling data per request method %s after %d seconds: (no request seen so far)", method, profiler_timedelta_start) def _scrub_headers(self, environ: types.WSGIEnviron) -> types.WSGIEnviron: """Mask passwords and cookies.""" @@ -268,14 +268,14 @@ class Application(ApplicationPartDelete, ApplicationPartHead, if profiler is not None: # Profiling per request if time_delta_seconds < self._profiling_per_request_min_duration: - logger.debug("Profiling data %s response for %r%s: (supressed because duration below minimum %.3f < %.3f)", request_method, unsafe_path, depthinfo, time_delta_seconds, self._profiling_per_request_min_duration) + logger.debug("Profiling data per request %s for %r%s: (suppressed because duration below minimum %.3f < %.3f)", request_method, unsafe_path, depthinfo, time_delta_seconds, self._profiling_per_request_min_duration) else: s = io.StringIO() stats = pstats.Stats(profiler, stream=s).sort_stats('cumulative') stats.print_stats(self._profiling_top_x_functions) # Print top X functions - logger.info("Profiling data %s response for %r%s: %s", request_method, unsafe_path, depthinfo, s.getvalue()) + logger.info("Profiling data per request %s for %r%s: %s", request_method, unsafe_path, depthinfo, s.getvalue()) else: - logger.debug("Profiling data %s response for %r%s: (supressed because of no data)", request_method, unsafe_path, depthinfo) + logger.debug("Profiling data per request %s for %r%s: (suppressed because of no data)", request_method, unsafe_path, depthinfo) elif self._profiling_per_request_method: self.profiler_per_request_method[request_method].disable() self.profiler_per_request_method_counter[request_method] += 1 From 2ff6f188176bacaa281e5dcfaf0ebf85c4edaaef Mon Sep 17 00:00:00 2001 From: Peter Bieringer Date: Mon, 8 Dec 2025 18:42:02 +0100 Subject: [PATCH 169/290] profiling: logwatch extension (incl. skip not interesting lines) --- contrib/logwatch/radicale | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/contrib/logwatch/radicale b/contrib/logwatch/radicale index 7cc1b1b8..1cdddc30 100644 --- a/contrib/logwatch/radicale +++ b/contrib/logwatch/radicale @@ -126,11 +126,35 @@ while (defined($ThisLine = )) { elsif ( $ThisLine =~ / (Failed login attempt) /o ) { $OtherEvents{$1}++; } + elsif ( $ThisLine =~ / (Profiling data per request method \S+) /o ) { + my $info = $1; + if ( $ThisLine =~ /(no request seen so far)/o ) { + $OtherEvents{$info . " - " . $1}++; + } else { + $OtherEvents{$info}++; + }; + } + elsif ( $ThisLine =~ / (Profiling data per request \S+) /o ) { + my $info = $1; + if ( $ThisLine =~ /(suppressed because duration below minimum|suppressed because of no data)/o ) { + $OtherEvents{$info . " - " . $1}++; + } else { + $OtherEvents{$info}++; + }; + } elsif ( $ThisLine =~ /\[(DEBUG|INFO)\] /o ) { # skip if DEBUG+INFO } else { # Report any unmatched entries... + if ($ThisLine =~ /^({\'| )/o) { + # skip profiling or raw header data + next; + }; + if ($ThisLine =~ /^$/o) { + # skip empty line + next; + }; $ThisLine =~ s/^\[\d+(\/Thread-\d+)?\] //; # remove process/Thread ID chomp($ThisLine); $OtherList{$ThisLine}++; From 5fb844b924e0ed44f939bd10d1cdfdb6c8469afd Mon Sep 17 00:00:00 2001 From: Peter Bieringer Date: Mon, 8 Dec 2025 18:54:46 +0100 Subject: [PATCH 170/290] profiling: add config hint --- config | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config b/config index 290fa368..781c4edf 100644 --- a/config +++ b/config @@ -328,7 +328,7 @@ # Log profiling data per request minium duration (seconds) #profiling_per_request_min_duration = 3 -# Log profiling data per request method interval +# Log profiling data per request method interval (seconds) #profiling_per_request_method_interval = 600 # Log profiling top X functions (limit) From d7f54fbc487300b41d35b57ec9d70efc3e4972d8 Mon Sep 17 00:00:00 2001 From: Peter Bieringer Date: Tue, 9 Dec 2025 08:26:27 +0100 Subject: [PATCH 171/290] profiling: cosmetics --- radicale/app/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/radicale/app/__init__.py b/radicale/app/__init__.py index 9b3c9ba3..51d87cfc 100644 --- a/radicale/app/__init__.py +++ b/radicale/app/__init__.py @@ -133,7 +133,7 @@ class Application(ApplicationPartDelete, ApplicationPartHead, self._profiling_per_request_method_interval = configuration.get("logging", "profiling_per_request_method_interval") self._profiling_top_x_functions = configuration.get("logging", "profiling_top_x_functions") if self._profiling in config.PROFILING: - logger.info("profiling: %s", self._profiling) + logger.info("profiling: %r", self._profiling) if self._profiling == "per_request": self._profiling_per_request = True elif self._profiling == "per_request_method": From c918a401a1c39a51523448257ba4976d2cb08913 Mon Sep 17 00:00:00 2001 From: Peter Bieringer Date: Tue, 9 Dec 2025 08:51:23 +0100 Subject: [PATCH 172/290] bugfix related to min-time calc --- contrib/logwatch/radicale | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/contrib/logwatch/radicale b/contrib/logwatch/radicale index 1cdddc30..024be3c1 100644 --- a/contrib/logwatch/radicale +++ b/contrib/logwatch/radicale @@ -27,7 +27,7 @@ sub ResponseTimesMinMaxSum($$) { if (! defined $ResponseTimes{$req}->{'min'}) { $ResponseTimes{$req}->{'min'} = $time; - } elsif ($ResponseTimes->{$req}->{'min'} > $time) { + } elsif ($ResponseTimes{$req}->{'min'} > $time) { $ResponseTimes{$req}->{'min'} = $time; } From 8021848bcf38e51dcd2a65cfd222039e0066cd0e Mon Sep 17 00:00:00 2001 From: Peter Bieringer Date: Wed, 10 Dec 2025 17:44:21 +0100 Subject: [PATCH 173/290] clarify description --- DOCUMENTATION.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/DOCUMENTATION.md b/DOCUMENTATION.md index 71670c68..7c9e59f4 100644 --- a/DOCUMENTATION.md +++ b/DOCUMENTATION.md @@ -1654,7 +1654,7 @@ Default: `False` _(>= 3.2.2)_ -Log request on `level = debug` +Log request header on `level = debug` Default: `False` @@ -1662,7 +1662,7 @@ Default: `False` _(>= 3.2.2)_ -Log request on `level = debug` +Log request content (body) on `level = debug` Default: `False` @@ -1670,7 +1670,7 @@ Default: `False` _(>= 3.2.2)_ -Log response on `level = debug` +Log response content (body) on `level = debug` Default: `False` From 1cbcc4099468855d37e717021530a318e6448f73 Mon Sep 17 00:00:00 2001 From: Peter Bieringer Date: Wed, 10 Dec 2025 17:45:47 +0100 Subject: [PATCH 174/290] fix typo --- config | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config b/config index 781c4edf..e10ce295 100644 --- a/config +++ b/config @@ -325,7 +325,7 @@ # Value: per_request | per_request_method | none #profiling = none -# Log profiling data per request minium duration (seconds) +# Log profiling data per request minimum duration (seconds) #profiling_per_request_min_duration = 3 # Log profiling data per request method interval (seconds) From b819febeb21303d8d09c0a944515fed2f8ddae84 Mon Sep 17 00:00:00 2001 From: Peter Bieringer Date: Wed, 10 Dec 2025 17:47:09 +0100 Subject: [PATCH 175/290] new option [logging] response_header_on_debug --- DOCUMENTATION.md | 8 ++++++++ config | 3 +++ radicale/config.py | 4 ++++ 3 files changed, 15 insertions(+) diff --git a/DOCUMENTATION.md b/DOCUMENTATION.md index 7c9e59f4..0f03668f 100644 --- a/DOCUMENTATION.md +++ b/DOCUMENTATION.md @@ -1666,6 +1666,14 @@ Log request content (body) on `level = debug` Default: `False` +##### response_header_on_debug + +_(>= 3.5.10)_ + +Log response header on `level = debug` + +Default: `False` + ##### response_content_on_debug _(>= 3.2.2)_ diff --git a/config b/config index e10ce295..91b12e0f 100644 --- a/config +++ b/config @@ -312,6 +312,9 @@ # Log request content on level=debug #request_content_on_debug = False +# Log response header on level=debug +#response_header_on_debug = False + # Log response content on level=debug #response_content_on_debug = False diff --git a/radicale/config.py b/radicale/config.py index cf59bcd1..e4af0a8f 100644 --- a/radicale/config.py +++ b/radicale/config.py @@ -577,6 +577,10 @@ This is an automated message. Please do not reply.""", "value": "False", "help": "log request content on level=debug", "type": bool}), + ("response_header_on_debug", { + "value": "False", + "help": "log response header on level=debug", + "type": bool}), ("response_content_on_debug", { "value": "False", "help": "log response content on level=debug", From dfb932a44859dc9382db1e90c65c081d767d3d2d Mon Sep 17 00:00:00 2001 From: Peter Bieringer Date: Wed, 10 Dec 2025 17:48:05 +0100 Subject: [PATCH 176/290] new options [logging] profiling_per_request_header profiling_per_request_xml --- DOCUMENTATION.md | 16 ++++++++++++++++ config | 6 ++++++ radicale/config.py | 8 ++++++++ 3 files changed, 30 insertions(+) diff --git a/DOCUMENTATION.md b/DOCUMENTATION.md index 0f03668f..0a8760df 100644 --- a/DOCUMENTATION.md +++ b/DOCUMENTATION.md @@ -1719,6 +1719,22 @@ Log profiling data per request minimum duration (seconds) before logging, otherw Default: `3` +##### profiling_per_request_header + +_(>= 3.5.10)_ + +Log profiling request header (if passing minimum duration) + +Default: `False` + +##### profiling_per_request_xml + +_(>= 3.5.10)_ + +Log profiling request XML (if passing minimum duration) + +Default: `False` + ##### profiling_per_request_method_interval _(>= 3.5.10)_ diff --git a/config b/config index 91b12e0f..75a314e8 100644 --- a/config +++ b/config @@ -331,6 +331,12 @@ # Log profiling data per request minimum duration (seconds) #profiling_per_request_min_duration = 3 +# Log profiling request header (if passing minimum duration) +#profiling_per_request_header = False + +# Log profiling request XML (if passing minimum duration) +#profiling_per_request_xml = False + # Log profiling data per request method interval (seconds) #profiling_per_request_method_interval = 600 diff --git a/radicale/config.py b/radicale/config.py index e4af0a8f..abe7e465 100644 --- a/radicale/config.py +++ b/radicale/config.py @@ -601,6 +601,14 @@ This is an automated message. Please do not reply.""", "value": "3", "help": "log profiling data per request minimum duration (seconds)", "type": int}), + ("profiling_per_request_header", { + "value": "False", + "help": "Log profiling request body (if passing minimum duration)", + "type": bool}), + ("profiling_per_request_xml", { + "value": "False", + "help": "Log profiling request XML (if passing minimum duration)", + "type": bool}), ("profiling_per_request_method_interval", { "value": "600", "help": "log profiling data per request method interval (seconds)", From 97875b951450e6058ef22d421cf0b13ea4cbc424 Mon Sep 17 00:00:00 2001 From: Peter Bieringer Date: Wed, 10 Dec 2025 17:50:20 +0100 Subject: [PATCH 177/290] log status of new/existing log/profile header/content options on startup --- radicale/app/__init__.py | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/radicale/app/__init__.py b/radicale/app/__init__.py index 51d87cfc..e790d71a 100644 --- a/radicale/app/__init__.py +++ b/radicale/app/__init__.py @@ -96,8 +96,15 @@ class Application(ApplicationPartDelete, ApplicationPartHead, super().__init__(configuration) self._mask_passwords = configuration.get("logging", "mask_passwords") self._bad_put_request_content = configuration.get("logging", "bad_put_request_content") + logger.info("log bad put request content: %s", self._bad_put_request_content) self._request_header_on_debug = configuration.get("logging", "request_header_on_debug") + self._request_content_on_debug = configuration.get("logging", "request_content_on_debug") + self._response_header_on_debug = configuration.get("logging", "response_header_on_debug") self._response_content_on_debug = configuration.get("logging", "response_content_on_debug") + logger.debug("log request header on debug: %s", self._request_header_on_debug) + logger.debug("log request content on debug: %s", self._request_content_on_debug) + logger.debug("log response header on debug: %s", self._response_header_on_debug) + logger.debug("log response content on debug: %s", self._response_content_on_debug) self._auth_delay = configuration.get("auth", "delay") self._auth_type = configuration.get("auth", "type") self._web_type = configuration.get("web", "type") @@ -130,6 +137,8 @@ class Application(ApplicationPartDelete, ApplicationPartHead, # Profiling options self._profiling = configuration.get("logging", "profiling") self._profiling_per_request_min_duration = configuration.get("logging", "profiling_per_request_min_duration") + self._profiling_per_request_header = configuration.get("logging", "profiling_per_request_header") + self._profiling_per_request_xml = configuration.get("logging", "profiling_per_request_xml") self._profiling_per_request_method_interval = configuration.get("logging", "profiling_per_request_method_interval") self._profiling_top_x_functions = configuration.get("logging", "profiling_top_x_functions") if self._profiling in config.PROFILING: @@ -142,6 +151,8 @@ class Application(ApplicationPartDelete, ApplicationPartHead, logger.info("profiling top X functions: %d", self._profiling_top_x_functions) if self._profiling_per_request: logger.info("profiling per request minimum duration: %d (below are skipped)", self._profiling_per_request_min_duration) + logger.info("profiling per request header: %s", self._profiling_per_request_header) + logger.info("profiling per request xml : %s", self._profiling_per_request_xml) if self._profiling_per_request_method: logger.info("profiling per request method interval: %d seconds", self._profiling_per_request_method_interval) # Profiling per request method initialization From bac608a37accaefbfcb948d398ac5b3643ff20c7 Mon Sep 17 00:00:00 2001 From: Peter Bieringer Date: Wed, 10 Dec 2025 17:52:54 +0100 Subject: [PATCH 178/290] add support for empty argument on pretty_xml --- radicale/xmlutils.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/radicale/xmlutils.py b/radicale/xmlutils.py index 4b9c51bf..17fbf961 100644 --- a/radicale/xmlutils.py +++ b/radicale/xmlutils.py @@ -26,7 +26,7 @@ import copy import xml.etree.ElementTree as ET from collections import OrderedDict from http import client -from typing import Dict, Mapping, Optional +from typing import Dict, Mapping, Optional, Union from urllib.parse import quote from radicale import item, pathutils @@ -56,7 +56,7 @@ for short, url in NAMESPACES.items(): ET.register_namespace("" if short == "D" else short, url) -def pretty_xml(element: ET.Element) -> str: +def pretty_xml(element: Union[ET.Element | None]) -> str: """Indent an ElementTree ``element`` and its children.""" def pretty_xml_recursive(element: ET.Element, level: int) -> None: indent = "\n" + level * " " @@ -71,6 +71,9 @@ def pretty_xml(element: ET.Element) -> str: sub_element.tail = indent elif level > 0 and not (element.tail or "").strip(): element.tail = indent + + if element is None: + return "" element = copy.deepcopy(element) pretty_xml_recursive(element, 0) return '\n%s' % ET.tostring(element, "unicode") From 183e55614407e232c7f0f9e8de4332d6241063f7 Mon Sep 17 00:00:00 2001 From: Peter Bieringer Date: Wed, 10 Dec 2025 18:01:14 +0100 Subject: [PATCH 179/290] Adjust: [logging] header/content debug log indended by space to be skipped by logwatch --- radicale/app/__init__.py | 8 ++++---- radicale/app/base.py | 6 +++--- radicale/app/put.py | 6 +++--- radicale/httputils.py | 4 ++-- radicale/utils.py | 14 ++++++++++++++ 5 files changed, 26 insertions(+), 12 deletions(-) diff --git a/radicale/app/__init__.py b/radicale/app/__init__.py index e790d71a..0bcded6d 100644 --- a/radicale/app/__init__.py +++ b/radicale/app/__init__.py @@ -38,7 +38,7 @@ import zlib from http import client from typing import Iterable, List, Mapping, Tuple, Union -from radicale import config, httputils, log, pathutils, types +from radicale import config, httputils, log, pathutils, types, utils from radicale.app.base import ApplicationBase from radicale.app.delete import ApplicationPartDelete from radicale.app.get import ApplicationPartGet @@ -177,7 +177,7 @@ class Application(ApplicationPartDelete, ApplicationPartHead, s = io.StringIO() stats = pstats.Stats(self.profiler_per_request_method[method], stream=s).sort_stats('cumulative') stats.print_stats(self._profiling_top_x_functions) # Print top X functions - logger.info("Profiling data per request method %s after %d seconds and %d requests: %s", method, profiler_timedelta_start, self.profiler_per_request_method_counter[method], s.getvalue()) + logger.info("Profiling data per request method %s after %d seconds and %d requests: %s", method, profiler_timedelta_start, self.profiler_per_request_method_counter[method], utils.textwrap_str(s.getvalue(), -1)) else: if shutdown: logger.info("Profiling data per request method %s after %d seconds: (no request seen so far)", method, profiler_timedelta_start) @@ -238,7 +238,7 @@ class Application(ApplicationPartDelete, ApplicationPartHead, if answer is not None: if isinstance(answer, str): if self._response_content_on_debug: - logger.debug("Response content:\n%s", answer) + logger.debug("Response content (nonXML):\n%s", utils.textwrap_str(answer)) else: logger.debug("Response content: suppressed by config/option [logging] response_content_on_debug") headers["Content-Type"] += "; charset=%s" % self._encoding @@ -329,7 +329,7 @@ class Application(ApplicationPartDelete, ApplicationPartHead, remote_host, remote_useragent, https_info) if self._request_header_on_debug: logger.debug("Request header:\n%s", - pprint.pformat(self._scrub_headers(environ))) + utils.textwrap_str(pprint.pformat(self._scrub_headers(environ)))) else: logger.debug("Request header: suppressed by config/option [logging] request_header_on_debug") diff --git a/radicale/app/base.py b/radicale/app/base.py index 6e3a7cd3..229f0f65 100644 --- a/radicale/app/base.py +++ b/radicale/app/base.py @@ -23,7 +23,7 @@ import xml.etree.ElementTree as ET from typing import Optional from radicale import (auth, config, hook, httputils, pathutils, rights, - storage, types, web, xmlutils) + storage, types, utils, web, xmlutils) from radicale.log import logger # HACK: https://github.com/tiran/defusedxml/issues/54 @@ -71,7 +71,7 @@ class ApplicationBase: if logger.isEnabledFor(logging.DEBUG): if self._request_content_on_debug: logger.debug("Request content (XML):\n%s", - xmlutils.pretty_xml(xml_content)) + utils.textwrap_str(xmlutils.pretty_xml(xml_content))) else: logger.debug("Request content (XML): suppressed by config/option [logging] request_content_on_debug") return xml_content @@ -80,7 +80,7 @@ class ApplicationBase: if logger.isEnabledFor(logging.DEBUG): if self._response_content_on_debug: logger.debug("Response content (XML):\n%s", - xmlutils.pretty_xml(xml_content)) + utils.textwrap_str(xmlutils.pretty_xml(xml_content))) else: logger.debug("Response content (XML): suppressed by config/option [logging] response_content_on_debug") f = io.BytesIO() diff --git a/radicale/app/put.py b/radicale/app/put.py index daec6339..89a401a8 100644 --- a/radicale/app/put.py +++ b/radicale/app/put.py @@ -99,7 +99,7 @@ def prepare(vobject_items: List[vobject.base.Component], path: str, content = vobject_item else: content = item._text - logger.warning("Problem during prepare item with UID '%s' (content below): %s\n%s", item.uid, e, content) + logger.warning("Problem during prepare item with UID '%s' (content below): %s\n%s", item.uid, e, utils.textwrap_str(content)) else: logger.warning("Problem during prepare item with UID '%s' (content suppressed in this loglevel): %s", item.uid, e) raise @@ -117,7 +117,7 @@ def prepare(vobject_items: List[vobject.base.Component], path: str, content = vobject_item else: content = item._text - logger.warning("Problem during prepare item with UID '%s' (content below): %s\n%s", item.uid, e, content) + logger.warning("Problem during prepare item with UID '%s' (content below): %s\n%s", item.uid, e, utils.textwrap_str(content)) else: logger.warning("Problem during prepare item with UID '%s' (content suppressed in this loglevel): %s", item.uid, e) raise @@ -180,7 +180,7 @@ class ApplicationPartPut(ApplicationBase): logger.warning( "Bad PUT request on %r (read_components): %s", path, e, exc_info=True) if self._log_bad_put_request_content: - logger.warning("Bad PUT request content of %r:\n%s", path, content) + logger.warning("Bad PUT request content of %r:\n%s", path, utils.textwrap_str(content)) else: logger.debug("Bad PUT request content: suppressed by config/option [logging] bad_put_request_content") return httputils.BAD_REQUEST diff --git a/radicale/httputils.py b/radicale/httputils.py index 23f10ec1..d97d0513 100644 --- a/radicale/httputils.py +++ b/radicale/httputils.py @@ -31,7 +31,7 @@ import time from http import client from typing import List, Mapping, Union, cast -from radicale import config, pathutils, types +from radicale import config, pathutils, types, utils from radicale.log import logger if sys.version_info < (3, 9): @@ -150,7 +150,7 @@ def read_request_body(configuration: "config.Configuration", content = decode_request(configuration, environ, read_raw_request_body(configuration, environ)) if configuration.get("logging", "request_content_on_debug"): - logger.debug("Request content:\n%s", content) + logger.debug("Request content:\n%s", utils.textwrap_str(content)) else: logger.debug("Request content: suppressed by config/option [logging] request_content_on_debug") return content diff --git a/radicale/utils.py b/radicale/utils.py index ba5c69ff..16540bde 100644 --- a/radicale/utils.py +++ b/radicale/utils.py @@ -21,6 +21,7 @@ import datetime import os import ssl import sys +import textwrap from importlib import import_module, metadata from typing import Callable, Sequence, Tuple, Type, TypeVar, Union @@ -291,3 +292,16 @@ def format_ut(unixtime: int) -> str: dt = datetime.datetime.fromtimestamp(unixtime, datetime.UTC) r = str(unixtime) + "(" + dt.strftime('%Y-%m-%dT%H:%M:%SZ') + ")" return r + + +def limit_str(content: str, limit: int) -> str: + length = len(content) + if limit > 0 and length >= limit: + return content[:limit] + ("...(shortened because original length %d > limit %d)" % (length, limit)) + else: + return content + + +def textwrap_str(content: str, limit: int = 2000) -> str: + # TODO: add support for config option and prefix + return textwrap.indent(limit_str(content, limit), " ", lambda line: True) From 64f5e58549a162f80448fd71ebfd1a61123db567 Mon Sep 17 00:00:00 2001 From: Peter Bieringer Date: Wed, 10 Dec 2025 18:09:04 +0100 Subject: [PATCH 180/290] add support for logging XML request conditionally for profiling --- radicale/app/__init__.py | 16 +++++++++------- radicale/app/base.py | 2 +- radicale/app/delete.py | 2 +- radicale/app/get.py | 2 +- radicale/app/mkcalendar.py | 2 +- radicale/app/mkcol.py | 2 +- radicale/app/move.py | 2 +- radicale/app/options.py | 2 +- radicale/app/propfind.py | 2 +- radicale/app/proppatch.py | 2 +- radicale/app/put.py | 4 ++-- radicale/app/report.py | 4 ++-- radicale/httputils.py | 30 +++++++++++++++--------------- radicale/tests/custom/web.py | 4 ++-- radicale/types.py | 2 +- radicale/web/none.py | 2 +- 16 files changed, 41 insertions(+), 39 deletions(-) diff --git a/radicale/app/__init__.py b/radicale/app/__init__.py index 0bcded6d..98123cd6 100644 --- a/radicale/app/__init__.py +++ b/radicale/app/__init__.py @@ -204,7 +204,7 @@ class Application(ApplicationPartDelete, ApplicationPartHead, "%s", environ.get("REQUEST_METHOD", "unknown"), environ.get("PATH_INFO", ""), e, exc_info=True) # Make minimal response - status, raw_headers, raw_answer = ( + status, raw_headers, raw_answer, xml_request = ( httputils.INTERNAL_SERVER_ERROR) assert isinstance(raw_answer, str) answer = raw_answer.encode("ascii") @@ -224,12 +224,14 @@ class Application(ApplicationPartDelete, ApplicationPartHead, unsafe_path = environ.get("PATH_INFO", "") https = environ.get("HTTPS", "") profiler = None + xml_request = None context = AuthContext() """Manage a request.""" def response(status: int, headers: types.WSGIResponseHeaders, - answer: Union[None, str, bytes]) -> _IntermediateResponse: + answer: Union[None, str, bytes], + xml_request: Union[None, str] = None) -> _IntermediateResponse: """Helper to create response from internal types.WSGIResponse""" headers = dict(headers) content_encoding = "plain" @@ -468,7 +470,7 @@ class Application(ApplicationPartDelete, ApplicationPartHead, elif self._profiling_per_request_method: self.profiler_per_request_method[request_method].enable() - status, headers, answer = function( + status, headers, answer, xml_request = function( environ, base_prefix, path, user, remote_host, remote_useragent) # Profiling @@ -478,13 +480,13 @@ class Application(ApplicationPartDelete, ApplicationPartHead, elif self._profiling_per_request_method: self.profiler_per_request_method[request_method].disable() - if (status, headers, answer) == httputils.NOT_ALLOWED: + if (status, headers, answer, xml_request) == httputils.NOT_ALLOWED: logger.info("Access to %r denied for %s", path, repr(user) if user else "anonymous user") else: - status, headers, answer = httputils.NOT_ALLOWED + status, headers, answer, xml_request = httputils.NOT_ALLOWED - if ((status, headers, answer) == httputils.NOT_ALLOWED and not user and + if ((status, headers, answer, xml_request) == httputils.NOT_ALLOWED and not user and not external_login): # Unknown or unauthorized user logger.debug("Asking client for authentication") @@ -494,4 +496,4 @@ class Application(ApplicationPartDelete, ApplicationPartHead, "WWW-Authenticate": "Basic realm=\"%s\"" % self._auth_realm}) - return response(status, headers, answer) + return response(status, headers, answer, xml_request) diff --git a/radicale/app/base.py b/radicale/app/base.py index 229f0f65..18a68b95 100644 --- a/radicale/app/base.py +++ b/radicale/app/base.py @@ -93,7 +93,7 @@ class ApplicationBase: """Generate XML error response.""" headers = {"Content-Type": "text/xml; charset=%s" % self._encoding} content = self._xml_response(xmlutils.webdav_error(human_tag)) - return status, headers, content + return status, headers, content, None class Access: diff --git a/radicale/app/delete.py b/radicale/app/delete.py index 61fe6c43..2201e998 100644 --- a/radicale/app/delete.py +++ b/radicale/app/delete.py @@ -110,4 +110,4 @@ class ApplicationPartDelete(ApplicationBase): for notification_item in hook_notification_item_list: self._hook.notify(notification_item) headers = {"Content-Type": "text/xml; charset=%s" % self._encoding} - return client.OK, headers, self._xml_response(xml_answer) + return client.OK, headers, self._xml_response(xml_answer), None diff --git a/radicale/app/get.py b/radicale/app/get.py index b8adb39a..2eac58f1 100644 --- a/radicale/app/get.py +++ b/radicale/app/get.py @@ -109,4 +109,4 @@ class ApplicationPartGet(ApplicationBase): if content_disposition: headers["Content-Disposition"] = content_disposition answer = item.serialize() - return client.OK, headers, answer + return client.OK, headers, answer, None diff --git a/radicale/app/mkcalendar.py b/radicale/app/mkcalendar.py index db14bfdc..53abcdbd 100644 --- a/radicale/app/mkcalendar.py +++ b/radicale/app/mkcalendar.py @@ -89,4 +89,4 @@ class ApplicationPartMkcalendar(ApplicationBase): logger.warning( "Bad MKCALENDAR request on %r: %s", path, e, exc_info=True) return httputils.BAD_REQUEST - return client.CREATED, {}, None + return client.CREATED, {}, None, xmlutils.pretty_xml(xml_content) diff --git a/radicale/app/mkcol.py b/radicale/app/mkcol.py index 72d5aa2b..45ad7c4a 100644 --- a/radicale/app/mkcol.py +++ b/radicale/app/mkcol.py @@ -94,4 +94,4 @@ class ApplicationPartMkcol(ApplicationBase): "Bad MKCOL request on %r (type:%s): %s", path, collection_type, e, exc_info=True) return httputils.BAD_REQUEST logger.info("MKCOL request %r (type:%s): %s", path, collection_type, "successful") - return client.CREATED, {}, None + return client.CREATED, {}, None, xmlutils.pretty_xml(xml_content) diff --git a/radicale/app/move.py b/radicale/app/move.py index ac4fcab4..b65f6600 100644 --- a/radicale/app/move.py +++ b/radicale/app/move.py @@ -127,4 +127,4 @@ class ApplicationPartMove(ApplicationBase): logger.warning( "Bad MOVE request on %r: %s", path, e, exc_info=True) return httputils.BAD_REQUEST - return client.NO_CONTENT if to_item else client.CREATED, {}, None + return client.NO_CONTENT if to_item else client.CREATED, {}, None, None diff --git a/radicale/app/options.py b/radicale/app/options.py index a869e2ab..9e347de2 100644 --- a/radicale/app/options.py +++ b/radicale/app/options.py @@ -33,4 +33,4 @@ class ApplicationPartOptions(ApplicationBase): "Allow": ", ".join( name[3:] for name in dir(self) if name.startswith("do_")), "DAV": httputils.DAV_HEADERS} - return client.OK, headers, None + return client.OK, headers, None, None diff --git a/radicale/app/propfind.py b/radicale/app/propfind.py index b546c5e1..d79d0269 100644 --- a/radicale/app/propfind.py +++ b/radicale/app/propfind.py @@ -410,4 +410,4 @@ class ApplicationPartPropfind(ApplicationBase): allowed_items, user, self._encoding) if xml_answer is None: return httputils.NOT_ALLOWED - return client.MULTI_STATUS, headers, self._xml_response(xml_answer) + return client.MULTI_STATUS, headers, self._xml_response(xml_answer), xmlutils.pretty_xml(xml_content) diff --git a/radicale/app/proppatch.py b/radicale/app/proppatch.py index 9d6dc221..caaf7b7a 100644 --- a/radicale/app/proppatch.py +++ b/radicale/app/proppatch.py @@ -131,4 +131,4 @@ class ApplicationPartProppatch(ApplicationBase): logger.warning( "Bad PROPPATCH request on %r: %s", path, e, exc_info=True) return httputils.BAD_REQUEST - return client.MULTI_STATUS, headers, self._xml_response(xml_answer) + return client.MULTI_STATUS, headers, self._xml_response(xml_answer), xmlutils.pretty_xml(xml_content) diff --git a/radicale/app/put.py b/radicale/app/put.py index 89a401a8..bd049158 100644 --- a/radicale/app/put.py +++ b/radicale/app/put.py @@ -334,7 +334,7 @@ class ApplicationPartPut(ApplicationBase): if (item and item.uid == prepared_item.uid): logger.debug("PUT request updated existing item %r", path) headers = {"ETag": etag} - return client.NO_CONTENT, headers, None + return client.NO_CONTENT, headers, None, None headers = {"ETag": etag} - return client.CREATED, headers, None + return client.CREATED, headers, None, None diff --git a/radicale/app/report.py b/radicale/app/report.py index dda4ba10..2bad1924 100644 --- a/radicale/app/report.py +++ b/radicale/app/report.py @@ -815,7 +815,7 @@ class ApplicationPartReport(ApplicationBase): "Bad REPORT request on %r: %s", path, e, exc_info=True) return httputils.BAD_REQUEST headers = {"Content-Type": "text/calendar; charset=%s" % self._encoding} - return status, headers, str(body) + return status, headers, str(body), xmlutils.pretty_xml(xml_content) else: try: status, xml_answer = xml_report( @@ -826,4 +826,4 @@ class ApplicationPartReport(ApplicationBase): "Bad REPORT request on %r: %s", path, e, exc_info=True) return httputils.BAD_REQUEST headers = {"Content-Type": "text/xml; charset=%s" % self._encoding} - return status, headers, self._xml_response(xml_answer) + return status, headers, self._xml_response(xml_answer), xmlutils.pretty_xml(xml_content) diff --git a/radicale/httputils.py b/radicale/httputils.py index d97d0513..30c6a7a0 100644 --- a/radicale/httputils.py +++ b/radicale/httputils.py @@ -49,42 +49,42 @@ else: NOT_ALLOWED: types.WSGIResponse = ( client.FORBIDDEN, (("Content-Type", "text/plain"),), - "Access to the requested resource forbidden.") + "Access to the requested resource forbidden.", None) FORBIDDEN: types.WSGIResponse = ( client.FORBIDDEN, (("Content-Type", "text/plain"),), - "Action on the requested resource refused.") + "Action on the requested resource refused.", None) BAD_REQUEST: types.WSGIResponse = ( - client.BAD_REQUEST, (("Content-Type", "text/plain"),), "Bad Request") + client.BAD_REQUEST, (("Content-Type", "text/plain"),), "Bad Request", None) NOT_FOUND: types.WSGIResponse = ( client.NOT_FOUND, (("Content-Type", "text/plain"),), - "The requested resource could not be found.") + "The requested resource could not be found.", None) CONFLICT: types.WSGIResponse = ( client.CONFLICT, (("Content-Type", "text/plain"),), - "Conflict in the request.") + "Conflict in the request.", None) METHOD_NOT_ALLOWED: types.WSGIResponse = ( client.METHOD_NOT_ALLOWED, (("Content-Type", "text/plain"),), - "The method is not allowed on the requested resource.") + "The method is not allowed on the requested resource.", None) PRECONDITION_FAILED: types.WSGIResponse = ( client.PRECONDITION_FAILED, - (("Content-Type", "text/plain"),), "Precondition failed.") + (("Content-Type", "text/plain"),), "Precondition failed.", None) REQUEST_TIMEOUT: types.WSGIResponse = ( client.REQUEST_TIMEOUT, (("Content-Type", "text/plain"),), - "Connection timed out.") + "Connection timed out.", None) REQUEST_ENTITY_TOO_LARGE: types.WSGIResponse = ( client.REQUEST_ENTITY_TOO_LARGE, (("Content-Type", "text/plain"),), - "Request body too large.") + "Request body too large.", None) REMOTE_DESTINATION: types.WSGIResponse = ( client.BAD_GATEWAY, (("Content-Type", "text/plain"),), - "Remote destination not supported.") + "Remote destination not supported.", None) DIRECTORY_LISTING: types.WSGIResponse = ( client.FORBIDDEN, (("Content-Type", "text/plain"),), - "Directory listings are not supported.") + "Directory listings are not supported.", None) INSUFFICIENT_STORAGE: types.WSGIResponse = ( client.INSUFFICIENT_STORAGE, (("Content-Type", "text/plain"),), - "Insufficient Storage. Please contact the administrator.") + "Insufficient Storage. Please contact the administrator.", None) INTERNAL_SERVER_ERROR: types.WSGIResponse = ( client.INTERNAL_SERVER_ERROR, (("Content-Type", "text/plain"),), - "A server error occurred. Please contact the administrator.") + "A server error occurred. Please contact the administrator.", None) DAV_HEADERS: str = "1, 2, 3, calendar-access, addressbook, extended-mkcol" @@ -159,7 +159,7 @@ def read_request_body(configuration: "config.Configuration", def redirect(location: str, status: int = client.FOUND) -> types.WSGIResponse: return (status, {"Location": location, "Content-Type": "text/plain"}, - "Redirected to %s" % location) + "Redirected to %s" % location, None) def _serve_traversable( @@ -214,7 +214,7 @@ def _serve_traversable( # adjust on the fly default main.js of InfCloud installation logger.debug("Adjust on-the-fly default InfCloud main.js in served page: %r", path) answer = answer.replace(b"'InfCloud - the open source CalDAV/CardDAV web client'", b"'InfCloud - the open source CalDAV/CardDAV web client - served through Radicale CalDAV/CardDAV server'") - return client.OK, headers, answer + return client.OK, headers, answer, None def serve_resource( diff --git a/radicale/tests/custom/web.py b/radicale/tests/custom/web.py index 695bbe81..2570fec0 100644 --- a/radicale/tests/custom/web.py +++ b/radicale/tests/custom/web.py @@ -28,9 +28,9 @@ class Web(web.BaseWeb): def get(self, environ: types.WSGIEnviron, base_prefix: str, path: str, user: str) -> types.WSGIResponse: - return client.OK, {"Content-Type": "text/plain"}, "custom" + return client.OK, {"Content-Type": "text/plain"}, "custom", None def post(self, environ: types.WSGIEnviron, base_prefix: str, path: str, user: str) -> types.WSGIResponse: content = httputils.read_request_body(self.configuration, environ) - return client.OK, {"Content-Type": "text/plain"}, "echo:" + content + return client.OK, {"Content-Type": "text/plain"}, "echo:" + content, None diff --git a/radicale/types.py b/radicale/types.py index 6899a755..6c7a7198 100644 --- a/radicale/types.py +++ b/radicale/types.py @@ -20,7 +20,7 @@ from typing import (Any, Callable, ContextManager, Iterator, List, Mapping, runtime_checkable) WSGIResponseHeaders = Union[Mapping[str, str], Sequence[Tuple[str, str]]] -WSGIResponse = Tuple[int, WSGIResponseHeaders, Union[None, str, bytes]] +WSGIResponse = Tuple[int, WSGIResponseHeaders, Union[None, str, bytes], Union[None, str]] WSGIEnviron = Mapping[str, Any] WSGIStartResponse = Callable[[str, List[Tuple[str, str]]], Any] diff --git a/radicale/web/none.py b/radicale/web/none.py index 263992ec..15e255c7 100644 --- a/radicale/web/none.py +++ b/radicale/web/none.py @@ -32,4 +32,4 @@ class Web(web.BaseWeb): assert pathutils.sanitize_path(path) == path if path != "/.web": return httputils.redirect(base_prefix + "/.web") - return client.OK, {"Content-Type": "text/plain"}, "Radicale works!" + return client.OK, {"Content-Type": "text/plain"}, "Radicale works!", None From a60e86671be6ffeb7a7c857708b9a8ddacdbbd21 Mon Sep 17 00:00:00 2001 From: Peter Bieringer Date: Wed, 10 Dec 2025 18:10:51 +0100 Subject: [PATCH 181/290] extend copyright --- radicale/tests/custom/web.py | 3 ++- radicale/types.py | 3 ++- radicale/web/none.py | 3 ++- radicale/xmlutils.py | 3 ++- 4 files changed, 8 insertions(+), 4 deletions(-) diff --git a/radicale/tests/custom/web.py b/radicale/tests/custom/web.py index 2570fec0..ee8bc6e6 100644 --- a/radicale/tests/custom/web.py +++ b/radicale/tests/custom/web.py @@ -1,5 +1,6 @@ # This file is part of Radicale - CalDAV and CardDAV server -# Copyright © 2017-2018 Unrud +# Copyright © 2017-2021 Unrud +# Copyright © 2025-2025 Peter Bieringer # # This library is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by diff --git a/radicale/types.py b/radicale/types.py index 6c7a7198..175869d7 100644 --- a/radicale/types.py +++ b/radicale/types.py @@ -1,5 +1,6 @@ # This file is part of Radicale - CalDAV and CardDAV server -# Copyright © 2020 Unrud +# Copyright © 2020-2023 Unrud +# Copyright © 2024-2025 Peter Bieringer # # This library is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by diff --git a/radicale/web/none.py b/radicale/web/none.py index 15e255c7..59cc341e 100644 --- a/radicale/web/none.py +++ b/radicale/web/none.py @@ -1,5 +1,6 @@ # This file is part of Radicale - CalDAV and CardDAV server -# Copyright © 2017-2018 Unrud +# Copyright © 2017-2022 Unrud +# Copyright © 2025-2025 Peter Bieringer # # This library is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by diff --git a/radicale/xmlutils.py b/radicale/xmlutils.py index 17fbf961..becfecca 100644 --- a/radicale/xmlutils.py +++ b/radicale/xmlutils.py @@ -2,7 +2,8 @@ # Copyright © 2008 Nicolas Kandel # Copyright © 2008 Pascal Halter # Copyright © 2008-2015 Guillaume Ayoub -# Copyright © 2017-2018 Unrud +# Copyright © 2017-2021 Unrud +# Copyright © 2025-2025 Peter Bieringer # # This library is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by From 2b2c417b1be3fe490ff8323bfe1518dfaea8e526 Mon Sep 17 00:00:00 2001 From: Peter Bieringer Date: Wed, 10 Dec 2025 18:11:20 +0100 Subject: [PATCH 182/290] implement log of response header --- radicale/app/__init__.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/radicale/app/__init__.py b/radicale/app/__init__.py index 98123cd6..fc7e1981 100644 --- a/radicale/app/__init__.py +++ b/radicale/app/__init__.py @@ -262,6 +262,11 @@ class Application(ApplicationPartDelete, ApplicationPartHead, # Add extra headers set in configuration headers.update(self._extra_headers) + if self._response_header_on_debug: + logger.debug("Response header:\n%s", utils.textwrap_str(pprint.pformat(headers))) + else: + logger.debug("Response header: suppressed by config/option [logging] response_header_on_debug") + # Start response time_end = datetime.datetime.now() time_delta_seconds = (time_end - time_begin).total_seconds() From f015b8e7a1829023c539a8c56dfac8950ae321be Mon Sep 17 00:00:00 2001 From: Peter Bieringer Date: Wed, 10 Dec 2025 18:12:08 +0100 Subject: [PATCH 183/290] avoid raise condition with parallel running profilers --- radicale/app/__init__.py | 21 +++++++++++++++++---- 1 file changed, 17 insertions(+), 4 deletions(-) diff --git a/radicale/app/__init__.py b/radicale/app/__init__.py index fc7e1981..df02d14b 100644 --- a/radicale/app/__init__.py +++ b/radicale/app/__init__.py @@ -224,6 +224,7 @@ class Application(ApplicationPartDelete, ApplicationPartHead, unsafe_path = environ.get("PATH_INFO", "") https = environ.get("HTTPS", "") profiler = None + profiler_active = False xml_request = None context = AuthContext() @@ -471,9 +472,19 @@ class Application(ApplicationPartDelete, ApplicationPartHead, # Profiling if self._profiling_per_request: profiler = cProfile.Profile() - profiler.enable() + try: + profiler.enable() + except ValueError: + profiler_active = False + else: + profiler_active = True elif self._profiling_per_request_method: - self.profiler_per_request_method[request_method].enable() + try: + self.profiler_per_request_method[request_method].enable() + except ValueError: + profiler_active = False + else: + profiler_active = True status, headers, answer, xml_request = function( environ, base_prefix, path, user, remote_host, remote_useragent) @@ -481,9 +492,11 @@ class Application(ApplicationPartDelete, ApplicationPartHead, # Profiling if self._profiling_per_request: if profiler is not None: - profiler.disable() + if profiler_active is True: + profiler.disable() elif self._profiling_per_request_method: - self.profiler_per_request_method[request_method].disable() + if profiler_active is True: + self.profiler_per_request_method[request_method].disable() if (status, headers, answer, xml_request) == httputils.NOT_ALLOWED: logger.info("Access to %r denied for %s", path, From c0af7bf1ba04a37391ec0a3e0ee8bd62454e860a Mon Sep 17 00:00:00 2001 From: Peter Bieringer Date: Wed, 10 Dec 2025 18:21:21 +0100 Subject: [PATCH 184/290] extend profiling by logging header and XML request in case min duration exceeded --- radicale/app/__init__.py | 37 +++++++++++++++++++++++++++---------- 1 file changed, 27 insertions(+), 10 deletions(-) diff --git a/radicale/app/__init__.py b/radicale/app/__init__.py index df02d14b..63fe561e 100644 --- a/radicale/app/__init__.py +++ b/radicale/app/__init__.py @@ -175,9 +175,11 @@ class Application(ApplicationPartDelete, ApplicationPartHead, for method in REQUEST_METHODS: if self.profiler_per_request_method_counter[method] > 0: s = io.StringIO() + s.write("**Profiling statistics BEGIN**\n") stats = pstats.Stats(self.profiler_per_request_method[method], stream=s).sort_stats('cumulative') stats.print_stats(self._profiling_top_x_functions) # Print top X functions - logger.info("Profiling data per request method %s after %d seconds and %d requests: %s", method, profiler_timedelta_start, self.profiler_per_request_method_counter[method], utils.textwrap_str(s.getvalue(), -1)) + s.write("**Profiling statistics END**\n") + logger.info("Profiling data per request method %s after %d seconds and %d requests:\n%s", method, profiler_timedelta_start, self.profiler_per_request_method_counter[method], utils.textwrap_str(s.getvalue(), -1)) else: if shutdown: logger.info("Profiling data per request method %s after %d seconds: (no request seen so far)", method, profiler_timedelta_start) @@ -284,17 +286,32 @@ class Application(ApplicationPartDelete, ApplicationPartHead, # Profiling end if self._profiling_per_request: - if profiler is not None: - # Profiling per request - if time_delta_seconds < self._profiling_per_request_min_duration: - logger.debug("Profiling data per request %s for %r%s: (suppressed because duration below minimum %.3f < %.3f)", request_method, unsafe_path, depthinfo, time_delta_seconds, self._profiling_per_request_min_duration) + if profiler_active is True: + if profiler is not None: + # Profiling per request + if time_delta_seconds < self._profiling_per_request_min_duration: + logger.debug("Profiling data per request %s for %r%s: (suppressed because duration below minimum %.3f < %.3f)", request_method, unsafe_path, depthinfo, time_delta_seconds, self._profiling_per_request_min_duration) + else: + s = io.StringIO() + s.write("**Profiling statistics BEGIN**\n") + stats = pstats.Stats(profiler, stream=s).sort_stats('cumulative') + stats.print_stats(self._profiling_top_x_functions) # Print top X functions + s.write("**Profiling statistics END**\n") + if self._profiling_per_request_header: + s.write("**Profiling request header BEGIN**\n") + s.write(pprint.pformat(self._scrub_headers(environ))) + s.write("\n**Profiling request header END**") + if self._profiling_per_request_xml: + if xml_request is not None: + s.write("\n**Profiling request content (XML) BEGIN**\n") + if xml_request is not None: + s.write(xml_request) + s.write("**Profiling request content (XML) END**") + logger.info("Profiling data per request %s for %r%s:\n%s", request_method, unsafe_path, depthinfo, utils.textwrap_str(s.getvalue(), -1)) else: - s = io.StringIO() - stats = pstats.Stats(profiler, stream=s).sort_stats('cumulative') - stats.print_stats(self._profiling_top_x_functions) # Print top X functions - logger.info("Profiling data per request %s for %r%s: %s", request_method, unsafe_path, depthinfo, s.getvalue()) + logger.debug("Profiling data per request %s for %r%s: (suppressed because of no data)", request_method, unsafe_path, depthinfo) else: - logger.debug("Profiling data per request %s for %r%s: (suppressed because of no data)", request_method, unsafe_path, depthinfo) + logger.info("Profiling data per request %s for %r%s: (not available because of concurrent running profiling request)", request_method, unsafe_path, depthinfo) elif self._profiling_per_request_method: self.profiler_per_request_method[request_method].disable() self.profiler_per_request_method_counter[request_method] += 1 From 12c37af1db7c041828f599e4afd6dd9c72e7441e Mon Sep 17 00:00:00 2001 From: Peter Bieringer Date: Wed, 10 Dec 2025 18:21:57 +0100 Subject: [PATCH 185/290] update changelog --- CHANGELOG.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 599a108e..c4fd54a8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,8 @@ * Extend: [auth] imap: add fallback support for LOGIN towards remote IMAP server (replaced in 3.5.0) * Fix: improper detection of HTTP_X_FORWARDED_PORT on MOVE * Extend: [logging] with profiling log per reqest or regular per request method +* New: [logging] option to log response header on debug loglevel +* Adjust: [logging] header/content debug log indended by space to be skipped by logwatch ## 3.5.9 * Extend: [auth] add support for type http_remote_user From f879bd5c5d1dac51c5f5a7e699f16d2eaae4b38b Mon Sep 17 00:00:00 2001 From: Peter Bieringer Date: Wed, 10 Dec 2025 18:34:22 +0100 Subject: [PATCH 186/290] bugfix --- radicale/xmlutils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/radicale/xmlutils.py b/radicale/xmlutils.py index becfecca..4c31bbb1 100644 --- a/radicale/xmlutils.py +++ b/radicale/xmlutils.py @@ -57,7 +57,7 @@ for short, url in NAMESPACES.items(): ET.register_namespace("" if short == "D" else short, url) -def pretty_xml(element: Union[ET.Element | None]) -> str: +def pretty_xml(element: Union[ET.Element, None]) -> str: """Indent an ElementTree ``element`` and its children.""" def pretty_xml_recursive(element: ET.Element, level: int) -> None: indent = "\n" + level * " " From 4f8c3fff05f835095495e06f1b1430fb318a5158 Mon Sep 17 00:00:00 2001 From: Peter Bieringer Date: Wed, 10 Dec 2025 21:08:16 +0100 Subject: [PATCH 187/290] remove unnecessary open+read for mtime+size cache --- radicale/storage/multifilesystem/get.py | 23 +++++++++++++++++++++-- 1 file changed, 21 insertions(+), 2 deletions(-) diff --git a/radicale/storage/multifilesystem/get.py b/radicale/storage/multifilesystem/get.py index f74c8fb6..234bfb3b 100644 --- a/radicale/storage/multifilesystem/get.py +++ b/radicale/storage/multifilesystem/get.py @@ -68,8 +68,21 @@ class CollectionPartGet(CollectionPartCache, CollectionPartLock, else: path = os.path.join(self._filesystem_path, href) try: - with open(path, "rb") as f: - raw_text = f.read() + if self._storage._use_mtime_and_size_for_item_cache is True: + # try to avoid "open" + if not os.path.isfile(path): + if not os.path.exists(path): + raise FileNotFoundError(path) + if os.path.isdir(path): + raise IsADirectoryError(path) + if not os.access(path, os.R_OK): + raise PermissionError(path) + else: + with open(path, "rb") as f: + # early read of the content + if self._storage._debug_cache_actions is True: + logger.debug("Item cache early read: %r", path) + raw_text = f.read() except (FileNotFoundError, IsADirectoryError): return None except PermissionError: @@ -100,6 +113,12 @@ class CollectionPartGet(CollectionPartCache, CollectionPartLock, # Check if another process created the file in the meantime cache_content = self._load_item_cache(href, cache_hash) if cache_content is None: + if self._storage._use_mtime_and_size_for_item_cache is True: + # delayed read of the content + if self._storage._debug_cache_actions is True: + logger.debug("Item cache late read : %r", path) + with open(path, "rb") as f: + raw_text = f.read() try: vobject_items = radicale_item.read_components( raw_text.decode(self._encoding)) From 344c32276398685e6f091394bd7a134e34a89dad Mon Sep 17 00:00:00 2001 From: Peter Bieringer Date: Wed, 10 Dec 2025 21:10:48 +0100 Subject: [PATCH 188/290] changelog for 4f8c3fff05f835095495e06f1b1430fb318a5158 --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index c4fd54a8..4a1f0cd1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,7 @@ * Extend: [logging] with profiling log per reqest or regular per request method * New: [logging] option to log response header on debug loglevel * Adjust: [logging] header/content debug log indended by space to be skipped by logwatch +* Improve: remove unnecessary open+read for mtime+size cache ## 3.5.9 * Extend: [auth] add support for type http_remote_user From 1b5d83e1328e7216d7fa2753b77ffd602f32ba22 Mon Sep 17 00:00:00 2001 From: Peter Bieringer Date: Thu, 11 Dec 2025 06:28:50 +0100 Subject: [PATCH 189/290] cosmetics --- radicale/storage/multifilesystem/get.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/radicale/storage/multifilesystem/get.py b/radicale/storage/multifilesystem/get.py index 234bfb3b..ce162d3a 100644 --- a/radicale/storage/multifilesystem/get.py +++ b/radicale/storage/multifilesystem/get.py @@ -114,7 +114,7 @@ class CollectionPartGet(CollectionPartCache, CollectionPartLock, cache_content = self._load_item_cache(href, cache_hash) if cache_content is None: if self._storage._use_mtime_and_size_for_item_cache is True: - # delayed read of the content + # late read of the content if self._storage._debug_cache_actions is True: logger.debug("Item cache late read : %r", path) with open(path, "rb") as f: From cd792ef67c9902422362dfc4b1f4bf2d4a19c817 Mon Sep 17 00:00:00 2001 From: Peter Bieringer Date: Thu, 11 Dec 2025 08:39:53 +0100 Subject: [PATCH 190/290] add selection of XML query flags to request status log line --- radicale/app/__init__.py | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/radicale/app/__init__.py b/radicale/app/__init__.py index 63fe561e..d96de77f 100644 --- a/radicale/app/__init__.py +++ b/radicale/app/__init__.py @@ -275,10 +275,22 @@ class Application(ApplicationPartDelete, ApplicationPartHead, time_delta_seconds = (time_end - time_begin).total_seconds() status_text = "%d %s" % ( status, client.responses.get(status, "Unknown")) + flags = [] + if xml_request is not None: + if "" in xml_request: + flags.append("sync-token") + if "" in xml_request: + flags.append("getctag") + if flags: + flags_text = " (" + " ".join(flags) + ")" + else: + flags_text = "" if answer is not None: - logger.info("%s response status for %r%s in %.3f seconds %s %s bytes: %s", + logger.info("%s response status for %r%s in %.3f seconds %s %s bytes%s: %s", request_method, unsafe_path, depthinfo, - (time_end - time_begin).total_seconds(), content_encoding, str(len(answer)), status_text) + (time_end - time_begin).total_seconds(), content_encoding, str(len(answer)), + flags_text, + status_text) else: logger.info("%s response status for %r%s in %.3f seconds: %s", request_method, unsafe_path, depthinfo, From 225555caa0c3ea2a8dfe242b6c91fa4d92c2c6d6 Mon Sep 17 00:00:00 2001 From: Peter Bieringer Date: Thu, 11 Dec 2025 08:41:03 +0100 Subject: [PATCH 191/290] add support for timing analysis based on logged flags --- contrib/logwatch/radicale | 64 ++++++++++++++++++++++++++++++--------- 1 file changed, 50 insertions(+), 14 deletions(-) diff --git a/contrib/logwatch/radicale b/contrib/logwatch/radicale index 024be3c1..03ffbc48 100644 --- a/contrib/logwatch/radicale +++ b/contrib/logwatch/radicale @@ -80,6 +80,28 @@ sub MaxLength($) { return $length; } +sub ConvertTokens($) { + my %tokens_h; + # unique + foreach my $token (split(" ", $_[0])) { + $tokens_h{$token} = 1; + } + # map tokens + my @result_a; + if (defined $tokens_h{"sync-token"}) { + push @result_a, "ST"; + } + if (defined $tokens_h{"getctag"}) { + push @result_a, "GCT"; + } + # TODO: add potential others which causing long duration + $result = ""; + if (scalar(@result_a) > 0) { + $result = ":F=" . join(",", @result_a); + } + return $result; +} + while (defined($ThisLine = )) { # count loglevel if ( $ThisLine =~ /\[(DEBUG|INFO|WARNING|ERROR|CRITICAL)\] /o ) { @@ -109,6 +131,16 @@ while (defined($ThisLine = )) { $req .= ":R=" . $4; ResponseTimesMinMaxSum($req, $1) if ($Detail >= 10); ResponseSizesMinMaxSum($req, $2, $3) if ($Detail >= 10); + } elsif ( $ThisLine =~ / \S+ response status for .* with depth '(\d)' in ([0-9.]+) seconds (\S+) (\d+) bytes \((.*)\): (\d+)/o ) { + $req .= ":D=" . $1 . ":R=" . $6; + $reqWithFlags = $req . ConvertTokens($5); + ResponseTimesMinMaxSum($reqWithFlags, $2) if ($Detail >= 10); + ResponseSizesMinMaxSum($req, $3, $4) if ($Detail >= 10); + } elsif ( $ThisLine =~ / \S+ response status for .* in ([0-9.]+) seconds (\S+) (\d+) bytes \((.*)\): (\d+)/o ) { + $req .= ":R=" . $5; + $reqWithFlags = $req . ConvertTokens($4); + ResponseTimesMinMaxSum($reqWithFlags, $1) if ($Detail >= 10); + ResponseSizesMinMaxSum($req, $2, $3) if ($Detail >= 10); } $Responses{$req}++; } @@ -192,14 +224,15 @@ if (keys %Requests) { if (keys %Responses) { $sum = Sum(\%Responses); + $length = MaxLength(\%Responses); print "\n**Response result counters ((D= R=)**\n"; - printf "%-18s | %7s | %5s |\n", "Response", "cnt", "ratio"; - print "-" x38 . "\n"; + printf "%-" . $length . "s | %7s | %5s |\n", "Response", "cnt", "ratio"; + print "-" x($length + 20) . "\n"; foreach my $req (sort keys %Responses) { - printf "%-18s | %7d | %3d%% |\n", $req, $Responses{$req}, int(($Responses{$req} * 100) / $sum); + printf "%-" . $length . "s | %7d | %3d%% |\n", $req, $Responses{$req}, int(($Responses{$req} * 100) / $sum); } - print "-" x38 . "\n"; - printf "%-18s | %7d | %3d%% |\n", "", $sum, 100; + print "-" x($length + 20) . "\n"; + printf "%-" . $length . "s | %7d | %3d%% |\n", "", $sum, 100; } if (keys %Logins) { @@ -216,32 +249,35 @@ if (keys %Logins) { } if (keys %ResponseTimes) { - print "\n**Response timings (counts, seconds) (D= R=)**\n"; - printf "%-18s | %7s | %7s | %7s | %7s |\n", "Response", "cnt", "min", "max", "avg"; - print "-" x60 . "\n"; + $length = MaxLength(\%ResponseTimes); + print "\n**Response timings (counts, seconds) (D= R= F=)**\n"; + print "* Flags: ST:sync-token GCT:getctag\n"; + printf "%-" . $length . "s | %7s | %7s | %7s | %7s |\n", "Response", "cnt", "min", "max", "avg"; + print "-" x($length + 42) . "\n"; foreach my $req (sort keys %ResponseTimes) { - printf "%-18s | %7d | %7.3f | %7.3f | %7.3f |\n", $req + printf "%-" . $length . "s | %7d | %7.3f | %7.3f | %7.3f |\n", $req , $ResponseTimes{$req}->{'cnt'} , $ResponseTimes{$req}->{'min'} , $ResponseTimes{$req}->{'max'} , $ResponseTimes{$req}->{'sum'} / $ResponseTimes{$req}->{'cnt'}; } - print "-" x60 . "\n"; + print "-" x($length + 42) . "\n"; } if (keys %ResponseSizes) { for my $type (sort keys %ResponseSizes) { + $length = MaxLength($ResponseSizes{$type}); print "\n**Response sizes (counts, bytes: $type) (D= R=)**\n"; - printf "%-18s | %7s | %9s | %9s | %9s |\n", "Response", "cnt", "min", "max", "avg"; - print "-" x66 . "\n"; + printf "%-" . $length . "s | %7s | %9s | %9s | %9s |\n", "Response", "cnt", "min", "max", "avg"; + print "-" x($length + 48) . "\n"; foreach my $req (sort keys %{$ResponseSizes{$type}}) { - printf "%-18s | %7d | %9d | %9d | %9d |\n", $req + printf "%-" . $length . "s | %7d | %9d | %9d | %9d |\n", $req , $ResponseSizes{$type}->{$req}->{'cnt'} , $ResponseSizes{$type}->{$req}->{'min'} , $ResponseSizes{$type}->{$req}->{'max'} , $ResponseSizes{$type}->{$req}->{'sum'} / $ResponseSizes{$type}->{$req}->{'cnt'}; } - print "-" x66 . "\n"; + print "-" x($length + 48) . "\n"; } } From 4e0008b5e414af9b962c2432b1db7440c8fb0fa6 Mon Sep 17 00:00:00 2001 From: Peter Bieringer Date: Thu, 11 Dec 2025 08:43:07 +0100 Subject: [PATCH 192/290] changelog for this log extension --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 4a1f0cd1..3609a39e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,7 @@ * New: [logging] option to log response header on debug loglevel * Adjust: [logging] header/content debug log indended by space to be skipped by logwatch * Improve: remove unnecessary open+read for mtime+size cache +* Extend: add selected XML query properties to request result log line for improved timing analysis incl. logwatch support ## 3.5.9 * Extend: [auth] add support for type http_remote_user From b3a65627e4243c184b5d01f63acf35baeceb79c8 Mon Sep 17 00:00:00 2001 From: Peter Bieringer Date: Thu, 11 Dec 2025 08:46:22 +0100 Subject: [PATCH 193/290] enrich class with max-content-length --- radicale/app/base.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/radicale/app/base.py b/radicale/app/base.py index 18a68b95..a5536cce 100644 --- a/radicale/app/base.py +++ b/radicale/app/base.py @@ -39,6 +39,7 @@ class ApplicationBase: _rights: rights.BaseRights _web: web.BaseWeb _encoding: str + _max_content_length: int _permit_delete_collection: bool _permit_overwrite_collection: bool _strict_preconditions: bool @@ -51,6 +52,7 @@ class ApplicationBase: self._rights = rights.load(configuration) self._web = web.load(configuration) self._encoding = configuration.get("encoding", "request") + self._max_content_length = configuration.get("server", "max_content_length") self._log_bad_put_request_content = configuration.get("logging", "bad_put_request_content") self._response_content_on_debug = configuration.get("logging", "response_content_on_debug") self._request_content_on_debug = configuration.get("logging", "request_content_on_debug") From fcfbde9608e2a1840e84f45c0c57f65bcb23db32 Mon Sep 17 00:00:00 2001 From: Peter Bieringer Date: Thu, 11 Dec 2025 08:47:01 +0100 Subject: [PATCH 194/290] feed max-content-length into functions --- radicale/app/propfind.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/radicale/app/propfind.py b/radicale/app/propfind.py index d79d0269..90f94a0d 100644 --- a/radicale/app/propfind.py +++ b/radicale/app/propfind.py @@ -34,7 +34,7 @@ from radicale.log import logger def xml_propfind(base_prefix: str, path: str, xml_request: Optional[ET.Element], allowed_items: Iterable[Tuple[types.CollectionOrItem, str]], - user: str, encoding: str) -> Optional[ET.Element]: + user: str, encoding: str, max_content_length: int) -> Optional[ET.Element]: """Read and answer PROPFIND requests. Read rfc4918-9.1 for info. @@ -71,14 +71,14 @@ def xml_propfind(base_prefix: str, path: str, write = permission == "w" multistatus.append(xml_propfind_response( base_prefix, path, item, props, user, encoding, write=write, - allprop=allprop, propname=propname)) + allprop=allprop, propname=propname, max_content_length=max_content_length)) return multistatus def xml_propfind_response( base_prefix: str, path: str, item: types.CollectionOrItem, - props: Sequence[str], user: str, encoding: str, write: bool = False, + props: Sequence[str], user: str, encoding: str, max_content_length: int, write: bool = False, propname: bool = False, allprop: bool = False) -> ET.Element: """Build and return a PROPFIND response.""" if propname and allprop or (props and (propname or allprop)): @@ -407,7 +407,7 @@ class ApplicationPartPropfind(ApplicationBase): headers = {"DAV": httputils.DAV_HEADERS, "Content-Type": "text/xml; charset=%s" % self._encoding} xml_answer = xml_propfind(base_prefix, path, xml_content, - allowed_items, user, self._encoding) + allowed_items, user, self._encoding, max_content_length=self._max_content_length) if xml_answer is None: return httputils.NOT_ALLOWED return client.MULTI_STATUS, headers, self._xml_response(xml_answer), xmlutils.pretty_xml(xml_content) From dd153df6cbde47c4931b9f0b19f457a50644374a Mon Sep 17 00:00:00 2001 From: Peter Bieringer Date: Thu, 11 Dec 2025 08:47:26 +0100 Subject: [PATCH 195/290] add support for max-resource-size using max-content-length --- radicale/app/propfind.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/radicale/app/propfind.py b/radicale/app/propfind.py index 90f94a0d..7daf49e1 100644 --- a/radicale/app/propfind.py +++ b/radicale/app/propfind.py @@ -111,6 +111,7 @@ def xml_propfind_response( props.append(xmlutils.make_clark("D:supported-report-set")) props.append(xmlutils.make_clark("D:resourcetype")) props.append(xmlutils.make_clark("D:owner")) + props.append(xmlutils.make_clark("C:max-resource-size")) if is_collection and collection.is_principal: props.append(xmlutils.make_clark("C:calendar-user-address-set")) @@ -239,6 +240,8 @@ def xml_propfind_response( child_element.text = xmlutils.make_href( base_prefix, "/%s/" % collection.owner) element.append(child_element) + elif tag == xmlutils.make_clark("C:max-resource-size"): + element.text = str(max_content_length) elif is_collection: if tag == xmlutils.make_clark("D:getcontenttype"): if is_leaf: From c3dfd5584da82517967233e0b2acbeff17c0fa2e Mon Sep 17 00:00:00 2001 From: Peter Bieringer Date: Thu, 11 Dec 2025 08:47:49 +0100 Subject: [PATCH 196/290] add test case for max-resource-size and getctag --- radicale/tests/test_base.py | 34 ++++++++++++++++++++++++++++++++++ 1 file changed, 34 insertions(+) diff --git a/radicale/tests/test_base.py b/radicale/tests/test_base.py index afcca1b2..8cf05340 100644 --- a/radicale/tests/test_base.py +++ b/radicale/tests/test_base.py @@ -692,6 +692,40 @@ permissions: RrWw""") status, prop = response["ICAL:calendar-color"] assert status == 404 and not prop.text + def test_propfind_max_resource_size(self) -> None: + """Read property C:max-resource-size""" + self.mkcalendar("/calendar.ics/") + event = get_file_content("event1.ics") + self.put("/calendar.ics/event.ics", event) + _, responses = self.propfind("/calendar.ics/", """\ + + + + + + """) + response = responses["/calendar.ics/"] + assert not isinstance(response, int) + status, prop = response["C:max-resource-size"] + assert status == 200 and prop.text + + def test_propfind_getctag(self) -> None: + """Read property CS:getctag""" + self.mkcalendar("/calendar.ics/") + event = get_file_content("event1.ics") + self.put("/calendar.ics/event.ics", event) + _, responses = self.propfind("/calendar.ics/", """\ + + + + + +""") + response = responses["/calendar.ics/"] + assert not isinstance(response, int) + status, prop = response["CS:getctag"] + assert status == 200 and prop.text + def test_proppatch(self) -> None: """Set/Remove a property and read it back.""" self.mkcalendar("/calendar.ics/") From 6e3b277ff54a00e048747801734c7d3767c3227e Mon Sep 17 00:00:00 2001 From: Peter Bieringer Date: Thu, 11 Dec 2025 08:59:44 +0100 Subject: [PATCH 197/290] reduce to 80% for base64 coverage --- radicale/app/propfind.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/radicale/app/propfind.py b/radicale/app/propfind.py index 7daf49e1..4d985429 100644 --- a/radicale/app/propfind.py +++ b/radicale/app/propfind.py @@ -241,7 +241,8 @@ def xml_propfind_response( base_prefix, "/%s/" % collection.owner) element.append(child_element) elif tag == xmlutils.make_clark("C:max-resource-size"): - element.text = str(max_content_length) + # RFC4791#5.2.5 use 80% of max_content_length to cover base64 encoding + element.text = str(int(max_content_length * 0.8)) elif is_collection: if tag == xmlutils.make_clark("D:getcontenttype"): if is_leaf: From ebdd3aac704cd1f49055b8d48e8c7d5e273b4ef9 Mon Sep 17 00:00:00 2001 From: Peter Bieringer Date: Thu, 11 Dec 2025 09:00:49 +0100 Subject: [PATCH 198/290] changelog for add support for max-resource-size using max-content-length --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 3609a39e..79ae1c43 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,6 +10,7 @@ * Adjust: [logging] header/content debug log indended by space to be skipped by logwatch * Improve: remove unnecessary open+read for mtime+size cache * Extend: add selected XML query properties to request result log line for improved timing analysis incl. logwatch support +* Add: support PROPFIND/max-resource-size by 80% of max_content_length option ## 3.5.9 * Extend: [auth] add support for type http_remote_user From b51dcccd19cac724e1808ca961161ca2c3907ffa Mon Sep 17 00:00:00 2001 From: Peter Bieringer Date: Thu, 11 Dec 2025 22:44:19 +0100 Subject: [PATCH 199/290] improve type specification --- radicale/config.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/radicale/config.py b/radicale/config.py index abe7e465..01e5d6f8 100644 --- a/radicale/config.py +++ b/radicale/config.py @@ -600,7 +600,7 @@ This is an automated message. Please do not reply.""", ("profiling_per_request_min_duration", { "value": "3", "help": "log profiling data per request minimum duration (seconds)", - "type": int}), + "type": positive_int}), ("profiling_per_request_header", { "value": "False", "help": "Log profiling request body (if passing minimum duration)", @@ -612,11 +612,11 @@ This is an automated message. Please do not reply.""", ("profiling_per_request_method_interval", { "value": "600", "help": "log profiling data per request method interval (seconds)", - "type": int}), + "type": positive_int}), ("profiling_top_x_functions", { "value": "10", "help": "log profiling top X functions (limit)", - "type": int}), + "type": positive_int}), ("mask_passwords", { "value": "True", "help": "mask passwords in logs", From e66d83cd849e4281426ad87ae960cfb0f50bbf98 Mon Sep 17 00:00:00 2001 From: Peter Bieringer Date: Thu, 11 Dec 2025 22:50:35 +0100 Subject: [PATCH 200/290] add explanation to default value --- DOCUMENTATION.md | 4 ++-- config | 2 +- radicale/config.py | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/DOCUMENTATION.md b/DOCUMENTATION.md index 0a8760df..5936147a 100644 --- a/DOCUMENTATION.md +++ b/DOCUMENTATION.md @@ -797,9 +797,9 @@ Default: `8` The maximum size of the request body. (bytes) -Default: `100000000` +Default: `100000000` (100 Mbyte) -In case of using a reverse proxy in front of check also there related option +In case of using a reverse proxy in front of check also there related option. ##### timeout diff --git a/config b/config index 75a314e8..bca1b4fa 100644 --- a/config +++ b/config @@ -21,7 +21,7 @@ # Max parallel connections #max_connections = 8 -# Max size of request body (bytes) +# Max size of request body (bytes), default: 100 Mbyte # In case of using a reverse proxy in front of check also there related option #max_content_length = 100000000 diff --git a/radicale/config.py b/radicale/config.py index 01e5d6f8..fb548042 100644 --- a/radicale/config.py +++ b/radicale/config.py @@ -162,7 +162,7 @@ DEFAULT_CONFIG_SCHEMA: types.CONFIG_SCHEMA = OrderedDict([ "type": positive_int}), ("max_content_length", { "value": "100000000", - "help": "maximum size of request body in bytes", + "help": "maximum size of request body in bytes (default: 100 Mbyte)", "type": positive_int}), ("timeout", { "value": "30", From bc2a14481f72cd4d4eada6b1c926442709dd13c9 Mon Sep 17 00:00:00 2001 From: Peter Bieringer Date: Thu, 11 Dec 2025 22:53:02 +0100 Subject: [PATCH 201/290] add new option max_resource_size --- DOCUMENTATION.md | 12 ++++++++++++ config | 5 +++++ radicale/config.py | 4 ++++ 3 files changed, 21 insertions(+) diff --git a/DOCUMENTATION.md b/DOCUMENTATION.md index 5936147a..daeedb98 100644 --- a/DOCUMENTATION.md +++ b/DOCUMENTATION.md @@ -801,6 +801,18 @@ Default: `100000000` (100 Mbyte) In case of using a reverse proxy in front of check also there related option. +##### max_resource_size + +_(>= 3.5.10)_ + +The maximum size of a resource. (bytes) + +Default: `10000000` (10 Mbyte) + +Limited to 80% of max_content_length to cover plain base64 encoded payload. + +Announced to clients requesting "max-resource-size" via PROPFIND. + ##### timeout Socket timeout. (seconds) diff --git a/config b/config index bca1b4fa..2f85c4e0 100644 --- a/config +++ b/config @@ -25,6 +25,11 @@ # In case of using a reverse proxy in front of check also there related option #max_content_length = 100000000 +# Max resource size (bytes), default: 10 Mbyte +# Limited to 80% of max_content_length to cover plain base64 encoded payload +# Announced to clients requesting "max-resource-size" via PROPFIND +#max_ressource_size = 10000000 + # Socket timeout (seconds) #timeout = 30 diff --git a/radicale/config.py b/radicale/config.py index fb548042..519269bc 100644 --- a/radicale/config.py +++ b/radicale/config.py @@ -164,6 +164,10 @@ DEFAULT_CONFIG_SCHEMA: types.CONFIG_SCHEMA = OrderedDict([ "value": "100000000", "help": "maximum size of request body in bytes (default: 100 Mbyte)", "type": positive_int}), + ("max_resource_size", { + "value": "10000000", + "help": "maximum size of resource (default: 10 Mbyte)", + "type": positive_int}), ("timeout", { "value": "30", "help": "socket timeout", From e1b0721f9ec7562e343ee8c3c44c96791a2d6125 Mon Sep 17 00:00:00 2001 From: Peter Bieringer Date: Thu, 11 Dec 2025 22:55:41 +0100 Subject: [PATCH 202/290] change to dedicated option for propfind/max_resource_size --- radicale/app/__init__.py | 9 +++++++++ radicale/app/base.py | 3 +-- radicale/app/propfind.py | 12 ++++++------ 3 files changed, 16 insertions(+), 8 deletions(-) diff --git a/radicale/app/__init__.py b/radicale/app/__init__.py index d96de77f..7d02c2f8 100644 --- a/radicale/app/__init__.py +++ b/radicale/app/__init__.py @@ -73,6 +73,7 @@ class Application(ApplicationPartDelete, ApplicationPartHead, _auth_delay: float _internal_server: bool _max_content_length: int + _max_resource_size: int _auth_realm: str _auth_type: str _web_type: str @@ -95,6 +96,14 @@ class Application(ApplicationPartDelete, ApplicationPartHead, """ super().__init__(configuration) self._mask_passwords = configuration.get("logging", "mask_passwords") + self._max_content_length = configuration.get("server", "max_content_length") + self._max_resource_size = configuration.get("server", "max_resource_size") + if (self._max_resource_size > (self._max_content_length * 0.8)): + max_resource_size_limited = int(self._max_content_length * 0.8) + logger.warning("max_resource_size capped to: %d bytes (from %d to 80%% of max_content_length %d)", max_resource_size_limited, self._max_resource_size, self._max_content_length) + self._max_resource_size = max_resource_size_limited + else: + logger.info("max_resource_size set to: %d bytes", self._max_resource_size) self._bad_put_request_content = configuration.get("logging", "bad_put_request_content") logger.info("log bad put request content: %s", self._bad_put_request_content) self._request_header_on_debug = configuration.get("logging", "request_header_on_debug") diff --git a/radicale/app/base.py b/radicale/app/base.py index a5536cce..aa0af7a2 100644 --- a/radicale/app/base.py +++ b/radicale/app/base.py @@ -39,7 +39,7 @@ class ApplicationBase: _rights: rights.BaseRights _web: web.BaseWeb _encoding: str - _max_content_length: int + _max_resource_size: int _permit_delete_collection: bool _permit_overwrite_collection: bool _strict_preconditions: bool @@ -52,7 +52,6 @@ class ApplicationBase: self._rights = rights.load(configuration) self._web = web.load(configuration) self._encoding = configuration.get("encoding", "request") - self._max_content_length = configuration.get("server", "max_content_length") self._log_bad_put_request_content = configuration.get("logging", "bad_put_request_content") self._response_content_on_debug = configuration.get("logging", "response_content_on_debug") self._request_content_on_debug = configuration.get("logging", "request_content_on_debug") diff --git a/radicale/app/propfind.py b/radicale/app/propfind.py index 4d985429..2ef89315 100644 --- a/radicale/app/propfind.py +++ b/radicale/app/propfind.py @@ -34,7 +34,7 @@ from radicale.log import logger def xml_propfind(base_prefix: str, path: str, xml_request: Optional[ET.Element], allowed_items: Iterable[Tuple[types.CollectionOrItem, str]], - user: str, encoding: str, max_content_length: int) -> Optional[ET.Element]: + user: str, encoding: str, max_resource_size: int) -> Optional[ET.Element]: """Read and answer PROPFIND requests. Read rfc4918-9.1 for info. @@ -71,14 +71,14 @@ def xml_propfind(base_prefix: str, path: str, write = permission == "w" multistatus.append(xml_propfind_response( base_prefix, path, item, props, user, encoding, write=write, - allprop=allprop, propname=propname, max_content_length=max_content_length)) + allprop=allprop, propname=propname, max_resource_size=max_resource_size)) return multistatus def xml_propfind_response( base_prefix: str, path: str, item: types.CollectionOrItem, - props: Sequence[str], user: str, encoding: str, max_content_length: int, write: bool = False, + props: Sequence[str], user: str, encoding: str, max_resource_size: int, write: bool = False, propname: bool = False, allprop: bool = False) -> ET.Element: """Build and return a PROPFIND response.""" if propname and allprop or (props and (propname or allprop)): @@ -241,8 +241,8 @@ def xml_propfind_response( base_prefix, "/%s/" % collection.owner) element.append(child_element) elif tag == xmlutils.make_clark("C:max-resource-size"): - # RFC4791#5.2.5 use 80% of max_content_length to cover base64 encoding - element.text = str(int(max_content_length * 0.8)) + # RFC4791#5.2.5 + element.text = str(max_resource_size) elif is_collection: if tag == xmlutils.make_clark("D:getcontenttype"): if is_leaf: @@ -411,7 +411,7 @@ class ApplicationPartPropfind(ApplicationBase): headers = {"DAV": httputils.DAV_HEADERS, "Content-Type": "text/xml; charset=%s" % self._encoding} xml_answer = xml_propfind(base_prefix, path, xml_content, - allowed_items, user, self._encoding, max_content_length=self._max_content_length) + allowed_items, user, self._encoding, max_resource_size=self._max_resource_size) if xml_answer is None: return httputils.NOT_ALLOWED return client.MULTI_STATUS, headers, self._xml_response(xml_answer), xmlutils.pretty_xml(xml_content) From edd6195b6b9d0a0714eca8ad19df5268a8f79a9e Mon Sep 17 00:00:00 2001 From: Peter Bieringer Date: Thu, 11 Dec 2025 23:07:58 +0100 Subject: [PATCH 203/290] add support for max-resource-size check on PUT --- radicale/app/put.py | 39 +++++++++++++++++++++++++++++++++------ 1 file changed, 33 insertions(+), 6 deletions(-) diff --git a/radicale/app/put.py b/radicale/app/put.py index bd049158..1014f95b 100644 --- a/radicale/app/put.py +++ b/radicale/app/put.py @@ -46,7 +46,7 @@ PRODID = u"-//Radicale//NONSGML Version " + utils.package_version("radicale") + def prepare(vobject_items: List[vobject.base.Component], path: str, - content_type: str, permission: bool, parent_permission: bool, + content_type: str, permission: bool, parent_permission: bool, max_resource_size: int, tag: Optional[str] = None, write_whole_collection: Optional[bool] = None) -> Tuple[ Iterator[radicale_item.Item], # items @@ -103,6 +103,13 @@ def prepare(vobject_items: List[vobject.base.Component], path: str, else: logger.warning("Problem during prepare item with UID '%s' (content suppressed in this loglevel): %s", item.uid, e) raise + size = len(item.serialize()) + if (size > max_resource_size): + logger.warning("PUT request contains item with UID %r size %d > limit %d: %r", item.uid, size, max_resource_size, path) + # Use OverflowError as flag for max_resource_size + raise OverflowError + else: + logger.debug("PUT request contains item with UID %r size %d <= limit %d: %r", item.uid, size, max_resource_size, path) items.append(item) elif write_whole_collection and tag == "VADDRESSBOOK": for vobject_item in vobject_items: @@ -121,12 +128,26 @@ def prepare(vobject_items: List[vobject.base.Component], path: str, else: logger.warning("Problem during prepare item with UID '%s' (content suppressed in this loglevel): %s", item.uid, e) raise + size = len(item.serialize()) + if (size > max_resource_size): + logger.warning("PUT request contains item with UID %r size %d > limit %d: %r", item.uid, size, max_resource_size, path) + # Use OverflowError as flag for max_resource_size + raise OverflowError + else: + logger.debug("PUT request contains item with UID %r size %d <= limit %d: %r", item.uid, size, max_resource_size, path) items.append(item) elif not write_whole_collection: vobject_item, = vobject_items item = radicale_item.Item(collection_path=collection_path, vobject_item=vobject_item) item.prepare() + size = len(item.serialize()) + if (size > max_resource_size): + logger.warning("PUT request contains item with UID %r size %d above limit %d: %r", item.uid, size, max_resource_size, path) + # Use OverflowError as flag for max_resource_size + raise OverflowError + else: + logger.debug("PUT request contains item with UID %r size %d below limit %d: %r", item.uid, size, max_resource_size, path) items.append(item) if write_whole_collection: @@ -188,7 +209,8 @@ class ApplicationPartPut(ApplicationBase): prepared_props, prepared_exc_info) = prepare( vobject_items, path, content_type, bool(rights.intersect(access.permissions, "Ww")), - bool(rights.intersect(access.parent_permissions, "w"))) + bool(rights.intersect(access.parent_permissions, "w")), + self._max_resource_size) with self._storage.acquire_lock("w", user, path=path, request="PUT"): item = next(iter(self._storage.discover(path)), None) @@ -252,13 +274,18 @@ class ApplicationPartPut(ApplicationBase): vobject_items, path, content_type, bool(rights.intersect(access.permissions, "Ww")), bool(rights.intersect(access.parent_permissions, "w")), + self._max_resource_size, tag, write_whole_collection) props = prepared_props if prepared_exc_info: - logger.warning( - "Bad PUT request on %r (prepare): %s", path, prepared_exc_info[1], - exc_info=prepared_exc_info) - return httputils.BAD_REQUEST + # Use OverflowError as flag for max_resource_size + if prepared_exc_info[0] == OverflowError: + return httputils.PRECONDITION_FAILED + else: + logger.warning( + "Bad PUT request on %r (prepare): %s", path, prepared_exc_info[1], + exc_info=prepared_exc_info) + return httputils.BAD_REQUEST if write_whole_collection: try: From 9a31087dfa1bdf8deecf763154375256dfa65318 Mon Sep 17 00:00:00 2001 From: Peter Bieringer Date: Thu, 11 Dec 2025 23:08:21 +0100 Subject: [PATCH 204/290] add testcases for max-resource-size --- radicale/tests/test_base.py | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/radicale/tests/test_base.py b/radicale/tests/test_base.py index 8cf05340..158d228b 100644 --- a/radicale/tests/test_base.py +++ b/radicale/tests/test_base.py @@ -142,6 +142,22 @@ permissions: RrWw""") assert "Event" in answer assert "UID:event" in answer + def test_add_event_exceed_size(self) -> None: + """Add an event which is exceeding max-resource-size.""" + self.configure({"server": {"max_resource_size": 20}}) + self.mkcalendar("/calendar.ics/") + event = get_file_content("event1.ics") + path = "/calendar.ics/event1.ics" + self.put(path, event, check=412) + + def test_add_events_exceed_size(self) -> None: + """Add multipe events where last is exceeding max-resource-size.""" + self.configure({"server": {"max_resource_size": 603}}) + self.mkcalendar("/calendar.ics/") + event = get_file_content("event_multiple3.ics") + path = "/calendar.ics/" + self.put(path, event, check=412) + def test_add_event_broken(self) -> None: """Add a broken event.""" self.mkcalendar("/calendar.ics/") From e444454b848696b4de103fbf39ef55e90f74ac35 Mon Sep 17 00:00:00 2001 From: Peter Bieringer Date: Thu, 11 Dec 2025 23:12:02 +0100 Subject: [PATCH 205/290] extend changelog --- CHANGELOG.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 79ae1c43..c0da9c33 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,7 +10,8 @@ * Adjust: [logging] header/content debug log indended by space to be skipped by logwatch * Improve: remove unnecessary open+read for mtime+size cache * Extend: add selected XML query properties to request result log line for improved timing analysis incl. logwatch support -* Add: support PROPFIND/max-resource-size by 80% of max_content_length option +* Add: [server] max_resource_size option +* Add: support PROPFIND/max-resource-size by max_resource_size (capped to 80% of max_content_length) ## 3.5.9 * Extend: [auth] add support for type http_remote_user From c1f39d24f7989ed0905757bf4b404dca6e1f2b90 Mon Sep 17 00:00:00 2001 From: Peter Bieringer Date: Thu, 11 Dec 2025 23:14:08 +0100 Subject: [PATCH 206/290] add new test ics --- radicale/tests/static/event_multiple3.ics | 40 +++++++++++++++++++++++ 1 file changed, 40 insertions(+) create mode 100644 radicale/tests/static/event_multiple3.ics diff --git a/radicale/tests/static/event_multiple3.ics b/radicale/tests/static/event_multiple3.ics new file mode 100644 index 00000000..c8275933 --- /dev/null +++ b/radicale/tests/static/event_multiple3.ics @@ -0,0 +1,40 @@ +BEGIN:VCALENDAR +PRODID:-//Mozilla.org/NONSGML Mozilla Calendar V1.1//EN +VERSION:2.0 +BEGIN:VTIMEZONE +TZID:Europe/Paris +X-LIC-LOCATION:Europe/Paris +BEGIN:DAYLIGHT +TZOFFSETFROM:+0100 +TZOFFSETTO:+0200 +TZNAME:CEST +DTSTART:19700329T020000 +RRULE:FREQ=YEARLY;BYDAY=-1SU;BYMONTH=3 +END:DAYLIGHT +BEGIN:STANDARD +TZOFFSETFROM:+0200 +TZOFFSETTO:+0100 +TZNAME:CET +DTSTART:19701025T030000 +RRULE:FREQ=YEARLY;BYDAY=-1SU;BYMONTH=10 +END:STANDARD +END:VTIMEZONE +BEGIN:VEVENT +UID:event +SUMMARY:Event +DTSTART;TZID=Europe/Paris:20130901T190000 +DTEND;TZID=Europe/Paris:20130901T200000 +END:VEVENT +BEGIN:VTODO +UID:todo +DTSTART;TZID=Europe/Paris:20130901T220000 +DURATION:PT1H +SUMMARY:Todo +END:VTODO +BEGIN:VEVENT +UID:event2 +SUMMARY:Event-with-longer-description +DTSTART;TZID=Europe/Paris:20130901T190000 +DTEND;TZID=Europe/Paris:20130901T200000 +END:VEVENT +END:VCALENDAR From a7e9521b381dc7c5511dd2ee4f48b36802703d9c Mon Sep 17 00:00:00 2001 From: Peter Bieringer Date: Fri, 12 Dec 2025 08:49:13 +0100 Subject: [PATCH 207/290] honor RFC4791#5.2.5: SHOULD NOT be returned by a PROPFIND DAV:allprop request --- radicale/app/propfind.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/radicale/app/propfind.py b/radicale/app/propfind.py index 2ef89315..3976af91 100644 --- a/radicale/app/propfind.py +++ b/radicale/app/propfind.py @@ -111,7 +111,9 @@ def xml_propfind_response( props.append(xmlutils.make_clark("D:supported-report-set")) props.append(xmlutils.make_clark("D:resourcetype")) props.append(xmlutils.make_clark("D:owner")) - props.append(xmlutils.make_clark("C:max-resource-size")) + if not allprop: + # RFC4791#5.2.5: SHOULD NOT be returned by a PROPFIND DAV:allprop request + props.append(xmlutils.make_clark("C:max-resource-size")) if is_collection and collection.is_principal: props.append(xmlutils.make_clark("C:calendar-user-address-set")) From 81b3c2873737fc00b2426d9d81795fe389ab39b2 Mon Sep 17 00:00:00 2001 From: Peter Bieringer Date: Fri, 12 Dec 2025 08:49:23 +0100 Subject: [PATCH 208/290] test RFC4791#5.2.5: SHOULD NOT be returned by a PROPFIND DAV:allprop request --- radicale/tests/test_base.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/radicale/tests/test_base.py b/radicale/tests/test_base.py index 158d228b..c1d4e5ab 100644 --- a/radicale/tests/test_base.py +++ b/radicale/tests/test_base.py @@ -692,11 +692,13 @@ permissions: RrWw""") assert not isinstance(response, int) status, prop = response["D:sync-token"] assert status == 200 and prop.text + assert "C:max-resource-size" not in response _, responses = self.propfind("/calendar.ics/event.ics", propfind) response = responses["/calendar.ics/event.ics"] assert not isinstance(response, int) status, prop = response["D:getetag"] assert status == 200 and prop.text + assert "C:max-resource-size" not in response def test_propfind_nonexistent(self) -> None: """Read a property that does not exist.""" From 2411b6385202d500a9e57afcbb9abe5b6d1faf1c Mon Sep 17 00:00:00 2001 From: Peter Bieringer Date: Sat, 13 Dec 2025 07:59:02 +0100 Subject: [PATCH 209/290] add function for calculating units for integers --- radicale/utils.py | 49 +++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 49 insertions(+) diff --git a/radicale/utils.py b/radicale/utils.py index 16540bde..eee80c99 100644 --- a/radicale/utils.py +++ b/radicale/utils.py @@ -53,6 +53,15 @@ DATETIME_MAX_UNIXTIME: int = (datetime.MAXYEAR - 1970) * 365 * 24 * 60 * 60 DATETIME_MIN_UNIXTIME: int = (datetime.MINYEAR - 1970) * 365 * 24 * 60 * 60 +# Number units +UNIT_g: int = (1000 * 1000 * 1000) +UNIT_m: int = (1000 * 1000) +UNIT_k: int = (1000) +UNIT_G: int = (1024 * 1024 * 1024) +UNIT_M: int = (1024 * 1024) +UNIT_K: int = (1024) + + def load_plugin(internal_types: Sequence[str], module_name: str, class_name: str, base_class: Type[_T_co], configuration: "config.Configuration") -> _T_co: @@ -294,6 +303,46 @@ def format_ut(unixtime: int) -> str: return r +def format_int(value: int, binary: bool = False) -> str: + if binary: + if value > UNIT_G: + value = value / UNIT_G + unit = "G" + elif value > UNIT_M: + value = value / UNIT_M + unit = "M" + elif value > UNIT_K: + value = value / UNIT_K + unit = "K" + else: + unit = "" + else: + if value > UNIT_g: + value = value / UNIT_g + unit = "g" + elif value > UNIT_m: + value = value / UNIT_m + unit = "m" + elif value > UNIT_k: + value = value / UNIT_k + unit = "k" + else: + unit = "" + return ("%.1f %s" % (value, unit)) + + if unixtime <= DATETIME_MIN_UNIXTIME: + r = str(unixtime) + "(<=MIN:" + str(DATETIME_MIN_UNIXTIME) + ")" + elif unixtime >= DATETIME_MAX_UNIXTIME: + r = str(unixtime) + "(>=MAX:" + str(DATETIME_MAX_UNIXTIME) + ")" + else: + if sys.version_info < (3, 11): + dt = datetime.datetime.utcfromtimestamp(unixtime) + else: + dt = datetime.datetime.fromtimestamp(unixtime, datetime.UTC) + r = str(unixtime) + "(" + dt.strftime('%Y-%m-%dT%H:%M:%SZ') + ")" + return r + + def limit_str(content: str, limit: int) -> str: length = len(content) if limit > 0 and length >= limit: From b8ed32f3ee6030e302d5f3fc1d38a50cb9397006 Mon Sep 17 00:00:00 2001 From: Peter Bieringer Date: Sat, 13 Dec 2025 07:59:28 +0100 Subject: [PATCH 210/290] log max-content-length on startup, add formatted int to raw values --- radicale/app/__init__.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/radicale/app/__init__.py b/radicale/app/__init__.py index 7d02c2f8..48b7b7f3 100644 --- a/radicale/app/__init__.py +++ b/radicale/app/__init__.py @@ -98,12 +98,13 @@ class Application(ApplicationPartDelete, ApplicationPartHead, self._mask_passwords = configuration.get("logging", "mask_passwords") self._max_content_length = configuration.get("server", "max_content_length") self._max_resource_size = configuration.get("server", "max_resource_size") + logger.info("max_content_length to: %d bytes (%sbytes)", self._max_content_length, utils.format_int(self._max_content_length, binary=True)) if (self._max_resource_size > (self._max_content_length * 0.8)): max_resource_size_limited = int(self._max_content_length * 0.8) - logger.warning("max_resource_size capped to: %d bytes (from %d to 80%% of max_content_length %d)", max_resource_size_limited, self._max_resource_size, self._max_content_length) + logger.warning("max_resource_size capped to: %d bytes (%sbytes) (from %d to 80%% of max_content_length)", max_resource_size_limited, utils.format_int(max_resource_size_limited, binary=True), self._max_resource_size) self._max_resource_size = max_resource_size_limited else: - logger.info("max_resource_size set to: %d bytes", self._max_resource_size) + logger.info("max_resource_size set to: %d bytes (%sbytes)", self._max_resource_size, utils.format_int(self._max_resource_size, binary=True)) self._bad_put_request_content = configuration.get("logging", "bad_put_request_content") logger.info("log bad put request content: %s", self._bad_put_request_content) self._request_header_on_debug = configuration.get("logging", "request_header_on_debug") From 690914e49f86d6b7af029f93621bfbdfe00d182e Mon Sep 17 00:00:00 2001 From: Peter Bieringer Date: Sat, 13 Dec 2025 08:00:06 +0100 Subject: [PATCH 211/290] add unit --- radicale/auth/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/radicale/auth/__init__.py b/radicale/auth/__init__.py index 7eadb18a..9114a9f0 100644 --- a/radicale/auth/__init__.py +++ b/radicale/auth/__init__.py @@ -142,7 +142,7 @@ class BaseAuth: if self._lc_username is True and self._uc_username is True: raise RuntimeError("auth.lc_username and auth.uc_username cannot be enabled together") self._auth_delay = configuration.get("auth", "delay") - logger.info("auth.delay: %f", self._auth_delay) + logger.info("auth.delay: %f seconds", self._auth_delay) self._failed_auth_delay = 0 self._lock = threading.Lock() # cache_successful_logins From c73a53d815c91a73d456d9bcf6ae2940afbbb351 Mon Sep 17 00:00:00 2001 From: Peter Bieringer Date: Sat, 13 Dec 2025 08:06:44 +0100 Subject: [PATCH 212/290] remove copy-paste overhead --- radicale/utils.py | 12 ------------ 1 file changed, 12 deletions(-) diff --git a/radicale/utils.py b/radicale/utils.py index eee80c99..14c0f0f3 100644 --- a/radicale/utils.py +++ b/radicale/utils.py @@ -330,18 +330,6 @@ def format_int(value: int, binary: bool = False) -> str: unit = "" return ("%.1f %s" % (value, unit)) - if unixtime <= DATETIME_MIN_UNIXTIME: - r = str(unixtime) + "(<=MIN:" + str(DATETIME_MIN_UNIXTIME) + ")" - elif unixtime >= DATETIME_MAX_UNIXTIME: - r = str(unixtime) + "(>=MAX:" + str(DATETIME_MAX_UNIXTIME) + ")" - else: - if sys.version_info < (3, 11): - dt = datetime.datetime.utcfromtimestamp(unixtime) - else: - dt = datetime.datetime.fromtimestamp(unixtime, datetime.UTC) - r = str(unixtime) + "(" + dt.strftime('%Y-%m-%dT%H:%M:%SZ') + ")" - return r - def limit_str(content: str, limit: int) -> str: length = len(content) From cf98ef0b4aaaaa40a99e0a93cc9a59355acc6041 Mon Sep 17 00:00:00 2001 From: Peter Bieringer Date: Sat, 13 Dec 2025 08:06:58 +0100 Subject: [PATCH 213/290] rename function and fix type --- radicale/app/__init__.py | 6 +++--- radicale/utils.py | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/radicale/app/__init__.py b/radicale/app/__init__.py index 48b7b7f3..4bad79ad 100644 --- a/radicale/app/__init__.py +++ b/radicale/app/__init__.py @@ -98,13 +98,13 @@ class Application(ApplicationPartDelete, ApplicationPartHead, self._mask_passwords = configuration.get("logging", "mask_passwords") self._max_content_length = configuration.get("server", "max_content_length") self._max_resource_size = configuration.get("server", "max_resource_size") - logger.info("max_content_length to: %d bytes (%sbytes)", self._max_content_length, utils.format_int(self._max_content_length, binary=True)) + logger.info("max_content_length set to: %d bytes (%sbytes)", self._max_content_length, utils.format_unit(self._max_content_length, binary=True)) if (self._max_resource_size > (self._max_content_length * 0.8)): max_resource_size_limited = int(self._max_content_length * 0.8) - logger.warning("max_resource_size capped to: %d bytes (%sbytes) (from %d to 80%% of max_content_length)", max_resource_size_limited, utils.format_int(max_resource_size_limited, binary=True), self._max_resource_size) + logger.warning("max_resource_size set to: %d bytes (%sbytes) (capped from %d to 80%% of max_content_length)", max_resource_size_limited, utils.format_unit(max_resource_size_limited, binary=True), self._max_resource_size) self._max_resource_size = max_resource_size_limited else: - logger.info("max_resource_size set to: %d bytes (%sbytes)", self._max_resource_size, utils.format_int(self._max_resource_size, binary=True)) + logger.info("max_resource_size set to: %d bytes (%sbytes)", self._max_resource_size, utils.format_unit(self._max_resource_size, binary=True)) self._bad_put_request_content = configuration.get("logging", "bad_put_request_content") logger.info("log bad put request content: %s", self._bad_put_request_content) self._request_header_on_debug = configuration.get("logging", "request_header_on_debug") diff --git a/radicale/utils.py b/radicale/utils.py index 14c0f0f3..5c0f6c03 100644 --- a/radicale/utils.py +++ b/radicale/utils.py @@ -303,7 +303,7 @@ def format_ut(unixtime: int) -> str: return r -def format_int(value: int, binary: bool = False) -> str: +def format_unit(value: float, binary: bool = False) -> str: if binary: if value > UNIT_G: value = value / UNIT_G From cbece4f276785ee00faa481cdf2d18ab0c4456fa Mon Sep 17 00:00:00 2001 From: Peter Bieringer Date: Wed, 17 Dec 2025 08:29:52 +0100 Subject: [PATCH 214/290] catch 2 more xml tokens --- radicale/app/__init__.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/radicale/app/__init__.py b/radicale/app/__init__.py index 4bad79ad..6a8f7b2c 100644 --- a/radicale/app/__init__.py +++ b/radicale/app/__init__.py @@ -289,8 +289,12 @@ class Application(ApplicationPartDelete, ApplicationPartHead, if xml_request is not None: if "" in xml_request: flags.append("sync-token") + if "" in xml_request: + flags.append("getetag") if "" in xml_request: flags.append("getctag") + if " Date: Wed, 17 Dec 2025 08:30:12 +0100 Subject: [PATCH 215/290] analyse two more xml tokens --- contrib/logwatch/radicale | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/contrib/logwatch/radicale b/contrib/logwatch/radicale index 03ffbc48..8acbe0ae 100644 --- a/contrib/logwatch/radicale +++ b/contrib/logwatch/radicale @@ -91,9 +91,15 @@ sub ConvertTokens($) { if (defined $tokens_h{"sync-token"}) { push @result_a, "ST"; } + if (defined $tokens_h{"sync-collection"}) { + push @result_a, "SC"; + } if (defined $tokens_h{"getctag"}) { push @result_a, "GCT"; } + if (defined $tokens_h{"getetag"}) { + push @result_a, "GET"; + } # TODO: add potential others which causing long duration $result = ""; if (scalar(@result_a) > 0) { @@ -251,7 +257,7 @@ if (keys %Logins) { if (keys %ResponseTimes) { $length = MaxLength(\%ResponseTimes); print "\n**Response timings (counts, seconds) (D= R= F=)**\n"; - print "* Flags: ST:sync-token GCT:getctag\n"; + print "* Flags: ST:sync-token SC:sync-collection GCT:getctag GET:getetag\n"; printf "%-" . $length . "s | %7s | %7s | %7s | %7s |\n", "Response", "cnt", "min", "max", "avg"; print "-" x($length + 42) . "\n"; foreach my $req (sort keys %ResponseTimes) { From 61136c1e7a321ee7ae561cd87f249013b22e20c5 Mon Sep 17 00:00:00 2001 From: Peter Bieringer Date: Wed, 17 Dec 2025 08:41:01 +0100 Subject: [PATCH 216/290] log max_connections on startup --- radicale/server.py | 1 + 1 file changed, 1 insertion(+) diff --git a/radicale/server.py b/radicale/server.py index 55e112e2..1dc3dee4 100644 --- a/radicale/server.py +++ b/radicale/server.py @@ -339,6 +339,7 @@ def serve(configuration: config.Configuration, # Fallback to busy waiting. (select(...) blocks SIGINT on Windows.) select_timeout = 1.0 max_connections: int = configuration.get("server", "max_connections") + logger.info("Maximum parallel connections: %d", max_connections) logger.info("Radicale server ready") logger.debug("TRACE: Radicale server ready ('logging/trace_on_debug' is active)") logger.debug("TRACE/SERVER: Radicale server ready ('logging/trace_on_debug' is active - either with 'SERVER' or empty filter)") From ec007832394bbed6b59f9ed594a30f67b6e620bb Mon Sep 17 00:00:00 2001 From: Peter Bieringer Date: Thu, 18 Dec 2025 08:01:15 +0100 Subject: [PATCH 217/290] Release 3.5.10 --- CHANGELOG.md | 2 +- pyproject.toml | 2 +- setup.py.legacy | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index c0da9c33..ab674508 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,6 @@ # Changelog -## 3.5.10.dev +## 3.5.10 * Improve: logging of broken calendar items during PUT * Add: logging of broken contact items during PUT * Extend: [auth] imap: add fallback support for LOGIN towards remote IMAP server (replaced in 3.5.0) diff --git a/pyproject.toml b/pyproject.toml index 688ce0a6..fef620e7 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -3,7 +3,7 @@ name = "Radicale" # When the version is updated, a new section in the CHANGELOG.md file must be # added too. readme = "README.md" -version = "3.5.10.dev" +version = "3.5.10" authors = [{name = "Guillaume Ayoub", email = "guillaume.ayoub@kozea.fr"}, {name = "Unrud", email = "unrud@outlook.com"}, {name = "Peter Bieringer", email = "pb@bieringer.de"}] license = {text = "GNU GPL v3"} description = "CalDAV and CardDAV Server" diff --git a/setup.py.legacy b/setup.py.legacy index 0797fdd0..9c003813 100644 --- a/setup.py.legacy +++ b/setup.py.legacy @@ -20,7 +20,7 @@ from setuptools import find_packages, setup # When the version is updated, a new section in the CHANGELOG.md file must be # added too. -VERSION = "3.5.10.dev" +VERSION = "3.5.10" with open("README.md", encoding="utf-8") as f: long_description = f.read() From 8b20490be0505207a7f68d1ed138d57c36138807 Mon Sep 17 00:00:00 2001 From: Peter Bieringer Date: Thu, 18 Dec 2025 08:06:52 +0100 Subject: [PATCH 218/290] cosmetics --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index ab674508..cf3c6945 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,7 +6,7 @@ * Extend: [auth] imap: add fallback support for LOGIN towards remote IMAP server (replaced in 3.5.0) * Fix: improper detection of HTTP_X_FORWARDED_PORT on MOVE * Extend: [logging] with profiling log per reqest or regular per request method -* New: [logging] option to log response header on debug loglevel +* Add: [logging] option to log response header on debug loglevel * Adjust: [logging] header/content debug log indended by space to be skipped by logwatch * Improve: remove unnecessary open+read for mtime+size cache * Extend: add selected XML query properties to request result log line for improved timing analysis incl. logwatch support From adce04e94712b6e013d4a59c9dcbb1fca520bce3 Mon Sep 17 00:00:00 2001 From: kalsi-avneet <4151485+kalsi-avneet@users.noreply.github.com> Date: Sun, 21 Dec 2025 21:23:09 +0530 Subject: [PATCH 219/290] Docker publish workflow: Add stable, major, and minor tags For every release, add the following tags: 1. "stable" - The latest stable release can be obtained using this tag 2. major and minor The "latest" tag remains as-is. It will always point to the latest image (stable or otherwise) --- .github/workflows/docker-publish.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/docker-publish.yml b/.github/workflows/docker-publish.yml index 3bee7778..4c50e211 100644 --- a/.github/workflows/docker-publish.yml +++ b/.github/workflows/docker-publish.yml @@ -37,8 +37,11 @@ jobs: flavor: latest=true tags: | type=semver,pattern={{version}} + type=semver,pattern={{major}} + type=semver,pattern={{major}}.{{minor}} type=schedule,prefix=nightly-,pattern={{date 'YYYYMMDD'}} type=raw,enable=${{ github.event_name == 'workflow_dispatch' }},value=workflow_dispatch-{{branch}}-{{sha}} + type=raw,enable=${{ github.event_name == 'release' }},value=stable - name: Set up QEMU uses: docker/setup-qemu-action@v3 From f1a150e8f3c6403e6730aba737030cc7abaa6e80 Mon Sep 17 00:00:00 2001 From: Peter Bieringer Date: Fri, 26 Dec 2025 07:54:08 +0100 Subject: [PATCH 220/290] 3.5.11.dev prep --- CHANGELOG.md | 2 ++ pyproject.toml | 2 +- setup.py.legacy | 2 +- 3 files changed, 4 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index cf3c6945..b82eb2ff 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,7 @@ # Changelog +## 3.5.11.dev + ## 3.5.10 * Improve: logging of broken calendar items during PUT * Add: logging of broken contact items during PUT diff --git a/pyproject.toml b/pyproject.toml index fef620e7..847f6fd6 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -3,7 +3,7 @@ name = "Radicale" # When the version is updated, a new section in the CHANGELOG.md file must be # added too. readme = "README.md" -version = "3.5.10" +version = "3.5.11.dev" authors = [{name = "Guillaume Ayoub", email = "guillaume.ayoub@kozea.fr"}, {name = "Unrud", email = "unrud@outlook.com"}, {name = "Peter Bieringer", email = "pb@bieringer.de"}] license = {text = "GNU GPL v3"} description = "CalDAV and CardDAV Server" diff --git a/setup.py.legacy b/setup.py.legacy index 9c003813..959614f3 100644 --- a/setup.py.legacy +++ b/setup.py.legacy @@ -20,7 +20,7 @@ from setuptools import find_packages, setup # When the version is updated, a new section in the CHANGELOG.md file must be # added too. -VERSION = "3.5.10" +VERSION = "3.5.11.dev" with open("README.md", encoding="utf-8") as f: long_description = f.read() From 22f6570af5b16289d2d285783b74e59c1570d005 Mon Sep 17 00:00:00 2001 From: Peter Bieringer Date: Fri, 26 Dec 2025 07:55:23 +0100 Subject: [PATCH 221/290] logwatch extension+review --- contrib/logwatch/radicale | 228 +++++++++++++++++++++++++++----------- 1 file changed, 166 insertions(+), 62 deletions(-) diff --git a/contrib/logwatch/radicale b/contrib/logwatch/radicale index 8acbe0ae..c659f62f 100644 --- a/contrib/logwatch/radicale +++ b/contrib/logwatch/radicale @@ -3,19 +3,32 @@ # Copyright © 2024-2025 Peter Bieringer # # Detail levels -# >= 5: Logins -# >= 10: ResponseTimes +# < 5 : Request + ResponseCounters +# >= 5 : incl. Logins +# >= 10: incl. ResponseTimes + ResponseSize +# >= 15: incl. ResponseTimes + ResponseSize incl. RequestFlags +# >= 18: incl. UserAgents +# >= 20: incl. Locations where supported, anonymize logins + +use Digest::SHA; $Detail = $ENV{'LOGWATCH_DETAIL_LEVEL'} || 0; +my %ResponseTimesLocUsr; +my %ResponseSizesLocUsr; my %ResponseTimes; my %ResponseSizes; my %Responses; my %Requests; +my %UserAgents; my %Logins; my %Loglevel; my %OtherEvents; +my %Locations; +my %LocationsFile; +my %LoginsHash; + my $sum; my $length; @@ -108,6 +121,36 @@ sub ConvertTokens($) { return $result; } +sub ConvertLoc($) { + my $loc = $_[0]; + if (defined $Locations{$loc}) { + # from cache + return ":L=" . $Locations{$loc}; + } elsif (defined $LocationsFile{$loc}) { + # from cache + return ":L=" . $LocationsFile{$loc}; + } + + if ($loc =~ /\/'$/o) { + $Locations{$loc} = "L=" . substr(Digest::SHA::sha256_hex($loc), 0, 8); + return ":" . $Locations{$loc}; + } else { + $LocationsFile{$loc} = "L="; + return ":" . $Locations{$loc}; + } +} + +sub ConvertLogin($) { + my $login = $_[0]; + if (defined $LoginsHash{$loginc}) { + # from cache + return $LoginsHash{$login}; + } + + $LoginsHash{$login} = "U=" . substr(Digest::SHA::sha256_hex($login), 0, 8); + return $LoginsHash{$login}; +} + while (defined($ThisLine = )) { # count loglevel if ( $ThisLine =~ /\[(DEBUG|INFO|WARNING|ERROR|CRITICAL)\] /o ) { @@ -123,42 +166,60 @@ while (defined($ThisLine = )) { } elsif ( $ThisLine =~ / (\S+) response status/o ) { my $req = $1; - if ( $ThisLine =~ / \S+ response status for .* with depth '(\d)' in ([0-9.]+) seconds: (\d+)/o ) { - $req .= ":D=" . $1 . ":R=" . $3; + if ( $ThisLine =~ / \S+ response status for (.*) with depth '(\d)' in ([0-9.]+) seconds: (\d+)/o ) { + $req .= ":D=" . $2 . ":R=" . $4; + $req .= ConvertLoc($1) if ($Detail >= 20); ResponseTimesMinMaxSum($req, $2) if ($Detail >= 10); - } elsif ( $ThisLine =~ / \S+ response status for .* in ([0-9.]+) seconds: (\d+)/o ) { - $req .= ":R=" . $2; + } elsif ( $ThisLine =~ / \S+ response status for (.*) in ([0-9.]+) seconds: (\d+)/o ) { + $req .= ":R=" . $3; + $req .= ConvertLoc($1) if ($Detail >= 20); ResponseTimesMinMaxSum($req, $1) if ($Detail >= 10); - } elsif ( $ThisLine =~ / \S+ response status for .* with depth '(\d)' in ([0-9.]+) seconds (\S+) (\d+) bytes: (\d+)/o ) { - $req .= ":D=" . $1 . ":R=" . $5; - ResponseTimesMinMaxSum($req, $2) if ($Detail >= 10); - ResponseSizesMinMaxSum($req, $3, $4) if ($Detail >= 10); - } elsif ( $ThisLine =~ / \S+ response status for .* in ([0-9.]+) seconds (\S+) (\d+) bytes: (\d+)/o ) { - $req .= ":R=" . $4; - ResponseTimesMinMaxSum($req, $1) if ($Detail >= 10); - ResponseSizesMinMaxSum($req, $2, $3) if ($Detail >= 10); - } elsif ( $ThisLine =~ / \S+ response status for .* with depth '(\d)' in ([0-9.]+) seconds (\S+) (\d+) bytes \((.*)\): (\d+)/o ) { - $req .= ":D=" . $1 . ":R=" . $6; - $reqWithFlags = $req . ConvertTokens($5); - ResponseTimesMinMaxSum($reqWithFlags, $2) if ($Detail >= 10); - ResponseSizesMinMaxSum($req, $3, $4) if ($Detail >= 10); - } elsif ( $ThisLine =~ / \S+ response status for .* in ([0-9.]+) seconds (\S+) (\d+) bytes \((.*)\): (\d+)/o ) { + } elsif ( $ThisLine =~ / \S+ response status for (.*) with depth '(\d)' in ([0-9.]+) seconds (\S+) (\d+) bytes: (\d+)/o ) { + $req .= ":D=" . $2 . ":R=" . $6; + $req .= ConvertLoc($1) if ($Detail >= 20); + ResponseTimesMinMaxSum($req, $3) if ($Detail >= 10); + ResponseSizesMinMaxSum($req, $4, $5) if ($Detail >= 10); + } elsif ( $ThisLine =~ / \S+ response status for (.*) in ([0-9.]+) seconds (\S+) (\d+) bytes: (\d+)/o ) { $req .= ":R=" . $5; - $reqWithFlags = $req . ConvertTokens($4); - ResponseTimesMinMaxSum($reqWithFlags, $1) if ($Detail >= 10); - ResponseSizesMinMaxSum($req, $2, $3) if ($Detail >= 10); + $req .= ConvertLoc($1) if ($Detail >= 20); + ResponseTimesMinMaxSum($req, $2) if ($Detail >= 10); + ResponseSizesMinMaxSum($req, $3, $4) if ($Detail >= 10); + } elsif ( $ThisLine =~ / \S+ response status for (.*) with depth '(\d)' in ([0-9.]+) seconds (\S+) (\d+) bytes \((.*)\): (\d+)/o ) { + $req .= ":D=" . $2 . ":R=" . $7; + $req .= ConvertLoc($1) if ($Detail >= 20); + $req .= ConvertTokens($6) if ($Detail >= 15); + ResponseTimesMinMaxSum($req, $3) if ($Detail >= 10); + ResponseSizesMinMaxSum($req, $4, $5) if ($Detail >= 10); + } elsif ( $ThisLine =~ / \S+ response status for (.*) in ([0-9.]+) seconds (\S+) (\d+) bytes \((.*)\): (\d+)/o ) { + $req .= ":R=" . $6; + $req .= ConvertLoc($1) if ($Detail >= 20); + $req .= ConvertTokens($6) if ($Detail >= 15); + ResponseTimesMinMaxSum($req, $2) if ($Detail >= 10); + ResponseSizesMinMaxSum($req, $3, $4) if ($Detail >= 10); } $Responses{$req}++; } - elsif ( $ThisLine =~ / (\S+) request for/o ) { + elsif ( $ThisLine =~ / (\S+) request for ('[^']+')/o ) { my $req = $1; - if ( $ThisLine =~ / \S+ request for .* with depth '(\d)' received/o ) { + my $loc = $2; + if ( $ThisLine =~ / with depth '(\d)' received/o ) { $req .= ":D=" . $1; } + $req .= ConvertLoc($loc) if ($Detail >= 20); $Requests{$req}++; + + if ( $ThisLine =~ /using ('.*')/o ) { + my $ua = $1; + # remove unexpected chars + $ua =~ s/[\x00-\x1F\x7F-\xFF]//g; + $ua .= ConvertLoc($loc) if ($Detail >= 20); + $UserAgents{$ua}++ if ($Detail >= 18); + } } elsif ( $ThisLine =~ / (Successful login): '([^']+)'/o ) { - $Logins{$2}++ if ($Detail >= 5); + my $login = $2; + $login = ConvertLogin($login) if ($Detail >= 20); + $Logins{$login}++ if ($Detail >= 5); $OtherEvents{$1}++; } elsif ( $ThisLine =~ / (Failed login attempt) /o ) { @@ -207,57 +268,76 @@ if ($Started) { if (keys %Loglevel) { $sum = Sum(\%Loglevel); print "\n**Loglevel counters**\n"; - printf "%-18s | %7s | %5s |\n", "Loglevel", "cnt", "ratio"; - print "-" x38 . "\n"; + printf "%-18s | %7s | %9s |\n", "Loglevel", "cnt", "ratio"; + print "-" x42 . "\n"; foreach my $level (sort keys %Loglevel) { - printf "%-18s | %7d | %3d%% |\n", $level, $Loglevel{$level}, int(($Loglevel{$level} * 100) / $sum); + printf "%-18s | %7d | %7.3f%% |\n", $level, $Loglevel{$level}, (($Loglevel{$level} * 100) / $sum); } - print "-" x38 . "\n"; - printf "%-18s | %7d | %3d%% |\n", "", $sum, 100; -} - -if (keys %Requests) { - $sum = Sum(\%Requests); - print "\n**Request counters (D=)**\n"; - printf "%-18s | %7s | %5s |\n", "Request", "cnt", "ratio"; - print "-" x38 . "\n"; - foreach my $req (sort keys %Requests) { - printf "%-18s | %7d | %3d%% |\n", $req, $Requests{$req}, int(($Requests{$req} * 100) / $sum); - } - print "-" x38 . "\n"; - printf "%-18s | %7d | %3d%% |\n", "", $sum, 100; -} - -if (keys %Responses) { - $sum = Sum(\%Responses); - $length = MaxLength(\%Responses); - print "\n**Response result counters ((D= R=)**\n"; - printf "%-" . $length . "s | %7s | %5s |\n", "Response", "cnt", "ratio"; - print "-" x($length + 20) . "\n"; - foreach my $req (sort keys %Responses) { - printf "%-" . $length . "s | %7d | %3d%% |\n", $req, $Responses{$req}, int(($Responses{$req} * 100) / $sum); - } - print "-" x($length + 20) . "\n"; - printf "%-" . $length . "s | %7d | %3d%% |\n", "", $sum, 100; + print "-" x42 . "\n"; + printf "%-18s | %7d | %7.3f%% |\n", "", $sum, 100; } if (keys %Logins) { $sum = Sum(\%Logins); $length = MaxLength(\%Logins); print "\n**Successful login counters**\n"; - printf "%-" . $length . "s | %7s | %5s |\n", "Login", "cnt", "ratio"; - print "-" x($length + 20) . "\n"; + printf "%-" . $length . "s | %7s | %9s |\n", "Login", "cnt", "ratio"; + print "-" x($length + 24) . "\n"; foreach my $login (sort keys %Logins) { - printf "%-" . $length . "s | %7d | %3d%% |\n", $login, $Logins{$login}, int(($Logins{$login} * 100) / $sum); + printf "%-" . $length . "s | %7d | %7.3f%% |\n", $login, $Logins{$login}, (($Logins{$login} * 100) / $sum); } - print "-" x($length + 20) . "\n"; - printf "%-" . $length . "s | %7d | %3d%% |\n", "", $sum, 100; + print "-" x($length + 24) . "\n"; + printf "%-" . $length . "s | %7d | %7.3d%% |\n", "", $sum, 100; +} + +if (keys %UserAgents) { + $sum = Sum(\%UserAgents); + $length = MaxLength(\%UserAgents); + print "\n**UserAgent Counters**\n"; + print "* Location: L= -> see below L= -> see raw log\n" if (scalar(keys %Locations) > 0); + printf "%-" . $length . "s | %7s | %9s |\n", "UserAgent", "cnt", "ratio"; + print "-" x($length + 24) . "\n"; + foreach my $ua (sort keys %UserAgents) { + printf "%-" . $length . "s | %7d | %7.3f%% |\n", $ua, $UserAgents{$ua}, (($UserAgents{$ua} * 100) / $sum); + } + print "-" x($length + 24) . "\n"; + printf "%-" . $length . "s | %7d | %7.3d%% |\n", "", $sum, 100; +} + +if (keys %Requests) { + $sum = Sum(\%Requests); + $length = MaxLength(\%Requests); + print "\n**Request counters (D=)**\n"; + print "* Location: L= -> see below L= -> see raw log\n" if (scalar(keys %Locations) > 0); + printf "%-" . $length . "s | %7s | %9s |\n", "Request", "cnt", "ratio"; + print "-" x($length + 24) . "\n"; + foreach my $req (sort keys %Requests) { + printf "%-" . $length . "s | %7d | %7.3f%% |\n", $req, $Requests{$req}, (($Requests{$req} * 100) / $sum); + } + print "-" x($length + 24) . "\n"; + printf "%-18s | %7d | %7.3f%% |\n", "", $sum, 100; +} + +if (keys %Responses) { + $sum = Sum(\%Responses); + $length = MaxLength(\%Responses); + print "\n**Response result counters ((D= R=)**\n"; + print "* Flags: ST:sync-token SC:sync-collection GCT:getctag GET:getetag\n" if ($Detail >= 15); + print "* Location: L= -> see below L= -> see raw log\n" if (scalar(keys %Locations) > 0); + printf "%-" . $length . "s | %7s | %9s |\n", "Response", "cnt", "ratio"; + print "-" x($length + 24) . "\n"; + foreach my $req (sort keys %Responses) { + printf "%-" . $length . "s | %7d | %7.3f%% |\n", $req, $Responses{$req}, (($Responses{$req} * 100) / $sum); + } + print "-" x($length + 24) . "\n"; + printf "%-" . $length . "s | %7d | %7.3f%% |\n", "", $sum, 100; } if (keys %ResponseTimes) { $length = MaxLength(\%ResponseTimes); print "\n**Response timings (counts, seconds) (D= R= F=)**\n"; - print "* Flags: ST:sync-token SC:sync-collection GCT:getctag GET:getetag\n"; + print "* Flags: ST:sync-token SC:sync-collection GCT:getctag GET:getetag\n" if ($Detail >= 15); + print "* Location: L= -> see below L= -> see raw log\n" if (scalar(keys %Locations) > 0); printf "%-" . $length . "s | %7s | %7s | %7s | %7s |\n", "Response", "cnt", "min", "max", "avg"; print "-" x($length + 42) . "\n"; foreach my $req (sort keys %ResponseTimes) { @@ -274,6 +354,8 @@ if (keys %ResponseSizes) { for my $type (sort keys %ResponseSizes) { $length = MaxLength($ResponseSizes{$type}); print "\n**Response sizes (counts, bytes: $type) (D= R=)**\n"; + print "* Flags: ST:sync-token SC:sync-collection GCT:getctag GET:getetag\n" if ($Detail >= 15); + print "* Location: L= -> see below L= -> see raw log\n" if (scalar(keys %Locations) > 0); printf "%-" . $length . "s | %7s | %9s | %9s | %9s |\n", "Response", "cnt", "min", "max", "avg"; print "-" x($length + 48) . "\n"; foreach my $req (sort keys %{$ResponseSizes{$type}}) { @@ -301,6 +383,28 @@ if (keys %OtherList) { } } +if (scalar(keys %LoginsHash) > 0) { + print "\n**Map of login hashes (REMOVE THIS FOR PRIVACY REASONS before submit)**\n"; + $length = MaxLength(\%LoginsHash); + printf "%-10s | %-" . $length . "s | \n", "Hash", "Login"; + print "-" x($length + 15) . "\n"; + foreach my $login (sort { $LoginsHash{$a} cmp $LoginsHash{$b} } keys %LoginsHash) { + printf "%10s | %-" . $length . "s |\n", $LoginsHash{$login}, $login; + } + print "-" x($length + 15) . "\n"; +} + +if (scalar(keys %Locations) > 0) { + print "\n**Map of location hashes (REMOVE THIS FOR PRIVACY REASONS before submit)**\n"; + $length = MaxLength(\%Locations); + printf "%-10s | %-" . $length . "s | \n", "Hash", "Location"; + print "-" x($length + 15) . "\n"; + foreach my $loc (sort { $Locations{$a} cmp $Locations{$b} } keys %Locations) { + printf "%10s | %-" . $length . "s |\n", $Locations{$loc}, $loc; + } + print "-" x($length + 15) . "\n"; +} + exit(0); # vim: shiftwidth=3 tabstop=3 syntax=perl et smartindent From 99ee86c6a4a1b21ebc274ec16433d677c5a762bf Mon Sep 17 00:00:00 2001 From: Peter Bieringer Date: Fri, 26 Dec 2025 07:56:12 +0100 Subject: [PATCH 222/290] changelog for 22f6570af5b16289d2d285783b74e59c1570d005 --- CHANGELOG.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index b82eb2ff..6bd266f1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,8 @@ ## 3.5.11.dev +* Extend: logwatch script + ## 3.5.10 * Improve: logging of broken calendar items during PUT * Add: logging of broken contact items during PUT From 6bd7a5b5e322e05e1f209f4b7ca139d779918b1e Mon Sep 17 00:00:00 2001 From: Peter Bieringer Date: Sun, 28 Dec 2025 07:42:44 +0100 Subject: [PATCH 223/290] add checksum and hexdump support --- radicale/utils.py | 113 ++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 113 insertions(+) diff --git a/radicale/utils.py b/radicale/utils.py index 5c0f6c03..6a1fdade 100644 --- a/radicale/utils.py +++ b/radicale/utils.py @@ -22,7 +22,9 @@ import os import ssl import sys import textwrap +from hashlib import sha256 from importlib import import_module, metadata +from string import ascii_letters, digits, punctuation from typing import Callable, Sequence, Tuple, Type, TypeVar, Union from radicale import config @@ -342,3 +344,114 @@ def limit_str(content: str, limit: int) -> str: def textwrap_str(content: str, limit: int = 2000) -> str: # TODO: add support for config option and prefix return textwrap.indent(limit_str(content, limit), " ", lambda line: True) + + +def dataToHex(data, count): + result = '' + for item in range(count): + if ((item > 0) and ((item % 8) == 0)): + result += ' ' + if (item < len(data)): + result += '%02x' % data[item] + ' ' + else: + result += ' ' + return result + + +def dataToAscii(data, count): + result = '' + for item in range(count): + if (item < len(data)): + char = chr(data[item]) + if char in ascii_letters or \ + char in digits or \ + char in punctuation or \ + char == ' ': + result += char + else: + result += '.' + return result + + +def dataToSpecial(data, count): + result = '' + for item in range(count): + if (item < len(data)): + char = chr(data[item]) + if char == '\r': + result += 'C' + elif char == '\n': + result += 'L' + elif ord(char) == 0xc2: + result += 'u' + else: + result += '.' + return result + + +def hexdump_str(content: str, limit: int = 2000) -> str: + + result = "" + index = 0 + size = 16 + bytestring = content.encode("utf-8") + length = len(bytestring) + + while (index < length) and (index < limit): + data = bytestring[index:index+size] + hex = dataToHex(data, size) + ascii = dataToAscii(data, size) + special = dataToSpecial(data, size) + result += '%08x ' % index + result += hex + result += '|' + result += '%-16s' % ascii + result += '|' + result += '%-16s' % special + result += '|' + result += '\n' + index += size + + return result + + +def hexdump_line(line: str, limit: int = 200) -> str: + result = "" + length_str = len(line) + bytestring = line.encode("utf-8") + length = len(bytestring) + size = length + if (size > limit): + size = limit + + hex = dataToHex(bytestring, size) + ascii = dataToAscii(bytestring, size) + special = dataToSpecial(bytestring, size) + result += '%3d/%3d' % (length_str, length) + result += ': ' + result += hex + result += '|' + result += ascii + result += '|' + result += special + result += '|' + result += '\n' + + return result + + +def hexdump_lines(lines: str, limit: int = 200) -> str: + result = "" + counter = 0 + for line in lines.splitlines(True): + result += '% 4d ' % counter + result += hexdump_line(line) + counter += 1 + + return result + + +def sha256_str(content: str) -> str: + _hash = sha256() + _hash.update(content.encode("utf-8")) + return _hash.hexdigest() From 76abfe719b3087aff0931dc097b2a3e5a212e43e Mon Sep 17 00:00:00 2001 From: Peter Bieringer Date: Sun, 28 Dec 2025 08:01:05 +0100 Subject: [PATCH 224/290] only call expensive debug logging on debug level --- radicale/app/__init__.py | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/radicale/app/__init__.py b/radicale/app/__init__.py index 6a8f7b2c..54016c81 100644 --- a/radicale/app/__init__.py +++ b/radicale/app/__init__.py @@ -30,6 +30,7 @@ import base64 import cProfile import datetime import io +import logging import pprint import pstats import random @@ -253,9 +254,11 @@ class Application(ApplicationPartDelete, ApplicationPartHead, if answer is not None: if isinstance(answer, str): if self._response_content_on_debug: - logger.debug("Response content (nonXML):\n%s", utils.textwrap_str(answer)) + if logger.isEnabledFor(logging.DEBUG): + logger.debug("Response content (nonXML):\n%s", utils.textwrap_str(answer)) else: - logger.debug("Response content: suppressed by config/option [logging] response_content_on_debug") + if logger.isEnabledFor(logging.DEBUG): + logger.debug("Response content: suppressed by config/option [logging] response_content_on_debug") headers["Content-Type"] += "; charset=%s" % self._encoding answer = answer.encode(self._encoding) accept_encoding = [ @@ -276,9 +279,11 @@ class Application(ApplicationPartDelete, ApplicationPartHead, headers.update(self._extra_headers) if self._response_header_on_debug: - logger.debug("Response header:\n%s", utils.textwrap_str(pprint.pformat(headers))) + if logger.isEnabledFor(logging.DEBUG): + logger.debug("Response header:\n%s", utils.textwrap_str(pprint.pformat(headers))) else: - logger.debug("Response header: suppressed by config/option [logging] response_header_on_debug") + if logger.isEnabledFor(logging.DEBUG): + logger.debug("Response header: suppressed by config/option [logging] response_header_on_debug") # Start response time_end = datetime.datetime.now() From 22d253c95ad7512e3dacf914d3cbd0d1ac77f5e6 Mon Sep 17 00:00:00 2001 From: Peter Bieringer Date: Sun, 28 Dec 2025 08:07:32 +0100 Subject: [PATCH 225/290] log_bad_put_request_content: log hexdump of request on debug level --- radicale/app/put.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/radicale/app/put.py b/radicale/app/put.py index 1014f95b..4c1caf14 100644 --- a/radicale/app/put.py +++ b/radicale/app/put.py @@ -202,6 +202,10 @@ class ApplicationPartPut(ApplicationBase): "Bad PUT request on %r (read_components): %s", path, e, exc_info=True) if self._log_bad_put_request_content: logger.warning("Bad PUT request content of %r:\n%s", path, utils.textwrap_str(content)) + if logger.isEnabledFor(logging.DEBUG): + logger.debug("Request content (sha256sum): %s", utils.sha256_str(content)) + logger.debug("Request content (hexdump):\n%s", utils.hexdump_str(content)) + logger.debug("Request content (hexdump/lines):\n%s", utils.hexdump_lines(content)) else: logger.debug("Bad PUT request content: suppressed by config/option [logging] bad_put_request_content") return httputils.BAD_REQUEST From 7e1890d63062edebf2f98e78fc30f79c5e1d62a5 Mon Sep 17 00:00:00 2001 From: Peter Bieringer Date: Sun, 28 Dec 2025 08:08:45 +0100 Subject: [PATCH 226/290] request log: add checksum on debug level, call expensive debug only on debug level --- radicale/httputils.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/radicale/httputils.py b/radicale/httputils.py index 30c6a7a0..81e01715 100644 --- a/radicale/httputils.py +++ b/radicale/httputils.py @@ -24,6 +24,7 @@ Helper functions for HTTP. """ import contextlib +import logging import os import pathlib import sys @@ -150,9 +151,12 @@ def read_request_body(configuration: "config.Configuration", content = decode_request(configuration, environ, read_raw_request_body(configuration, environ)) if configuration.get("logging", "request_content_on_debug"): - logger.debug("Request content:\n%s", utils.textwrap_str(content)) + if logger.isEnabledFor(logging.DEBUG): + logger.debug("Request content (sha256sum): %s", utils.sha256_str(content)) + logger.debug("Request content:\n%s", utils.textwrap_str(content)) else: - logger.debug("Request content: suppressed by config/option [logging] request_content_on_debug") + if logger.isEnabledFor(logging.DEBUG): + logger.debug("Request content: suppressed by config/option [logging] request_content_on_debug") return content From 680bce2cf0a702114b17ed848b46841530e5d55b Mon Sep 17 00:00:00 2001 From: Peter Bieringer Date: Sun, 28 Dec 2025 08:12:41 +0100 Subject: [PATCH 227/290] related to debug log extension --- CHANGELOG.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 6bd266f1..5e922bfc 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,6 +3,8 @@ ## 3.5.11.dev * Extend: logwatch script +* Extend: [logging] bad_put_request_content: log checksum and hexdump of request on debug level +* Extend: [logging] request_content_on_debug: log checksum of request on debug level ## 3.5.10 * Improve: logging of broken calendar items during PUT From fdcd3e2debbf58887ffa55fc81ee91395900deb1 Mon Sep 17 00:00:00 2001 From: Peter Bieringer Date: Sun, 28 Dec 2025 09:13:07 +0100 Subject: [PATCH 228/290] fix tox issues --- radicale/__main__.py | 17 ++++++++++++++++- radicale/item/__init__.py | 19 ++++++++++++++++++- 2 files changed, 34 insertions(+), 2 deletions(-) diff --git a/radicale/__main__.py b/radicale/__main__.py index b3576a60..a2668b6f 100644 --- a/radicale/__main__.py +++ b/radicale/__main__.py @@ -33,7 +33,7 @@ import sys from types import FrameType from typing import List, Optional, cast -from radicale import VERSION, config, log, server, storage, types +from radicale import VERSION, config, item, log, server, storage, types from radicale.log import logger @@ -65,6 +65,8 @@ def run() -> None: parser.add_argument("--version", action="version", version=VERSION) parser.add_argument("--verify-storage", action="store_true", help="check the storage for errors and exit") + parser.add_argument("--verify-item", action="store", nargs=1, + help="check the provided item file for errors and exit") parser.add_argument("-C", "--config", help="use specific configuration files", nargs="*") parser.add_argument("-D", "--debug", action="store_const", const="debug", @@ -194,6 +196,19 @@ def run() -> None: sys.exit(1) return + if args_ns.verify_item: + encoding = configuration.get("encoding", "stock") + logger.info("Item verification start using 'stock' encoding: %s", encoding) + try: + if not item.verify(args_ns.verify_item[0], encoding): + logger.critical("Item verification failed") + sys.exit(1) + except Exception as e: + logger.critical("An exception occurred during item " + "verification: %s", e, exc_info=False) + sys.exit(1) + return + # Create a socket pair to notify the server of program shutdown shutdown_socket, shutdown_socket_out = socket.socketpair() diff --git a/radicale/item/__init__.py b/radicale/item/__init__.py index a05304ff..7b240512 100644 --- a/radicale/item/__init__.py +++ b/radicale/item/__init__.py @@ -37,7 +37,7 @@ from typing import (Any, Callable, List, MutableMapping, Optional, Sequence, import vobject from radicale import storage # noqa:F401 -from radicale import pathutils +from radicale import pathutils, utils from radicale.item import filter as radicale_filter from radicale.log import logger @@ -335,6 +335,23 @@ def find_time_range(vobject_item: vobject.base.Component, tag: str return math.floor(start.timestamp()), math.ceil(end.timestamp()) +def verify(file: str, encoding: str): + logger.info("Verifying item: %s", file) + with open(file, "rb") as f: + content_raw = f.read() + content = content_raw.decode(encoding) + logger.info("Verifying item: %s has sha256sum %r", file, utils.sha256_str(content)) + try: + vobject_items = read_components(content) # noqa: F841 + except Exception as e: + logger.error("Verifying item: %s problem: %s", file, e) + logger.info("Request content (hexdump/lines):\n%s", utils.hexdump_lines(content)) + return False + else: + logger.info("Verifying item: %s successful", file) + return True + + class Item: """Class for address book and calendar entries.""" From baacba191b7e11fb26bc0344ea166aaff83b3f96 Mon Sep 17 00:00:00 2001 From: Peter Bieringer Date: Sun, 28 Dec 2025 09:13:18 +0100 Subject: [PATCH 229/290] adjust/extend copyright --- radicale/__main__.py | 2 +- radicale/item/__init__.py | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/radicale/__main__.py b/radicale/__main__.py index a2668b6f..e5eb68db 100644 --- a/radicale/__main__.py +++ b/radicale/__main__.py @@ -1,7 +1,7 @@ # This file is part of Radicale - CalDAV and CardDAV server # Copyright © 2011-2017 Guillaume Ayoub # Copyright © 2017-2022 Unrud -# Copyright © 2024-2024 Peter Bieringer +# Copyright © 2024-2025 Peter Bieringer # # This library is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by diff --git a/radicale/item/__init__.py b/radicale/item/__init__.py index 7b240512..be01c42e 100644 --- a/radicale/item/__init__.py +++ b/radicale/item/__init__.py @@ -3,7 +3,8 @@ # Copyright © 2008 Pascal Halter # Copyright © 2014 Jean-Marc Martins # Copyright © 2008-2017 Guillaume Ayoub -# Copyright © 2017-2018 Unrud +# Copyright © 2017-2022 Unrud +# Copyright © 2024-2025 Peter Bieringer # # This library is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by From 1b82b891097b796653cd7409a63ab747617d60c9 Mon Sep 17 00:00:00 2001 From: Peter Bieringer Date: Sun, 28 Dec 2025 09:15:13 +0100 Subject: [PATCH 230/290] extend for new option --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 5e922bfc..30124e0b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,7 @@ * Extend: logwatch script * Extend: [logging] bad_put_request_content: log checksum and hexdump of request on debug level * Extend: [logging] request_content_on_debug: log checksum of request on debug level +* Extend: add command line option "--verify-item " for dedicated item file analysis ## 3.5.10 * Improve: logging of broken calendar items during PUT From 63d7229773d739a77bc534b39ff0792722352e09 Mon Sep 17 00:00:00 2001 From: Peter Bieringer Date: Sun, 28 Dec 2025 09:25:38 +0100 Subject: [PATCH 231/290] extend doc related to command line args --- DOCUMENTATION.md | 38 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 38 insertions(+) diff --git a/DOCUMENTATION.md b/DOCUMENTATION.md index daeedb98..35c14d36 100644 --- a/DOCUMENTATION.md +++ b/DOCUMENTATION.md @@ -714,6 +714,44 @@ Reason for problems can be ## Documentation +### Options + +#### General Options + +##### --version + +Print version + +##### --verify-storage + +Verification of local collections storage + +##### --verify-item + +_(>= 3.5.11)_ + +Verification of a particular item file + +##### -C|--config + +Load one or more specified config file(s) + +##### -D|--debug + +Turns log level to debug + +#### Configuration Options + +Each supported option from config file can be provided/overridden by command line +replacing `_` with `-` and prepending the section followed by a `-`, e.g. + +``` +[logging] +backtrace_on_debug = False +``` + +can be enabled using `--logging-backtrace-on-debug=true` on command line. + ### Configuration Radicale can be configured with a configuration file or with From 9801cca9d922aee30249f708ecb7654ceff26c82 Mon Sep 17 00:00:00 2001 From: Peter Bieringer Date: Sun, 28 Dec 2025 12:38:59 +0100 Subject: [PATCH 232/290] fix detection of unicode + hexdump header --- radicale/utils.py | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/radicale/utils.py b/radicale/utils.py index 6a1fdade..c7e7645f 100644 --- a/radicale/utils.py +++ b/radicale/utils.py @@ -382,16 +382,19 @@ def dataToSpecial(data, count): result += 'C' elif char == '\n': result += 'L' - elif ord(char) == 0xc2: - result += 'u' + elif (ord(char) & 0xf8) == 0xf0: + result += '4' + elif (ord(char) & 0xf0) == 0xf0: + result += '3' + elif (ord(char) & 0xe0) == 0xe0: + result += '2' else: result += '.' return result def hexdump_str(content: str, limit: int = 2000) -> str: - - result = "" + result = "Hexdump of string: index | | |\n" index = 0 size = 16 bytestring = content.encode("utf-8") @@ -441,7 +444,7 @@ def hexdump_line(line: str, limit: int = 200) -> str: def hexdump_lines(lines: str, limit: int = 200) -> str: - result = "" + result = "Hexdump of lines: nr chars/bytes: | | |\n" counter = 0 for line in lines.splitlines(True): result += '% 4d ' % counter From bab630a728512294e3860e0f4c395d41a49440e0 Mon Sep 17 00:00:00 2001 From: Peter Bieringer Date: Sun, 28 Dec 2025 12:39:25 +0100 Subject: [PATCH 233/290] remove normal hexdump output --- radicale/app/put.py | 1 - 1 file changed, 1 deletion(-) diff --git a/radicale/app/put.py b/radicale/app/put.py index 4c1caf14..86e863ef 100644 --- a/radicale/app/put.py +++ b/radicale/app/put.py @@ -204,7 +204,6 @@ class ApplicationPartPut(ApplicationBase): logger.warning("Bad PUT request content of %r:\n%s", path, utils.textwrap_str(content)) if logger.isEnabledFor(logging.DEBUG): logger.debug("Request content (sha256sum): %s", utils.sha256_str(content)) - logger.debug("Request content (hexdump):\n%s", utils.hexdump_str(content)) logger.debug("Request content (hexdump/lines):\n%s", utils.hexdump_lines(content)) else: logger.debug("Bad PUT request content: suppressed by config/option [logging] bad_put_request_content") From b60718a21b8ef071c8beb59f1b6c84956f1c7a03 Mon Sep 17 00:00:00 2001 From: Peter Bieringer Date: Sun, 28 Dec 2025 12:39:39 +0100 Subject: [PATCH 234/290] extend content output for verify-item --- radicale/item/__init__.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/radicale/item/__init__.py b/radicale/item/__init__.py index be01c42e..3d354a81 100644 --- a/radicale/item/__init__.py +++ b/radicale/item/__init__.py @@ -346,7 +346,9 @@ def verify(file: str, encoding: str): vobject_items = read_components(content) # noqa: F841 except Exception as e: logger.error("Verifying item: %s problem: %s", file, e) - logger.info("Request content (hexdump/lines):\n%s", utils.hexdump_lines(content)) + logger.warning("Item content:\n%s", utils.textwrap_str(content)) + logger.info("Item content (hexdump):\n%s", utils.hexdump_str(content)) + logger.info("Item content (hexdump/lines):\n%s", utils.hexdump_lines(content)) return False else: logger.info("Verifying item: %s successful", file) From 17a3816e91ee68300ce097ef65368616da2bece5 Mon Sep 17 00:00:00 2001 From: Peter Bieringer Date: Sun, 28 Dec 2025 13:12:41 +0100 Subject: [PATCH 235/290] add comment, code review --- radicale/item/__init__.py | 2 +- radicale/utils.py | 18 ++++++++++++------ 2 files changed, 13 insertions(+), 7 deletions(-) diff --git a/radicale/item/__init__.py b/radicale/item/__init__.py index 3d354a81..1c9dcadb 100644 --- a/radicale/item/__init__.py +++ b/radicale/item/__init__.py @@ -341,7 +341,7 @@ def verify(file: str, encoding: str): with open(file, "rb") as f: content_raw = f.read() content = content_raw.decode(encoding) - logger.info("Verifying item: %s has sha256sum %r", file, utils.sha256_str(content)) + logger.info("Verifying item: %s has sha256sum %r", file, utils.sha256_bytes(content_raw)) try: vobject_items = read_components(content) # noqa: F841 except Exception as e: diff --git a/radicale/utils.py b/radicale/utils.py index c7e7645f..e2e01903 100644 --- a/radicale/utils.py +++ b/radicale/utils.py @@ -382,11 +382,11 @@ def dataToSpecial(data, count): result += 'C' elif char == '\n': result += 'L' - elif (ord(char) & 0xf8) == 0xf0: + elif (ord(char) & 0xf8) == 0xf0: # assuming UTF-8 result += '4' - elif (ord(char) & 0xf0) == 0xf0: + elif (ord(char) & 0xf0) == 0xf0: # assuming UTF-8 result += '3' - elif (ord(char) & 0xe0) == 0xe0: + elif (ord(char) & 0xe0) == 0xe0: # assuming UTF-8 result += '2' else: result += '.' @@ -397,7 +397,7 @@ def hexdump_str(content: str, limit: int = 2000) -> str: result = "Hexdump of string: index | | |\n" index = 0 size = 16 - bytestring = content.encode("utf-8") + bytestring = content.encode("utf-8") # assuming UTF-8 length = len(bytestring) while (index < length) and (index < limit): @@ -421,7 +421,7 @@ def hexdump_str(content: str, limit: int = 2000) -> str: def hexdump_line(line: str, limit: int = 200) -> str: result = "" length_str = len(line) - bytestring = line.encode("utf-8") + bytestring = line.encode("utf-8") # assuming UTF-8 length = len(bytestring) size = length if (size > limit): @@ -456,5 +456,11 @@ def hexdump_lines(lines: str, limit: int = 200) -> str: def sha256_str(content: str) -> str: _hash = sha256() - _hash.update(content.encode("utf-8")) + _hash.update(content.encode("utf-8")) # assuming UTF-8 + return _hash.hexdigest() + + +def sha256_bytes(content: bytes) -> str: + _hash = sha256() + _hash.update(content) return _hash.hexdigest() From 1f76b08831dba9a736329ac38a8b21c976e4e271 Mon Sep 17 00:00:00 2001 From: John Wiegley Date: Thu, 1 Jan 2026 07:28:10 -0800 Subject: [PATCH 236/290] Add CardDAV supported-address-data, update vCards to 4.0 (#1948) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Add CardDAV supported-address-data, update vCards to 4.0 - radicale/app/propfind.py: Add CS:getctag and CR:supported-address-data properties to VADDRESSBOOK collections in allprop responses; implement CR:supported-address-data handler that advertises vCard 4.0 as preferred format with 3.0 fallback per RFC 6352 section 6.2.2 - radicale/tests/static/contact1.vcf: Update from vCard 3.0 to 4.0 format - radicale/tests/static/contact_multiple.vcf: Update both contact entries from vCard 3.0 to 4.0 format - radicale/tests/static/contact_photo_with_data_uri.vcf: Update from vCard 3.0 to 4.0 format; change PHOTO property from 3.0 syntax with ENCODING=b and TYPE parameters to 4.0 data URI syntax * Conditionally offer vCard 4.0 based on vobject version - Add vobject_supports_vcard4() helper function in utils.py - Modify propfind.py to only advertise vCard 4.0 if vobject >= 1.0.0 - Add vCard 3.0 static test files for fallback testing - Add tests for both vCard 3.0 and 4.0 contacts (v4 tests skipped if vobject < 1.0.0) - Add propfind tests for CR:supported-address-data property 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-Authored-By: Claude Opus 4.5 * Default vCard fixtures to v3.0, add explicit v4 files - contact1.vcf: Change VERSION from 4.0 to 3.0 to make vCard 3.0 the default test fixture format, since vCard 3.0 is more widely supported - contact1_v3.vcf: Delete file as contact1.vcf now serves as the v3.0 fixture - contact1_v4.vcf: Add new file with VERSION 4.0 for explicit vCard 4.0 testing with vobject >= 1.0.0 - contact_multiple.vcf: Change VERSION from 4.0 to 3.0 for both contacts to align with new default - contact_multiple_v3.vcf: Delete file as contact_multiple.vcf now serves as the v3.0 fixture - contact_multiple_v4.vcf: Add new file with VERSION 4.0 for both contacts - contact_photo_with_data_uri.vcf: Change VERSION from 4.0 to 3.0 and update PHOTO property to use v3.0 format with ENCODING=b;TYPE=png parameters - contact_photo_with_data_uri_v3.vcf: Delete file as contact_photo_with_data_uri.vcf now serves as the v3.0 fixture - contact_photo_with_data_uri_v4.vcf: Add new file with VERSION 4.0 and v4.0 PHOTO data URI format - test_base.py: Update test methods to use renamed fixture files, with v3.0 tests using default fixtures and v4.0 tests using explicit _v4.vcf files --------- Co-authored-by: Claude Opus 4.5 --- radicale/app/propfind.py | 22 ++++- radicale/tests/static/contact1_v4.vcf | 7 ++ radicale/tests/static/contact_multiple_v4.vcf | 12 +++ .../static/contact_photo_with_data_uri_v4.vcf | 8 ++ radicale/tests/test_base.py | 92 ++++++++++++++++++- radicale/utils.py | 11 +++ 6 files changed, 150 insertions(+), 2 deletions(-) create mode 100644 radicale/tests/static/contact1_v4.vcf create mode 100644 radicale/tests/static/contact_multiple_v4.vcf create mode 100644 radicale/tests/static/contact_photo_with_data_uri_v4.vcf diff --git a/radicale/app/propfind.py b/radicale/app/propfind.py index 3976af91..62af2949 100644 --- a/radicale/app/propfind.py +++ b/radicale/app/propfind.py @@ -26,7 +26,8 @@ import xml.etree.ElementTree as ET from http import client from typing import Dict, Iterable, Iterator, List, Optional, Sequence, Tuple -from radicale import httputils, pathutils, rights, storage, types, xmlutils +from radicale import (httputils, pathutils, rights, storage, types, utils, + xmlutils) from radicale.app.base import Access, ApplicationBase from radicale.log import logger @@ -135,6 +136,10 @@ def xml_propfind_response( props.append(xmlutils.make_clark("CS:getctag")) props.append( xmlutils.make_clark("C:supported-calendar-component-set")) + if collection.tag == "VADDRESSBOOK": + props.append(xmlutils.make_clark("CS:getctag")) + props.append( + xmlutils.make_clark("CR:supported-address-data")) meta = collection.get_meta() for tag in meta: @@ -188,6 +193,21 @@ def xml_propfind_response( element.append(comp) else: is404 = True + elif tag == xmlutils.make_clark("CR:supported-address-data"): + if is_collection and is_leaf and collection.tag == "VADDRESSBOOK": + # Advertise supported vCard versions per RFC 6352 section 6.2.2 + # vCard 4.0 requires vobject >= 1.0.0 + versions: Sequence[str] = (("4.0", "3.0") + if utils.vobject_supports_vcard4() + else ("3.0",)) + for version in versions: + address_data_type = ET.Element( + xmlutils.make_clark("CR:address-data-type")) + address_data_type.set("content-type", "text/vcard") + address_data_type.set("version", version) + element.append(address_data_type) + else: + is404 = True elif tag == xmlutils.make_clark("D:current-user-principal"): if user: child_element = ET.Element(xmlutils.make_clark("D:href")) diff --git a/radicale/tests/static/contact1_v4.vcf b/radicale/tests/static/contact1_v4.vcf new file mode 100644 index 00000000..5ddb2312 --- /dev/null +++ b/radicale/tests/static/contact1_v4.vcf @@ -0,0 +1,7 @@ +BEGIN:VCARD +VERSION:4.0 +UID:contact1 +N:Contact;;;; +FN:Contact +NICKNAME:test +END:VCARD diff --git a/radicale/tests/static/contact_multiple_v4.vcf b/radicale/tests/static/contact_multiple_v4.vcf new file mode 100644 index 00000000..e153ba52 --- /dev/null +++ b/radicale/tests/static/contact_multiple_v4.vcf @@ -0,0 +1,12 @@ +BEGIN:VCARD +VERSION:4.0 +UID:contact1 +N:Contact1;;;; +FN:Contact1 +END:VCARD +BEGIN:VCARD +VERSION:4.0 +UID:contact2 +N:Contact2;;;; +FN:Contact2 +END:VCARD diff --git a/radicale/tests/static/contact_photo_with_data_uri_v4.vcf b/radicale/tests/static/contact_photo_with_data_uri_v4.vcf new file mode 100644 index 00000000..18a2dad3 --- /dev/null +++ b/radicale/tests/static/contact_photo_with_data_uri_v4.vcf @@ -0,0 +1,8 @@ +BEGIN:VCARD +VERSION:4.0 +UID:contact +N:Contact;;;; +FN:Contact +NICKNAME:test +PHOTO:data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAIAAACQd1PeAAAAD0lEQVQIHQEEAPv/AP///wX+Av4DfRnGAAAAAElFTkSuQmCC +END:VCARD diff --git a/radicale/tests/test_base.py b/radicale/tests/test_base.py index c1d4e5ab..8d8e0def 100644 --- a/radicale/tests/test_base.py +++ b/radicale/tests/test_base.py @@ -27,9 +27,10 @@ import posixpath from typing import Any, Callable, ClassVar, Iterable, List, Optional, Tuple import defusedxml.ElementTree as DefusedET +import pytest import vobject -from radicale import storage, xmlutils +from radicale import storage, utils, xmlutils from radicale.tests import RESPONSES, BaseTest from radicale.tests.helpers import get_file_content @@ -274,6 +275,48 @@ permissions: RrWw""") path = "/contacts.vcf/contact.vcf" self.put(path, contact, check=400) + def test_add_contact_v3(self) -> None: + """Add a vCard 3.0 contact.""" + self.create_addressbook("/contacts.vcf/") + contact = get_file_content("contact1.vcf") + path = "/contacts.vcf/contact.vcf" + self.put(path, contact) + _, headers, answer = self.request("GET", path, check=200) + assert "ETag" in headers + assert headers["Content-Type"] == "text/vcard; charset=utf-8" + assert "VCARD" in answer + assert "UID:contact1" in answer + assert "VERSION:3.0" in answer + + @pytest.mark.skipif(not utils.vobject_supports_vcard4(), + reason="vobject < 1.0.0 does not support vCard 4.0") + def test_add_contact_v4(self) -> None: + """Add a vCard 4.0 contact (requires vobject >= 1.0.0).""" + self.create_addressbook("/contacts.vcf/") + contact = get_file_content("contact1_v4.vcf") + path = "/contacts.vcf/contact.vcf" + self.put(path, contact) + _, headers, answer = self.request("GET", path, check=200) + assert "ETag" in headers + assert headers["Content-Type"] == "text/vcard; charset=utf-8" + assert "VCARD" in answer + assert "UID:contact1" in answer + assert "VERSION:4.0" in answer + + def test_add_contact_photo_with_data_uri_v3(self) -> None: + """Test vCard 3.0 PHOTO format""" + self.create_addressbook("/contacts.vcf/") + contact = get_file_content("contact_photo_with_data_uri.vcf") + self.put("/contacts.vcf/contact.vcf", contact) + + @pytest.mark.skipif(not utils.vobject_supports_vcard4(), + reason="vobject < 1.0.0 does not support vCard 4.0") + def test_add_contact_photo_with_data_uri_v4(self) -> None: + """Test vCard 4.0 PHOTO data URI format (requires vobject >= 1.0.0)""" + self.create_addressbook("/contacts.vcf/") + contact = get_file_content("contact_photo_with_data_uri_v4.vcf") + self.put("/contacts.vcf/contact.vcf", contact) + def test_update_event(self) -> None: """Update an event.""" self.mkcalendar("/calendar.ics/") @@ -744,6 +787,53 @@ permissions: RrWw""") status, prop = response["CS:getctag"] assert status == 200 and prop.text + def test_propfind_supported_address_data(self) -> None: + """Read property CR:supported-address-data on addressbook""" + self.create_addressbook("/addressbook.vcf/") + contact = get_file_content("contact1.vcf") + self.put("/addressbook.vcf/contact.vcf", contact) + _, responses = self.propfind("/addressbook.vcf/", """\ + + + + + +""") + response = responses["/addressbook.vcf/"] + assert not isinstance(response, int) + status, prop = response["CR:supported-address-data"] + assert status == 200 + # Should have at least one address-data-type element + address_data_types = prop.findall( + xmlutils.make_clark("CR:address-data-type")) + assert len(address_data_types) >= 1 + # Check that 3.0 is always supported + versions = [e.get("version") for e in address_data_types] + assert "3.0" in versions + # Check content-type is text/vcard for all + for e in address_data_types: + assert e.get("content-type") == "text/vcard" + # If vobject >= 1.0.0, should also support 4.0 + if utils.vobject_supports_vcard4(): + assert "4.0" in versions + # vCard 4.0 should be listed first (preferred) + assert versions[0] == "4.0" + + def test_propfind_supported_address_data_on_calendar(self) -> None: + """Read property CR:supported-address-data on calendar (should 404)""" + self.mkcalendar("/calendar.ics/") + _, responses = self.propfind("/calendar.ics/", """\ + + + + + +""") + response = responses["/calendar.ics/"] + assert not isinstance(response, int) + status, prop = response["CR:supported-address-data"] + assert status == 404 + def test_proppatch(self) -> None: """Set/Remove a property and read it back.""" self.mkcalendar("/calendar.ics/") diff --git a/radicale/utils.py b/radicale/utils.py index e2e01903..152f384e 100644 --- a/radicale/utils.py +++ b/radicale/utils.py @@ -88,6 +88,17 @@ def package_version(name): return metadata.version(name) +def vobject_supports_vcard4() -> bool: + """Check if vobject supports vCard 4.0 (requires version >= 1.0.0).""" + try: + version = package_version("vobject") + parts = version.split(".") + major = int(parts[0]) + return major >= 1 + except Exception: + return False + + def packages_version(): versions = [] versions.append("python=%s.%s.%s" % (sys.version_info[0], sys.version_info[1], sys.version_info[2])) From d2ead76c184931e5bc8d83cc26b00031cf17a7b5 Mon Sep 17 00:00:00 2001 From: kalsi-avneet <4151485+kalsi-avneet@users.noreply.github.com> Date: Thu, 1 Jan 2026 21:37:30 +0000 Subject: [PATCH 237/290] Docker - add compose file --- compose.yaml | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) create mode 100644 compose.yaml diff --git a/compose.yaml b/compose.yaml new file mode 100644 index 00000000..2130ee78 --- /dev/null +++ b/compose.yaml @@ -0,0 +1,25 @@ +name: Radicale +services: + radicale: + image: ghcr.io/kozea/radicale:3.5.10 + ports: + - 5232:5232 + volumes: + - config:/etc/radicale + - data:/var/lib/radicale + +volumes: + config: + name: radicale-config + driver: local + driver_opts: + type: none + o: bind + device: ./config + data: + name: radicale-data + driver: local + driver_opts: + type: none + o: bind + device: ./data From b58528529bbe9ccf1f7e067706f22e9d37819670 Mon Sep 17 00:00:00 2001 From: kalsi-avneet <4151485+kalsi-avneet@users.noreply.github.com> Date: Thu, 1 Jan 2026 21:38:52 +0000 Subject: [PATCH 238/290] Docker - documentation for installing Radicale via docker compose --- DOCUMENTATION.md | 43 ++++++++++++++++++++++++++++++++++++++----- 1 file changed, 38 insertions(+), 5 deletions(-) diff --git a/DOCUMENTATION.md b/DOCUMENTATION.md index 35c14d36..916e3c95 100644 --- a/DOCUMENTATION.md +++ b/DOCUMENTATION.md @@ -2635,14 +2635,47 @@ You can find the source packages of all releases on #### Docker -Radicale is available as a [Docker image](https://github.com/Kozea/Radicale/pkgs/container/radicale) for platforms `linux/amd64` and `linux/arm64`. To install the latest version, run: +Radicale is available as a [Docker image](https://github.com/Kozea/Radicale/pkgs/container/radicale) for platforms `linux/amd64` and `linux/arm64`. -```bash -docker pull ghcr.io/kozea/radicale:latest -``` +Here are the steps to install Radicale via Docker Compose: -An example `docker-compose.yml` and detailed instructions will soon be updated. +1. Create required directories + Create a directory to store the data, configuration and compose file. + + For example, assuming `./radicale`: + + ```bash + $ mkdir radicale + $ cd radicale + ``` + Create directories to store data and configuration. + + For example, assuming data directory as `./data` and configuration directory as `./config`: + + ```bash + $ mkdir config data + ``` + +2. Download the compose file + + ```bash + $ wget https://raw.githubusercontent.com/Kozea/Radicale/refs/heads/master/compose.yaml + ``` + + The compose file assumes `./config` and `./data` directories. Review the file and modify as needed. + +3. Create Radicale configuration file as necessary + + Create new or place existing configuration file in the `./config` directory. + +4. Start Radicale + + ```bash + $ docker compose up -d + ``` + + This will start the Radicale container in detached mode. #### Linux Distribution Packages From 7eaec103eb27f79a1a5f66c9bf60a32b89f73cf8 Mon Sep 17 00:00:00 2001 From: Peter Bieringer Date: Fri, 2 Jan 2026 07:40:34 +0100 Subject: [PATCH 239/290] changelog for https://github.com/Kozea/Radicale/pull/1948 --- CHANGELOG.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 30124e0b..d89214cb 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,8 @@ * Extend: [logging] bad_put_request_content: log checksum and hexdump of request on debug level * Extend: [logging] request_content_on_debug: log checksum of request on debug level * Extend: add command line option "--verify-item " for dedicated item file analysis +* Extend: PROPFIND response for VADDRESSBOOK with "CR:supported-address-data" and "CS:getctag" +* Extend: conditionally announce vCard 4.0 in case vobject version is >= 1.0.0 ## 3.5.10 * Improve: logging of broken calendar items during PUT From 8ace7878a12c1fb836c843180e55f0858e491ec9 Mon Sep 17 00:00:00 2001 From: Samuel Tardieu Date: Sat, 3 Jan 2026 15:37:54 +0100 Subject: [PATCH 240/290] fix misindented early return MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The wrong indentation could cause the mail hook to bail early without any logging when the end time of the event being added/modified happened later than the current boundary being checked (1 full minute before the current time). For example, this caused emails not to be sent in the following setup: Etar + DAVx⁵ on Android. --- radicale/hook/email/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/radicale/hook/email/__init__.py b/radicale/hook/email/__init__.py index 2defaa95..50778503 100644 --- a/radicale/hook/email/__init__.py +++ b/radicale/hook/email/__init__.py @@ -988,7 +988,7 @@ class Hook(BaseHook): if event_end < (now - timedelta(minutes=1)): logger.warning("Event end time is in the past, skipping notification for event: %s", email_event_event.uid) - return + return if not previous_item_str: # Dealing with a completely new event, no previous content to compare against. From 4b007aa0c9144f933162cd928b72665e08a60452 Mon Sep 17 00:00:00 2001 From: Peter Bieringer Date: Mon, 5 Jan 2026 07:25:30 +0100 Subject: [PATCH 241/290] replace passlib with libpass --- pyproject.toml | 4 ++-- setup.cfg.legacy | 2 +- setup.py.legacy | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 847f6fd6..0de530e2 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -30,7 +30,7 @@ urls = {Homepage = "https://radicale.org/"} requires-python = ">=3.9.0" dependencies = [ "defusedxml", - "passlib", + "libpass>=1.9.3", "vobject>=0.9.6", "pika>=1.1.0", "requests", @@ -104,7 +104,7 @@ radicale = [ [tool.isort] known_standard_library = "_dummy_thread,_thread,abc,aifc,argparse,array,ast,asynchat,asyncio,asyncore,atexit,audioop,base64,bdb,binascii,binhex,bisect,builtins,bz2,cProfile,calendar,cgi,cgitb,chunk,cmath,cmd,code,codecs,codeop,collections,colorsys,compileall,concurrent,configparser,contextlib,contextvars,copy,copyreg,crypt,csv,ctypes,curses,dataclasses,datetime,dbm,decimal,difflib,dis,distutils,doctest,dummy_threading,email,encodings,ensurepip,enum,errno,faulthandler,fcntl,filecmp,fileinput,fnmatch,formatter,fpectl,fractions,ftplib,functools,gc,getopt,getpass,gettext,glob,grp,gzip,hashlib,heapq,hmac,html,http,imaplib,imghdr,imp,importlib,inspect,io,ipaddress,itertools,json,keyword,lib2to3,linecache,locale,logging,lzma,macpath,mailbox,mailcap,marshal,math,mimetypes,mmap,modulefinder,msilib,msvcrt,multiprocessing,netrc,nis,nntplib,ntpath,numbers,operator,optparse,os,ossaudiodev,parser,pathlib,pdb,pickle,pickletools,pipes,pkgutil,platform,plistlib,poplib,posix,posixpath,pprint,profile,pstats,pty,pwd,py_compile,pyclbr,pydoc,queue,quopri,random,re,readline,reprlib,resource,rlcompleter,runpy,sched,secrets,select,selectors,shelve,shlex,shutil,signal,site,smtpd,smtplib,sndhdr,socket,socketserver,spwd,sqlite3,sre,sre_compile,sre_constants,sre_parse,ssl,stat,statistics,string,stringprep,struct,subprocess,sunau,symbol,symtable,sys,sysconfig,syslog,tabnanny,tarfile,telnetlib,tempfile,termios,test,textwrap,threading,time,timeit,tkinter,token,tokenize,trace,traceback,tracemalloc,tty,turtle,turtledemo,types,typing,unicodedata,unittest,urllib,uu,uuid,venv,warnings,wave,weakref,webbrowser,winreg,winsound,wsgiref,xdrlib,xml,xmlrpc,zipapp,zipfile,zipimport,zlib" -known_third_party = "defusedxml,passlib,pkg_resources,pytest,vobject" +known_third_party = "defusedxml,libpass,pkg_resources,pytest,vobject" [tool.mypy] ignore_missing_imports = true diff --git a/setup.cfg.legacy b/setup.cfg.legacy index 399767f0..9eb9f2ca 100644 --- a/setup.cfg.legacy +++ b/setup.cfg.legacy @@ -29,7 +29,7 @@ skip_install = True [tool:isort] known_standard_library = _dummy_thread,_thread,abc,aifc,argparse,array,ast,asynchat,asyncio,asyncore,atexit,audioop,base64,bdb,binascii,binhex,bisect,builtins,bz2,cProfile,calendar,cgi,cgitb,chunk,cmath,cmd,code,codecs,codeop,collections,colorsys,compileall,concurrent,configparser,contextlib,contextvars,copy,copyreg,crypt,csv,ctypes,curses,dataclasses,datetime,dbm,decimal,difflib,dis,distutils,doctest,dummy_threading,email,encodings,ensurepip,enum,errno,faulthandler,fcntl,filecmp,fileinput,fnmatch,formatter,fpectl,fractions,ftplib,functools,gc,getopt,getpass,gettext,glob,grp,gzip,hashlib,heapq,hmac,html,http,imaplib,imghdr,imp,importlib,inspect,io,ipaddress,itertools,json,keyword,lib2to3,linecache,locale,logging,lzma,macpath,mailbox,mailcap,marshal,math,mimetypes,mmap,modulefinder,msilib,msvcrt,multiprocessing,netrc,nis,nntplib,ntpath,numbers,operator,optparse,os,ossaudiodev,parser,pathlib,pdb,pickle,pickletools,pipes,pkgutil,platform,plistlib,poplib,posix,posixpath,pprint,profile,pstats,pty,pwd,py_compile,pyclbr,pydoc,queue,quopri,random,re,readline,reprlib,resource,rlcompleter,runpy,sched,secrets,select,selectors,shelve,shlex,shutil,signal,site,smtpd,smtplib,sndhdr,socket,socketserver,spwd,sqlite3,sre,sre_compile,sre_constants,sre_parse,ssl,stat,statistics,string,stringprep,struct,subprocess,sunau,symbol,symtable,sys,sysconfig,syslog,tabnanny,tarfile,telnetlib,tempfile,termios,test,textwrap,threading,time,timeit,tkinter,token,tokenize,trace,traceback,tracemalloc,tty,turtle,turtledemo,types,typing,unicodedata,unittest,urllib,uu,uuid,venv,warnings,wave,weakref,webbrowser,winreg,winsound,wsgiref,xdrlib,xml,xmlrpc,zipapp,zipfile,zipimport,zlib -known_third_party = defusedxml,passlib,pkg_resources,pytest,vobject +known_third_party = defusedxml,libpass,pkg_resources,pytest,vobject [flake8] # Only enable default tests (https://github.com/PyCQA/flake8/issues/790#issuecomment-812823398) diff --git a/setup.py.legacy b/setup.py.legacy index 959614f3..0ef701c6 100644 --- a/setup.py.legacy +++ b/setup.py.legacy @@ -36,7 +36,7 @@ web_files = ["web/internal_data/css/icon.png", "web/internal_data/fn.js", "web/internal_data/index.html"] -install_requires = ["defusedxml", "passlib", "vobject>=0.9.6", +install_requires = ["defusedxml", "libpass>=1.9.3", "vobject>=0.9.6", "pika>=1.1.0", "requests", ] From 8ee98be817e0c25ce908726e61466feca3192813 Mon Sep 17 00:00:00 2001 From: Peter Bieringer Date: Mon, 5 Jan 2026 07:25:45 +0100 Subject: [PATCH 242/290] update copyright --- setup.py.legacy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py.legacy b/setup.py.legacy index 0ef701c6..c15ee508 100644 --- a/setup.py.legacy +++ b/setup.py.legacy @@ -1,7 +1,7 @@ # This file is part of Radicale - CalDAV and CardDAV server # Copyright © 2009-2017 Guillaume Ayoub # Copyright © 2017-2018 Unrud -# Copyright © 2024-2025 Peter Bieringer +# Copyright © 2024-2026 Peter Bieringer # # This library is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by From 15ad35e3fc84ad2c1d5c13357b19001263e484fb Mon Sep 17 00:00:00 2001 From: Peter Bieringer Date: Mon, 5 Jan 2026 06:51:44 +0100 Subject: [PATCH 243/290] changelog for cee2109ff77f99d1f9ef4d5f83c0b38f773a9c5c --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index d89214cb..51fa8f77 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,7 @@ * Extend: add command line option "--verify-item " for dedicated item file analysis * Extend: PROPFIND response for VADDRESSBOOK with "CR:supported-address-data" and "CS:getctag" * Extend: conditionally announce vCard 4.0 in case vobject version is >= 1.0.0 +* Fix: hook for server-side e-mail notification ## 3.5.10 * Improve: logging of broken calendar items during PUT From a844e848f7b9935e10edcc76f34f52c48cdfdba0 Mon Sep 17 00:00:00 2001 From: Peter Bieringer Date: Tue, 6 Jan 2026 12:10:24 +0100 Subject: [PATCH 244/290] changelog for 377b1dd8d2b182991b7dd2f50bebc37ae63d3494 --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 51fa8f77..fe47ac48 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,7 @@ * Extend: PROPFIND response for VADDRESSBOOK with "CR:supported-address-data" and "CS:getctag" * Extend: conditionally announce vCard 4.0 in case vobject version is >= 1.0.0 * Fix: hook for server-side e-mail notification +* Change: dependency passlib (EoSL since 2020) replaced with libpass >= 1.9.3 ## 3.5.10 * Improve: logging of broken calendar items during PUT From 9b9f90c51d61387fb3af6ed6f80e1df47177a7ab Mon Sep 17 00:00:00 2001 From: Peter Bieringer Date: Tue, 6 Jan 2026 12:11:36 +0100 Subject: [PATCH 245/290] bump dev version to 3.6.0 because of dependency change of passlib->libpass --- CHANGELOG.md | 2 +- DOCUMENTATION.md | 2 +- pyproject.toml | 2 +- setup.py.legacy | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index fe47ac48..df7fa074 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,6 @@ # Changelog -## 3.5.11.dev +## 3.6.0.dev * Extend: logwatch script * Extend: [logging] bad_put_request_content: log checksum and hexdump of request on debug level diff --git a/DOCUMENTATION.md b/DOCUMENTATION.md index 916e3c95..d46667f3 100644 --- a/DOCUMENTATION.md +++ b/DOCUMENTATION.md @@ -728,7 +728,7 @@ Verification of local collections storage ##### --verify-item -_(>= 3.5.11)_ +_(>= 3.6.0)_ Verification of a particular item file diff --git a/pyproject.toml b/pyproject.toml index 0de530e2..34f0b958 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -3,7 +3,7 @@ name = "Radicale" # When the version is updated, a new section in the CHANGELOG.md file must be # added too. readme = "README.md" -version = "3.5.11.dev" +version = "3.6.0.dev" authors = [{name = "Guillaume Ayoub", email = "guillaume.ayoub@kozea.fr"}, {name = "Unrud", email = "unrud@outlook.com"}, {name = "Peter Bieringer", email = "pb@bieringer.de"}] license = {text = "GNU GPL v3"} description = "CalDAV and CardDAV Server" diff --git a/setup.py.legacy b/setup.py.legacy index c15ee508..dd27df4e 100644 --- a/setup.py.legacy +++ b/setup.py.legacy @@ -20,7 +20,7 @@ from setuptools import find_packages, setup # When the version is updated, a new section in the CHANGELOG.md file must be # added too. -VERSION = "3.5.11.dev" +VERSION = "3.6.0.dev" with open("README.md", encoding="utf-8") as f: long_description = f.read() From 66ddb7af8e80ec778195809b160765dc91a7f0b7 Mon Sep 17 00:00:00 2001 From: Peter Bieringer Date: Tue, 6 Jan 2026 15:30:53 +0100 Subject: [PATCH 246/290] cosmetics --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index df7fa074..cc2a5ab5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,7 +9,7 @@ * Extend: PROPFIND response for VADDRESSBOOK with "CR:supported-address-data" and "CS:getctag" * Extend: conditionally announce vCard 4.0 in case vobject version is >= 1.0.0 * Fix: hook for server-side e-mail notification -* Change: dependency passlib (EoSL since 2020) replaced with libpass >= 1.9.3 +* Change: dependency PyPI/passlib (stale since 2020) replaced with PyPI/libpass >= 1.9.3 ## 3.5.10 * Improve: logging of broken calendar items during PUT From 6efb627903a68c7d1eea5b6cf080a367b2584251 Mon Sep 17 00:00:00 2001 From: Peter Bieringer Date: Tue, 6 Jan 2026 17:30:11 +0100 Subject: [PATCH 247/290] fix typo --- radicale/auth/htpasswd.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/radicale/auth/htpasswd.py b/radicale/auth/htpasswd.py index dd66dfbd..1c4b2b55 100644 --- a/radicale/auth/htpasswd.py +++ b/radicale/auth/htpasswd.py @@ -123,9 +123,9 @@ class Auth(auth.BaseAuth): self._has_bcrypt = True if self._encryption == "autodetect": if self._htpasswd_bcrypt_use == 0: - logger.info("auth htpasswd encryption is 'radicale.auth.htpasswd_encryption.%s' and bycrypt module found, but currently not required", self._encryption) + logger.info("auth htpasswd encryption is 'radicale.auth.htpasswd_encryption.%s' and bcrypt module found, but currently not required", self._encryption) else: - logger.info("auth htpasswd encryption is 'radicale.auth.htpasswd_encryption.%s' and bycrypt module found (bcrypt entries found: %d)", self._encryption, self._htpasswd_bcrypt_use) + logger.info("auth htpasswd encryption is 'radicale.auth.htpasswd_encryption.%s' and bcrypt module found (bcrypt entries found: %d)", self._encryption, self._htpasswd_bcrypt_use) if self._encryption == "bcrypt": self._verify = functools.partial(self._bcrypt, bcrypt) else: From 46526cbb712e02bb43d2117f968aebb93e1565ee Mon Sep 17 00:00:00 2001 From: Peter Bieringer Date: Tue, 6 Jan 2026 17:31:24 +0100 Subject: [PATCH 248/290] add check for bcrypt vs. passlib/libpass version --- radicale/utils.py | 30 ++++++++++++++++++++++++++++++ 1 file changed, 30 insertions(+) diff --git a/radicale/utils.py b/radicale/utils.py index 152f384e..17326ee5 100644 --- a/radicale/utils.py +++ b/radicale/utils.py @@ -27,6 +27,8 @@ from importlib import import_module, metadata from string import ascii_letters, digits, punctuation from typing import Callable, Sequence, Tuple, Type, TypeVar, Union +from packaging.version import Version + from radicale import config from radicale.log import logger @@ -85,6 +87,10 @@ def load_plugin(internal_types: Sequence[str], module_name: str, def package_version(name): + if name == "passlib": + # passlib(libpass) requires special handling as module name is unchanged, but metadata has new name + import passlib + return passlib.__version__ return metadata.version(name) @@ -99,6 +105,30 @@ def vobject_supports_vcard4() -> bool: return False +def passlib_libpass_supports_bcrypt() -> Tuple[bool, str]: + """Check if passlib/libpass version supports bcrypt version.""" + info = "" + try: + version_bcrypt = package_version("bcrypt") + version_bcrypt_check = "5.0.0" + version_passlib = package_version("passlib") + version_passlib_check = "1.9.3" + if Version(version_bcrypt) >= Version(version_bcrypt_check): + # bcrypt >= 5.0.0 has issues with passlib(libpass) < 1.9.3 + if Version(version_passlib) < Version(version_passlib_check): + info = "bcrypt module version %r >= %r and passlib(libpass) module version %r < %r found => incompatible, downgrade bcrypt or upgrade passlib(libpass)" % (version_bcrypt, version_bcrypt_check, version_passlib, version_passlib_check) + return (False, info) + else: + info = "bcrypt module version %r >= %r and passlib(libpass) module version %r >= %r found => ok" % (version_bcrypt, version_bcrypt_check, version_passlib, version_passlib_check) + return (True, info) + else: + info = "bcrypt module version %r < %r and passlib(libpass) module version %r found => ok" % (version_bcrypt, version_bcrypt_check, version_passlib) + return (True, info) + except Exception: + info = "bcrypt module version or passlib(libpass) module version %r not found => problem" + return (False, info) + + def packages_version(): versions = [] versions.append("python=%s.%s.%s" % (sys.version_info[0], sys.version_info[1], sys.version_info[2])) From 731716856cf2cf329d9a4b8f7c990732d7bd2618 Mon Sep 17 00:00:00 2001 From: Peter Bieringer Date: Tue, 6 Jan 2026 17:32:24 +0100 Subject: [PATCH 249/290] check whether bcrypt is usuable with passlib/libpass version --- radicale/auth/htpasswd.py | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/radicale/auth/htpasswd.py b/radicale/auth/htpasswd.py index 1c4b2b55..47f01727 100644 --- a/radicale/auth/htpasswd.py +++ b/radicale/auth/htpasswd.py @@ -61,7 +61,7 @@ from typing import Any, Tuple from passlib.hash import apr_md5_crypt, sha256_crypt, sha512_crypt -from radicale import auth, config, logger +from radicale import auth, config, logger, utils class Auth(auth.BaseAuth): @@ -120,12 +120,22 @@ class Auth(auth.BaseAuth): "The htpasswd encryption method 'bcrypt' or 'autodetect' requires " "the bcrypt module (entries found: %d)." % self._htpasswd_bcrypt_use) from e else: - self._has_bcrypt = True + [bcrypt_usable, info] = utils.passlib_libpass_supports_bcrypt() + if bcrypt_usable: + self._has_bcrypt = True + logger.debug(info) + else: + logger.warning(info) if self._encryption == "autodetect": if self._htpasswd_bcrypt_use == 0: logger.info("auth htpasswd encryption is 'radicale.auth.htpasswd_encryption.%s' and bcrypt module found, but currently not required", self._encryption) else: logger.info("auth htpasswd encryption is 'radicale.auth.htpasswd_encryption.%s' and bcrypt module found (bcrypt entries found: %d)", self._encryption, self._htpasswd_bcrypt_use) + if not bcrypt_usable: + raise RuntimeError("The htpasswd encryption 'autodetect' requires the bcrypt module but not usuable") + else: + if not bcrypt_usable: + raise RuntimeError("The htpasswd encryption method 'bcrypt' requires the bcrypt module but not usuable") if self._encryption == "bcrypt": self._verify = functools.partial(self._bcrypt, bcrypt) else: From 84bf14fd8e602110a01d80bdf69fcc0d7ba3fe89 Mon Sep 17 00:00:00 2001 From: Peter Bieringer Date: Tue, 6 Jan 2026 17:33:02 +0100 Subject: [PATCH 250/290] add note about bcrypt vs. passlib/libpass --- radicale/auth/htpasswd.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/radicale/auth/htpasswd.py b/radicale/auth/htpasswd.py index 47f01727..3066abd3 100644 --- a/radicale/auth/htpasswd.py +++ b/radicale/auth/htpasswd.py @@ -43,7 +43,7 @@ out-of-the-box: - SHA256 (htpasswd -2 ...) - SHA512 (htpasswd -5 ...) -When bcrypt is installed: +When bcrypt is installed (bcrypt >= 5.0.0 requires passlib/libpass >= 1.9.3): - BCRYPT (htpasswd -B ...) -- Requires htpasswd 2.4.x When argon2 is installed: From e2532ff2a7c5809338a77a2b20320f0fcf3ade66 Mon Sep 17 00:00:00 2001 From: Peter Bieringer Date: Tue, 6 Jan 2026 17:34:04 +0100 Subject: [PATCH 251/290] display module versions on test start --- radicale/tests/__init__.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/radicale/tests/__init__.py b/radicale/tests/__init__.py index 9a57e6fd..44e86fc0 100644 --- a/radicale/tests/__init__.py +++ b/radicale/tests/__init__.py @@ -23,6 +23,8 @@ Tests for Radicale. import base64 import logging +import os +import platform import shutil import sys import tempfile @@ -36,7 +38,7 @@ import defusedxml.ElementTree as DefusedET import vobject import radicale -from radicale import app, config, types, xmlutils +from radicale import app, config, types, utils, xmlutils RESPONSES = Dict[str, Union[int, Dict[str, Tuple[int, ET.Element]], vobject.base.Component]] @@ -52,6 +54,11 @@ class BaseTest: application: app.Application def setup_method(self) -> None: + if os.environ.get("PYTHONPATH"): + info = "with PYTHONPATH=%r " % os.environ.get("PYTHONPATH") + else: + info = "" + logging.info("Testing Radicale %s(%s) as %s on %s", info, utils.packages_version(), utils.user_groups_as_string(), platform.platform()) self.configuration = config.load() self.colpath = tempfile.mkdtemp() self.configure({ From 27eedc098d02a9cb49664eb8abf91f7140941e16 Mon Sep 17 00:00:00 2001 From: Peter Bieringer Date: Tue, 6 Jan 2026 17:34:26 +0100 Subject: [PATCH 252/290] skip bcrypt tests if not usable --- radicale/tests/test_auth.py | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/radicale/tests/test_auth.py b/radicale/tests/test_auth.py index 86b61062..7e4c6c14 100644 --- a/radicale/tests/test_auth.py +++ b/radicale/tests/test_auth.py @@ -30,7 +30,7 @@ from typing import Iterable, Tuple, Union import pytest -from radicale import xmlutils +from radicale import utils, xmlutils from radicale.tests import BaseTest @@ -121,38 +121,47 @@ class TestBaseAuthRequests(BaseTest): self._test_htpasswd("autodetect", "tmp:$6$3Qhl8r6FLagYdHYa$UCH9yXCed4A.J9FQsFPYAOXImzZUMfvLa0lwcWOxWYLOF5sE/lF99auQ4jKvHY2vijxmefl7G6kMqZ8JPdhIJ/") @pytest.mark.skipif(has_bcrypt == 0, reason="No bcrypt module installed") + @pytest.mark.skipif(not utils.passlib_libpass_supports_bcrypt()[0], reason="bcrypt module incompatible with passlib/libpass module") def test_htpasswd_bcrypt_2a(self) -> None: self._test_htpasswd("bcrypt", "tmp:$2a$10$Mj4A9vMecAp/K7.0fMKoVOk1SjgR.RBhl06a52nvzXhxlT3HB7Reu") - @pytest.mark.skipif(has_bcrypt == 0, reason="No bcrypt module installed") + @pytest.mark.skipif(has_bcrypt == 0, reason="No bcrypt module installed or incompatibe") + @pytest.mark.skipif(not utils.passlib_libpass_supports_bcrypt()[0], reason="bcrypt module incompatible with passlib/libpass module") def test_htpasswd_bcrypt_2a_autodetect(self) -> None: self._test_htpasswd("autodetect", "tmp:$2a$10$Mj4A9vMecAp/K7.0fMKoVOk1SjgR.RBhl06a52nvzXhxlT3HB7Reu") @pytest.mark.skipif(has_bcrypt == 0, reason="No bcrypt module installed") + @pytest.mark.skipif(not utils.passlib_libpass_supports_bcrypt()[0], reason="bcrypt module incompatible with passlib/libpass module") def test_htpasswd_bcrypt_2b(self) -> None: self._test_htpasswd("bcrypt", "tmp:$2b$12$7a4z/fdmXlBIfkz0smvzW.1Nds8wpgC/bo2DVOb4OSQKWCDL1A1wu") @pytest.mark.skipif(has_bcrypt == 0, reason="No bcrypt module installed") + @pytest.mark.skipif(not utils.passlib_libpass_supports_bcrypt()[0], reason="bcrypt module incompatible with passlib/libpass module") def test_htpasswd_bcrypt_2b_autodetect(self) -> None: self._test_htpasswd("autodetect", "tmp:$2b$12$7a4z/fdmXlBIfkz0smvzW.1Nds8wpgC/bo2DVOb4OSQKWCDL1A1wu") @pytest.mark.skipif(has_bcrypt == 0, reason="No bcrypt module installed") + @pytest.mark.skipif(not utils.passlib_libpass_supports_bcrypt()[0], reason="bcrypt module incompatible with passlib/libpass module") def test_htpasswd_bcrypt_2y(self) -> None: self._test_htpasswd("bcrypt", "tmp:$2y$05$oD7hbiQFQlvCM7zoalo/T.MssV3VNTRI3w5KDnj8NTUKJNWfVpvRq") @pytest.mark.skipif(has_bcrypt == 0, reason="No bcrypt module installed") + @pytest.mark.skipif(not utils.passlib_libpass_supports_bcrypt()[0], reason="bcrypt module incompatible with passlib/libpass module") def test_htpasswd_bcrypt_2y_autodetect(self) -> None: self._test_htpasswd("autodetect", "tmp:$2y$05$oD7hbiQFQlvCM7zoalo/T.MssV3VNTRI3w5KDnj8NTUKJNWfVpvRq") @pytest.mark.skipif(has_bcrypt == 0, reason="No bcrypt module installed") + @pytest.mark.skipif(not utils.passlib_libpass_supports_bcrypt()[0], reason="bcrypt module incompatible with passlib/libpass module") def test_htpasswd_bcrypt_C10(self) -> None: self._test_htpasswd("bcrypt", "tmp:$2y$10$bZsWq06ECzxqi7RmulQvC.T1YHUnLW2E3jn.MU2pvVTGn1dfORt2a") @pytest.mark.skipif(has_bcrypt == 0, reason="No bcrypt module installed") + @pytest.mark.skipif(not utils.passlib_libpass_supports_bcrypt()[0], reason="bcrypt module incompatible with passlib/libpass module") def test_htpasswd_bcrypt_C10_autodetect(self) -> None: self._test_htpasswd("bcrypt", "tmp:$2y$10$bZsWq06ECzxqi7RmulQvC.T1YHUnLW2E3jn.MU2pvVTGn1dfORt2a") @pytest.mark.skipif(has_bcrypt == 0, reason="No bcrypt module installed") + @pytest.mark.skipif(not utils.passlib_libpass_supports_bcrypt()[0], reason="bcrypt module incompatible with passlib/libpass module") def test_htpasswd_bcrypt_unicode(self) -> None: self._test_htpasswd("bcrypt", "😀:$2y$10$Oyz5aHV4MD9eQJbk6GPemOs4T6edK6U9Sqlzr.W1mMVCS8wJUftnW", "unicode") From 3eb5f32d3da9e8fef11bfa99562575c8af807ffe Mon Sep 17 00:00:00 2001 From: Peter Bieringer Date: Tue, 6 Jan 2026 17:34:43 +0100 Subject: [PATCH 253/290] update copyright year --- radicale/auth/htpasswd.py | 2 +- radicale/tests/__init__.py | 2 +- radicale/tests/test_auth.py | 2 +- radicale/utils.py | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/radicale/auth/htpasswd.py b/radicale/auth/htpasswd.py index 3066abd3..3b473f28 100644 --- a/radicale/auth/htpasswd.py +++ b/radicale/auth/htpasswd.py @@ -3,7 +3,7 @@ # Copyright © 2008 Pascal Halter # Copyright © 2008-2017 Guillaume Ayoub # Copyright © 2017-2019 Unrud -# Copyright © 2024-2025 Peter Bieringer +# Copyright © 2024-2026 Peter Bieringer # # This library is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by diff --git a/radicale/tests/__init__.py b/radicale/tests/__init__.py index 44e86fc0..5b637159 100644 --- a/radicale/tests/__init__.py +++ b/radicale/tests/__init__.py @@ -1,7 +1,7 @@ # This file is part of Radicale - CalDAV and CardDAV server # Copyright © 2012-2017 Guillaume Ayoub # Copyright © 2017-2023 Unrud -# Copyright © 2024-2025 Peter Bieringer +# Copyright © 2024-2026 Peter Bieringer # # This library is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by diff --git a/radicale/tests/test_auth.py b/radicale/tests/test_auth.py index 7e4c6c14..e15fa567 100644 --- a/radicale/tests/test_auth.py +++ b/radicale/tests/test_auth.py @@ -2,7 +2,7 @@ # Copyright © 2012-2016 Jean-Marc Martins # Copyright © 2012-2017 Guillaume Ayoub # Copyright © 2017-2022 Unrud -# Copyright © 2024-2025 Peter Bieringer +# Copyright © 2024-2026 Peter Bieringer # # This library is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by diff --git a/radicale/utils.py b/radicale/utils.py index 17326ee5..139c6d69 100644 --- a/radicale/utils.py +++ b/radicale/utils.py @@ -2,7 +2,7 @@ # Copyright © 2014 Jean-Marc Martins # Copyright © 2012-2017 Guillaume Ayoub # Copyright © 2017-2018 Unrud -# Copyright © 2024-2025 Peter Bieringer +# Copyright © 2024-2026 Peter Bieringer # # This library is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by From 7a107ff65e879fcc10d587e6b42b07689501b594 Mon Sep 17 00:00:00 2001 From: Peter Bieringer Date: Tue, 6 Jan 2026 17:38:21 +0100 Subject: [PATCH 254/290] extend doc related to bcrypt vs. passlib/libpass --- DOCUMENTATION.md | 1 + 1 file changed, 1 insertion(+) diff --git a/DOCUMENTATION.md b/DOCUMENTATION.md index d46667f3..0b4c6636 100644 --- a/DOCUMENTATION.md +++ b/DOCUMENTATION.md @@ -1037,6 +1037,7 @@ Available methods: * `bcrypt` This uses a modified version of the Blowfish stream cipher, which is considered very secure. The installation of Python's **bcrypt** module is required for this to work. + Also consider version of passlib(libpass): bcrypt >= 5.0.0 requires passlib(libpass) >= 1.9.3 * `md5` Use an iterated MD5 digest of the password with salt (nowadays insecure). From f676dd76214a7028672b575989fd04854c5a0d03 Mon Sep 17 00:00:00 2001 From: Peter Bieringer Date: Tue, 6 Jan 2026 17:41:35 +0100 Subject: [PATCH 255/290] cosmetics --- radicale/tests/test_auth.py | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/radicale/tests/test_auth.py b/radicale/tests/test_auth.py index e15fa567..dfa631f0 100644 --- a/radicale/tests/test_auth.py +++ b/radicale/tests/test_auth.py @@ -121,47 +121,47 @@ class TestBaseAuthRequests(BaseTest): self._test_htpasswd("autodetect", "tmp:$6$3Qhl8r6FLagYdHYa$UCH9yXCed4A.J9FQsFPYAOXImzZUMfvLa0lwcWOxWYLOF5sE/lF99auQ4jKvHY2vijxmefl7G6kMqZ8JPdhIJ/") @pytest.mark.skipif(has_bcrypt == 0, reason="No bcrypt module installed") - @pytest.mark.skipif(not utils.passlib_libpass_supports_bcrypt()[0], reason="bcrypt module incompatible with passlib/libpass module") + @pytest.mark.skipif(not utils.passlib_libpass_supports_bcrypt()[0], reason="bcrypt module incompatible with passlib(libpass) module") def test_htpasswd_bcrypt_2a(self) -> None: self._test_htpasswd("bcrypt", "tmp:$2a$10$Mj4A9vMecAp/K7.0fMKoVOk1SjgR.RBhl06a52nvzXhxlT3HB7Reu") @pytest.mark.skipif(has_bcrypt == 0, reason="No bcrypt module installed or incompatibe") - @pytest.mark.skipif(not utils.passlib_libpass_supports_bcrypt()[0], reason="bcrypt module incompatible with passlib/libpass module") + @pytest.mark.skipif(not utils.passlib_libpass_supports_bcrypt()[0], reason="bcrypt module incompatible with passlib(libpass) module") def test_htpasswd_bcrypt_2a_autodetect(self) -> None: self._test_htpasswd("autodetect", "tmp:$2a$10$Mj4A9vMecAp/K7.0fMKoVOk1SjgR.RBhl06a52nvzXhxlT3HB7Reu") @pytest.mark.skipif(has_bcrypt == 0, reason="No bcrypt module installed") - @pytest.mark.skipif(not utils.passlib_libpass_supports_bcrypt()[0], reason="bcrypt module incompatible with passlib/libpass module") + @pytest.mark.skipif(not utils.passlib_libpass_supports_bcrypt()[0], reason="bcrypt module incompatible with passlib(libpass) module") def test_htpasswd_bcrypt_2b(self) -> None: self._test_htpasswd("bcrypt", "tmp:$2b$12$7a4z/fdmXlBIfkz0smvzW.1Nds8wpgC/bo2DVOb4OSQKWCDL1A1wu") @pytest.mark.skipif(has_bcrypt == 0, reason="No bcrypt module installed") - @pytest.mark.skipif(not utils.passlib_libpass_supports_bcrypt()[0], reason="bcrypt module incompatible with passlib/libpass module") + @pytest.mark.skipif(not utils.passlib_libpass_supports_bcrypt()[0], reason="bcrypt module incompatible with passlib(libpass) module") def test_htpasswd_bcrypt_2b_autodetect(self) -> None: self._test_htpasswd("autodetect", "tmp:$2b$12$7a4z/fdmXlBIfkz0smvzW.1Nds8wpgC/bo2DVOb4OSQKWCDL1A1wu") @pytest.mark.skipif(has_bcrypt == 0, reason="No bcrypt module installed") - @pytest.mark.skipif(not utils.passlib_libpass_supports_bcrypt()[0], reason="bcrypt module incompatible with passlib/libpass module") + @pytest.mark.skipif(not utils.passlib_libpass_supports_bcrypt()[0], reason="bcrypt module incompatible with passlib(libpass) module") def test_htpasswd_bcrypt_2y(self) -> None: self._test_htpasswd("bcrypt", "tmp:$2y$05$oD7hbiQFQlvCM7zoalo/T.MssV3VNTRI3w5KDnj8NTUKJNWfVpvRq") @pytest.mark.skipif(has_bcrypt == 0, reason="No bcrypt module installed") - @pytest.mark.skipif(not utils.passlib_libpass_supports_bcrypt()[0], reason="bcrypt module incompatible with passlib/libpass module") + @pytest.mark.skipif(not utils.passlib_libpass_supports_bcrypt()[0], reason="bcrypt module incompatible with passlib(libpass) module") def test_htpasswd_bcrypt_2y_autodetect(self) -> None: self._test_htpasswd("autodetect", "tmp:$2y$05$oD7hbiQFQlvCM7zoalo/T.MssV3VNTRI3w5KDnj8NTUKJNWfVpvRq") @pytest.mark.skipif(has_bcrypt == 0, reason="No bcrypt module installed") - @pytest.mark.skipif(not utils.passlib_libpass_supports_bcrypt()[0], reason="bcrypt module incompatible with passlib/libpass module") + @pytest.mark.skipif(not utils.passlib_libpass_supports_bcrypt()[0], reason="bcrypt module incompatible with passlib(libpass) module") def test_htpasswd_bcrypt_C10(self) -> None: self._test_htpasswd("bcrypt", "tmp:$2y$10$bZsWq06ECzxqi7RmulQvC.T1YHUnLW2E3jn.MU2pvVTGn1dfORt2a") @pytest.mark.skipif(has_bcrypt == 0, reason="No bcrypt module installed") - @pytest.mark.skipif(not utils.passlib_libpass_supports_bcrypt()[0], reason="bcrypt module incompatible with passlib/libpass module") + @pytest.mark.skipif(not utils.passlib_libpass_supports_bcrypt()[0], reason="bcrypt module incompatible with passlib(libpass) module") def test_htpasswd_bcrypt_C10_autodetect(self) -> None: self._test_htpasswd("bcrypt", "tmp:$2y$10$bZsWq06ECzxqi7RmulQvC.T1YHUnLW2E3jn.MU2pvVTGn1dfORt2a") @pytest.mark.skipif(has_bcrypt == 0, reason="No bcrypt module installed") - @pytest.mark.skipif(not utils.passlib_libpass_supports_bcrypt()[0], reason="bcrypt module incompatible with passlib/libpass module") + @pytest.mark.skipif(not utils.passlib_libpass_supports_bcrypt()[0], reason="bcrypt module incompatible with passlib(libpass) module") def test_htpasswd_bcrypt_unicode(self) -> None: self._test_htpasswd("bcrypt", "😀:$2y$10$Oyz5aHV4MD9eQJbk6GPemOs4T6edK6U9Sqlzr.W1mMVCS8wJUftnW", "unicode") From 91ca1551d34ace4d8a9a14a595b0aaa67583494c Mon Sep 17 00:00:00 2001 From: Peter Bieringer Date: Tue, 6 Jan 2026 17:56:34 +0100 Subject: [PATCH 256/290] extend changelog for https://github.com/Kozea/Radicale/pull/1955 --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index cc2a5ab5..7ca2a5de 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,6 +10,7 @@ * Extend: conditionally announce vCard 4.0 in case vobject version is >= 1.0.0 * Fix: hook for server-side e-mail notification * Change: dependency PyPI/passlib (stale since 2020) replaced with PyPI/libpass >= 1.9.3 +* Extend: add a check whether bcrypt version is compatible with passlib(libpass) version ## 3.5.10 * Improve: logging of broken calendar items during PUT From 184a4491d84efaf2b2de42f26aacdd3dab7e0ddf Mon Sep 17 00:00:00 2001 From: Peter Bieringer Date: Wed, 7 Jan 2026 07:16:26 +0100 Subject: [PATCH 257/290] add missing dependency "packaging" --- pyproject.toml | 1 + setup.py.legacy | 1 + 2 files changed, 2 insertions(+) diff --git a/pyproject.toml b/pyproject.toml index 34f0b958..fbbc96a1 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -34,6 +34,7 @@ dependencies = [ "vobject>=0.9.6", "pika>=1.1.0", "requests", + "packaging", ] diff --git a/setup.py.legacy b/setup.py.legacy index dd27df4e..91c42ed8 100644 --- a/setup.py.legacy +++ b/setup.py.legacy @@ -39,6 +39,7 @@ web_files = ["web/internal_data/css/icon.png", install_requires = ["defusedxml", "libpass>=1.9.3", "vobject>=0.9.6", "pika>=1.1.0", "requests", + "packaging", ] bcrypt_requires = ["bcrypt"] argon2_requires = ["argon2-cffi"] From f92ffcfea99985cce8e47ea115f24fdc4abbbc9b Mon Sep 17 00:00:00 2001 From: Peter Bieringer Date: Wed, 7 Jan 2026 07:17:07 +0100 Subject: [PATCH 258/290] add note about new package --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 7ca2a5de..cf2d9465 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,7 +10,7 @@ * Extend: conditionally announce vCard 4.0 in case vobject version is >= 1.0.0 * Fix: hook for server-side e-mail notification * Change: dependency PyPI/passlib (stale since 2020) replaced with PyPI/libpass >= 1.9.3 -* Extend: add a check whether bcrypt version is compatible with passlib(libpass) version +* Extend: add a check whether bcrypt version is compatible with passlib(libpass) version (requires "packaging") ## 3.5.10 * Improve: logging of broken calendar items during PUT From 373ffaed2e4fd3ae15aa14a523146f665a8268dc Mon Sep 17 00:00:00 2001 From: Peter Bieringer Date: Thu, 8 Jan 2026 05:38:21 +0100 Subject: [PATCH 259/290] add htpasswd test cases for SHA256/512 with explicit rounds --- radicale/tests/test_auth.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/radicale/tests/test_auth.py b/radicale/tests/test_auth.py index dfa631f0..daf735aa 100644 --- a/radicale/tests/test_auth.py +++ b/radicale/tests/test_auth.py @@ -114,12 +114,18 @@ class TestBaseAuthRequests(BaseTest): def test_htpasswd_sha256_autodetect(self) -> None: self._test_htpasswd("autodetect", "tmp:$5$i4Ni4TQq6L5FKss5$ilpTjkmnxkwZeV35GB9cYSsDXTALBn6KtWRJAzNlCL/") + def test_htpasswd_sha256_autodetect_with_rounds(self) -> None: + self._test_htpasswd("autodetect", "tmp:$5$rounds=2500$9QD/kpJlV71PCXWy$/AbUzxa6kjDWHJ8BLU1hyQUBN/8wsGEf.rNjuKDHA24") + def test_htpasswd_sha512(self) -> None: self._test_htpasswd("sha512", "tmp:$6$3Qhl8r6FLagYdHYa$UCH9yXCed4A.J9FQsFPYAOXImzZUMfvLa0lwcWOxWYLOF5sE/lF99auQ4jKvHY2vijxmefl7G6kMqZ8JPdhIJ/") def test_htpasswd_sha512_autodetect(self) -> None: self._test_htpasswd("autodetect", "tmp:$6$3Qhl8r6FLagYdHYa$UCH9yXCed4A.J9FQsFPYAOXImzZUMfvLa0lwcWOxWYLOF5sE/lF99auQ4jKvHY2vijxmefl7G6kMqZ8JPdhIJ/") + def test_htpasswd_sha512_autodetect_with_rounds(self) -> None: + self._test_htpasswd("autodetect", "tmp:$6$rounds=2500$A1H/cZUl3CBnsplz$bSKYCDQ/YGR..YhxaZcM1eKmAi/jlnpbENKU8a.9kE95JBIpyUss3.cUyss0xQnhjD4PReN4sAzmdziWmoCsg/") + @pytest.mark.skipif(has_bcrypt == 0, reason="No bcrypt module installed") @pytest.mark.skipif(not utils.passlib_libpass_supports_bcrypt()[0], reason="bcrypt module incompatible with passlib(libpass) module") def test_htpasswd_bcrypt_2a(self) -> None: From 5214287875635ecb5031c678fb5169d8d570a2b7 Mon Sep 17 00:00:00 2001 From: Peter Bieringer Date: Thu, 8 Jan 2026 05:39:44 +0100 Subject: [PATCH 260/290] replace too simple autodetection with hash length by proper regular expression --- radicale/auth/htpasswd.py | 23 +++++++---------------- 1 file changed, 7 insertions(+), 16 deletions(-) diff --git a/radicale/auth/htpasswd.py b/radicale/auth/htpasswd.py index 3b473f28..fb7def4d 100644 --- a/radicale/auth/htpasswd.py +++ b/radicale/auth/htpasswd.py @@ -191,37 +191,28 @@ class Auth(auth.BaseAuth): return ("ARGON2", argon2.verify(password, hash_value.strip())) def _md5apr1(self, hash_value: str, password: str) -> tuple[str, bool]: - if self._encryption == "autodetect" and len(hash_value) != 37: - return self._plain_fallback("MD5-APR1", hash_value, password) - else: - return ("MD5-APR1", apr_md5_crypt.verify(password, hash_value.strip())) + return ("MD5-APR1", apr_md5_crypt.verify(password, hash_value.strip())) def _sha256(self, hash_value: str, password: str) -> tuple[str, bool]: - if self._encryption == "autodetect" and len(hash_value) != 63: - return self._plain_fallback("SHA-256", hash_value, password) - else: - return ("SHA-256", sha256_crypt.verify(password, hash_value.strip())) + return ("SHA-256", sha256_crypt.verify(password, hash_value.strip())) def _sha512(self, hash_value: str, password: str) -> tuple[str, bool]: - if self._encryption == "autodetect" and len(hash_value) != 106: - return self._plain_fallback("SHA-512", hash_value, password) - else: - return ("SHA-512", sha512_crypt.verify(password, hash_value.strip())) + return ("SHA-512", sha512_crypt.verify(password, hash_value.strip())) def _autodetect(self, hash_value: str, password: str) -> tuple[str, bool]: - if hash_value.startswith("$apr1$", 0, 6): + if re.match(r"^\$apr1\$[A-Za-z0-9/.]{8}\$[A-Za-z0-9/.]{22}", hash_value): # MD5-APR1 return self._md5apr1(hash_value, password) - elif re.match(r"^\$2(a|b|x|y)?\$", hash_value): + elif re.match(r"^\$2(a|b|x|y)?\$[0-9]{2}\$[A-Za-z0-9/.]{53}", hash_value): # BCRYPT return self._verify_bcrypt(hash_value, password) elif re.match(r"^\$argon2(i|d|id)\$", hash_value): # ARGON2 return self._verify_argon2(hash_value, password) - elif hash_value.startswith("$5$", 0, 3): + elif re.match(r"^\$5\$(rounds=[0-9]+\$)?[A-Za-z0-9/.]{16}\$[A-Za-z0-9/.]{42}", hash_value): # SHA-256 return self._sha256(hash_value, password) - elif hash_value.startswith("$6$", 0, 3): + elif re.match(r"^\$6\$(rounds=[0-9]+\$)?[A-Za-z0-9/.]{16}\$[A-Za-z0-9/.]{85}", hash_value): # SHA-512 return self._sha512(hash_value, password) else: From 382d0a10862fd96c8a9adb6d859f64a98dd7fe00 Mon Sep 17 00:00:00 2001 From: Peter Bieringer Date: Thu, 8 Jan 2026 05:41:14 +0100 Subject: [PATCH 261/290] extend changelog --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index cf2d9465..a6e33ce4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,6 +11,7 @@ * Fix: hook for server-side e-mail notification * Change: dependency PyPI/passlib (stale since 2020) replaced with PyPI/libpass >= 1.9.3 * Extend: add a check whether bcrypt version is compatible with passlib(libpass) version (requires "packaging") +* Improve: autodetection of hashes in htpasswd (SHA256/SHA512 "rounds" are now supported) ## 3.5.10 * Improve: logging of broken calendar items during PUT From ffb2fb5e78c8b389597803ccb75f745c40593db7 Mon Sep 17 00:00:00 2001 From: Peter Bieringer Date: Sat, 10 Jan 2026 07:48:43 +0100 Subject: [PATCH 262/290] cosmetics --- radicale/auth/htpasswd.py | 2 +- radicale/utils.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/radicale/auth/htpasswd.py b/radicale/auth/htpasswd.py index fb7def4d..17c93553 100644 --- a/radicale/auth/htpasswd.py +++ b/radicale/auth/htpasswd.py @@ -43,7 +43,7 @@ out-of-the-box: - SHA256 (htpasswd -2 ...) - SHA512 (htpasswd -5 ...) -When bcrypt is installed (bcrypt >= 5.0.0 requires passlib/libpass >= 1.9.3): +When bcrypt is installed (bcrypt >= 5.0.0 requires passlib(libpass) >= 1.9.3): - BCRYPT (htpasswd -B ...) -- Requires htpasswd 2.4.x When argon2 is installed: diff --git a/radicale/utils.py b/radicale/utils.py index 139c6d69..54b80913 100644 --- a/radicale/utils.py +++ b/radicale/utils.py @@ -106,7 +106,7 @@ def vobject_supports_vcard4() -> bool: def passlib_libpass_supports_bcrypt() -> Tuple[bool, str]: - """Check if passlib/libpass version supports bcrypt version.""" + """Check if passlib(libpass) version supports bcrypt version.""" info = "" try: version_bcrypt = package_version("bcrypt") From 863d2e14d01e92046c6ef1eec26eac14511f36dc Mon Sep 17 00:00:00 2001 From: Peter Bieringer Date: Sat, 10 Jan 2026 07:48:55 +0100 Subject: [PATCH 263/290] change loglevel --- radicale/auth/htpasswd.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/radicale/auth/htpasswd.py b/radicale/auth/htpasswd.py index 17c93553..dd27fdec 100644 --- a/radicale/auth/htpasswd.py +++ b/radicale/auth/htpasswd.py @@ -123,7 +123,7 @@ class Auth(auth.BaseAuth): [bcrypt_usable, info] = utils.passlib_libpass_supports_bcrypt() if bcrypt_usable: self._has_bcrypt = True - logger.debug(info) + logger.info(info) else: logger.warning(info) if self._encryption == "autodetect": From 22a71c4290462521124e5397043f332263a2e4fb Mon Sep 17 00:00:00 2001 From: Peter Bieringer Date: Sat, 10 Jan 2026 07:52:03 +0100 Subject: [PATCH 264/290] Release 3.6.0 --- CHANGELOG.md | 2 +- pyproject.toml | 2 +- setup.py.legacy | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index a6e33ce4..ef280a36 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,6 @@ # Changelog -## 3.6.0.dev +## 3.6.0 * Extend: logwatch script * Extend: [logging] bad_put_request_content: log checksum and hexdump of request on debug level diff --git a/pyproject.toml b/pyproject.toml index fbbc96a1..3c0b4d1a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -3,7 +3,7 @@ name = "Radicale" # When the version is updated, a new section in the CHANGELOG.md file must be # added too. readme = "README.md" -version = "3.6.0.dev" +version = "3.6.0" authors = [{name = "Guillaume Ayoub", email = "guillaume.ayoub@kozea.fr"}, {name = "Unrud", email = "unrud@outlook.com"}, {name = "Peter Bieringer", email = "pb@bieringer.de"}] license = {text = "GNU GPL v3"} description = "CalDAV and CardDAV Server" diff --git a/setup.py.legacy b/setup.py.legacy index 91c42ed8..8a74a659 100644 --- a/setup.py.legacy +++ b/setup.py.legacy @@ -20,7 +20,7 @@ from setuptools import find_packages, setup # When the version is updated, a new section in the CHANGELOG.md file must be # added too. -VERSION = "3.6.0.dev" +VERSION = "3.6.0" with open("README.md", encoding="utf-8") as f: long_description = f.read() From f4f03538a0793d456cadfe07649db2d3a8591925 Mon Sep 17 00:00:00 2001 From: kalsi-avneet <4151485+kalsi-avneet@users.noreply.github.com> Date: Sun, 11 Jan 2026 18:38:35 +0000 Subject: [PATCH 265/290] Docker compose file - change tag to "stable" --- compose.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/compose.yaml b/compose.yaml index 2130ee78..0e4abb8d 100644 --- a/compose.yaml +++ b/compose.yaml @@ -1,7 +1,7 @@ name: Radicale services: radicale: - image: ghcr.io/kozea/radicale:3.5.10 + image: ghcr.io/kozea/radicale:stable ports: - 5232:5232 volumes: From d55375623843cd573ec030e8d150b63dd659ef1d Mon Sep 17 00:00:00 2001 From: Peter Bieringer Date: Mon, 19 Jan 2026 07:29:09 +0100 Subject: [PATCH 266/290] prepare 3.6.1.dev --- pyproject.toml | 2 +- setup.py.legacy | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 3c0b4d1a..cca107c0 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -3,7 +3,7 @@ name = "Radicale" # When the version is updated, a new section in the CHANGELOG.md file must be # added too. readme = "README.md" -version = "3.6.0" +version = "3.6.1.dev" authors = [{name = "Guillaume Ayoub", email = "guillaume.ayoub@kozea.fr"}, {name = "Unrud", email = "unrud@outlook.com"}, {name = "Peter Bieringer", email = "pb@bieringer.de"}] license = {text = "GNU GPL v3"} description = "CalDAV and CardDAV Server" diff --git a/setup.py.legacy b/setup.py.legacy index 8a74a659..50830dad 100644 --- a/setup.py.legacy +++ b/setup.py.legacy @@ -20,7 +20,7 @@ from setuptools import find_packages, setup # When the version is updated, a new section in the CHANGELOG.md file must be # added too. -VERSION = "3.6.0" +VERSION = "3.6.1.dev" with open("README.md", encoding="utf-8") as f: long_description = f.read() From 4fb16727f02c4b37b946e0d9ff00795195cd5081 Mon Sep 17 00:00:00 2001 From: Peter Bieringer Date: Mon, 19 Jan 2026 07:40:36 +0100 Subject: [PATCH 267/290] add note about dependency adjustments --- pyproject.toml | 1 + setup.py.legacy | 1 + 2 files changed, 2 insertions(+) diff --git a/pyproject.toml b/pyproject.toml index cca107c0..80e7fe37 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -28,6 +28,7 @@ classifiers = [ ] urls = {Homepage = "https://radicale.org/"} requires-python = ">=3.9.0" +# Hint: if bcyrpt < 5.0.0 is used, passlib(libpass) dependency can be downgraded/reverted by: sed -i 's|libpass[^"]*|passlib|' pyproject.toml dependencies = [ "defusedxml", "libpass>=1.9.3", diff --git a/setup.py.legacy b/setup.py.legacy index 50830dad..27957903 100644 --- a/setup.py.legacy +++ b/setup.py.legacy @@ -36,6 +36,7 @@ web_files = ["web/internal_data/css/icon.png", "web/internal_data/fn.js", "web/internal_data/index.html"] +# Hint: if bcyrpt < 5.0.0 is used, passlib(libpass) dependency can be downgraded/reverted by: sed -i 's|libpass[^"]*|passlib|' setup.py.legacy install_requires = ["defusedxml", "libpass>=1.9.3", "vobject>=0.9.6", "pika>=1.1.0", "requests", From 9a2bbb589cf79e5757e4d6185dbe53437349416e Mon Sep 17 00:00:00 2001 From: laurisvr Date: Wed, 21 Jan 2026 16:58:25 +0100 Subject: [PATCH 268/290] Fix MOVE failing with URL-encoded Destination header Destination header wasn't being decoded before urlparse(), and relative URLs (empty netloc) failed the host comparison. Ran into this with email usernames where @ gets encoded as %40." --- radicale/app/move.py | 27 +++++++++++++++++---------- 1 file changed, 17 insertions(+), 10 deletions(-) diff --git a/radicale/app/move.py b/radicale/app/move.py index b65f6600..168619e3 100644 --- a/radicale/app/move.py +++ b/radicale/app/move.py @@ -22,7 +22,7 @@ import errno import posixpath import re from http import client -from urllib.parse import urlparse +from urllib.parse import unquote, urlparse from radicale import httputils, pathutils, storage, types from radicale.app.base import Access, ApplicationBase @@ -51,15 +51,22 @@ class ApplicationPartMove(ApplicationBase): path: str, user: str, remote_host: str, remote_useragent: str) -> types.WSGIResponse: """Manage MOVE request.""" raw_dest = environ.get("HTTP_DESTINATION", "") - to_url = urlparse(raw_dest) - to_netloc_with_port = to_url.netloc - if to_url.port is None: - to_netloc_with_port += (":443" if to_url.scheme == "https" - else ":80") - if to_netloc_with_port != get_server_netloc(environ, force_port=True): - logger.info("Unsupported destination address: %r", raw_dest) - # Remote destination server, not supported - return httputils.REMOTE_DESTINATION + + # Decode URL-encoded characters (e.g. %40 -> @) before parsing + raw_dest_decoded = unquote(raw_dest) + to_url = urlparse(raw_dest_decoded) + + # Only check netloc for absolute URLs + if to_url.netloc: + to_netloc_with_port = to_url.netloc + if to_url.port is None: + to_netloc_with_port += (":443" if to_url.scheme == "https" + else ":80") + if to_netloc_with_port != get_server_netloc(environ, force_port=True): + logger.info("Unsupported destination address: %r", raw_dest) + # Remote destination server, not supported + return httputils.REMOTE_DESTINATION + access = Access(self._rights, user, path) if not access.check("w"): return httputils.NOT_ALLOWED From a75e3ebdec1cf0627e9a5254b307d9a0d4a1ebe2 Mon Sep 17 00:00:00 2001 From: Peter Bieringer Date: Thu, 22 Jan 2026 06:27:04 +0100 Subject: [PATCH 269/290] Testcase for PR#1968 (MOVE with URL-encoded Destination header) --- radicale/tests/test_base.py | 30 +++++++++++++++++++++++++++++- 1 file changed, 29 insertions(+), 1 deletion(-) diff --git a/radicale/tests/test_base.py b/radicale/tests/test_base.py index 8d8e0def..0acc22e4 100644 --- a/radicale/tests/test_base.py +++ b/radicale/tests/test_base.py @@ -1,7 +1,7 @@ # This file is part of Radicale - CalDAV and CardDAV server # Copyright © 2012-2017 Guillaume Ayoub # Copyright © 2017-2022 Unrud -# Copyright © 2024-2025 Peter Bieringer +# Copyright © 2024-2026 Peter Bieringer # # This library is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -28,6 +28,7 @@ from typing import Any, Callable, ClassVar, Iterable, List, Optional, Tuple import defusedxml.ElementTree as DefusedET import pytest +import urllib import vobject from radicale import storage, utils, xmlutils @@ -568,6 +569,33 @@ permissions: RrWw""") self.get(path1, check=404) self.get(path2) + def test_move_between_collections_with_at_native(self) -> None: + """Move a item.""" + self.mkcalendar("/calendar1@domain.ics/") + self.mkcalendar("/calendar2@domain.ics/") + event = get_file_content("event1.ics") + path1 = "/calendar1@domain.ics/event1.ics" + path2 = "/calendar2@domain.ics/event2.ics" + self.put(path1, event) + self.request("MOVE", path1, check=201, + HTTP_DESTINATION="http://127.0.0.1/"+path2) + self.get(path1, check=404) + self.get(path2) + + def test_move_between_collections_with_at_encoded(self) -> None: + """Move a item.""" + self.mkcalendar("/calendar1@domain.ics/") + self.mkcalendar("/calendar2@domain.ics/") + event = get_file_content("event1.ics") + path1 = "/calendar1@domain.ics/event1.ics" + path2 = "/calendar2@domain.ics/event2.ics" + path2_encoded = urllib.parse.quote(path2) + self.put(path1, event) + self.request("MOVE", path1, check=201, + HTTP_DESTINATION="http://127.0.0.1/"+path2_encoded) + self.get(path1, check=404) + self.get(path2) + def test_move_between_collections_duplicate_uid(self) -> None: """Move a item to a collection which already contains the UID.""" self.mkcalendar("/calendar1.ics/") From e409d628db5e1d8bc88f5bd56d55275b06f69c82 Mon Sep 17 00:00:00 2001 From: Peter Bieringer Date: Thu, 22 Jan 2026 06:31:41 +0100 Subject: [PATCH 270/290] changelog for https://github.com/Kozea/Radicale/pull/1968 --- CHANGELOG.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index ef280a36..15706e57 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,9 @@ # Changelog +## 3.6.1.dev + +* Fix: MOVE failing with URL-encoded destination header + ## 3.6.0 * Extend: logwatch script From 02a6cd55e3a84f0afdb44bfbe876dd9041401644 Mon Sep 17 00:00:00 2001 From: Peter Bieringer Date: Thu, 22 Jan 2026 06:35:12 +0100 Subject: [PATCH 271/290] fix for lint --- radicale/tests/test_base.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/radicale/tests/test_base.py b/radicale/tests/test_base.py index 0acc22e4..ab7954bf 100644 --- a/radicale/tests/test_base.py +++ b/radicale/tests/test_base.py @@ -24,11 +24,11 @@ Radicale tests with simple requests. import logging import os import posixpath +import urllib from typing import Any, Callable, ClassVar, Iterable, List, Optional, Tuple import defusedxml.ElementTree as DefusedET import pytest -import urllib import vobject from radicale import storage, utils, xmlutils From ae0882cc13801ced08f3bce6479477d4659414b9 Mon Sep 17 00:00:00 2001 From: Peter Bieringer Date: Thu, 22 Jan 2026 06:52:39 +0100 Subject: [PATCH 272/290] optimize test sequence resource-wise --- .github/workflows/test.yml | 25 ++++++++++++++++++++++--- 1 file changed, 22 insertions(+), 3 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 03dcd687..aca32dfa 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -2,11 +2,30 @@ name: Test on: [push, pull_request] jobs: - test: + + test-python-latest: + needs: lint strategy: matrix: os: [ubuntu-latest, macos-latest, windows-latest] - python-version: ['3.9', '3.10', '3.11', '3.12', '3.13', '3.14', 'pypy-3.9', 'pypy-3.10', 'pypy-3.11'] + python-version: ['3.14'] + runs-on: ${{ matrix.os }} + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: ${{ matrix.python-version }} + - name: Install Test dependencies + run: pip install tox + - name: Test with latest Python + run: tox -c pyproject.toml -e py + + test: + needs: test-python-latest + strategy: + matrix: + os: [ubuntu-latest, macos-latest, windows-latest] + python-version: ['3.9', '3.10', '3.11', '3.12', '3.13', pypy-3.9', 'pypy-3.10', 'pypy-3.11'] exclude: - os: windows-latest python-version: 'pypy-3.9' @@ -22,7 +41,7 @@ jobs: python-version: ${{ matrix.python-version }} - name: Install Test dependencies run: pip install tox - - name: Test + - name: Test with older Python run: tox -c pyproject.toml -e py coveralls-test: From 2ac3d269c6384c9b0e07516c8b5178117b94c8ff Mon Sep 17 00:00:00 2001 From: Peter Bieringer Date: Thu, 22 Jan 2026 06:56:47 +0100 Subject: [PATCH 273/290] use test on ubuntu as gatekeeper --- .github/workflows/test.yml | 23 ++++++++++++++++++++--- 1 file changed, 20 insertions(+), 3 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index aca32dfa..1c36fe69 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -3,11 +3,11 @@ on: [push, pull_request] jobs: - test-python-latest: + test-ubuntu-latest-python-latest: needs: lint strategy: matrix: - os: [ubuntu-latest, macos-latest, windows-latest] + os: [ubuntu-latest] python-version: ['3.14'] runs-on: ${{ matrix.os }} steps: @@ -17,7 +17,24 @@ jobs: python-version: ${{ matrix.python-version }} - name: Install Test dependencies run: pip install tox - - name: Test with latest Python + - name: Test with latest Python on Ubuntu + run: tox -c pyproject.toml -e py + + test-python-latest: + needs: test-ubuntu-latest-python-latest: + strategy: + matrix: + os: [macos-latest, windows-latest] + python-version: ['3.14'] + runs-on: ${{ matrix.os }} + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: ${{ matrix.python-version }} + - name: Install Test dependencies + run: pip install tox + - name: Test with latest Python on other OS run: tox -c pyproject.toml -e py test: From 34d7895bd43a812cc120136fa0331268b8534cdd Mon Sep 17 00:00:00 2001 From: Peter Bieringer Date: Thu, 22 Jan 2026 06:57:25 +0100 Subject: [PATCH 274/290] fix typo --- .github/workflows/test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 1c36fe69..a7f1660a 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -21,7 +21,7 @@ jobs: run: tox -c pyproject.toml -e py test-python-latest: - needs: test-ubuntu-latest-python-latest: + needs: test-ubuntu-latest-python-latest strategy: matrix: os: [macos-latest, windows-latest] From c71f2da6fcbe80fcbf63a5b3d85a1b1f809991a2 Mon Sep 17 00:00:00 2001 From: Peter Bieringer Date: Thu, 22 Jan 2026 07:19:56 +0100 Subject: [PATCH 275/290] fix typo --- .github/workflows/test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index a7f1660a..20fc5c05 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -42,7 +42,7 @@ jobs: strategy: matrix: os: [ubuntu-latest, macos-latest, windows-latest] - python-version: ['3.9', '3.10', '3.11', '3.12', '3.13', pypy-3.9', 'pypy-3.10', 'pypy-3.11'] + python-version: ['3.9', '3.10', '3.11', '3.12', '3.13', 'pypy-3.9', 'pypy-3.10', 'pypy-3.11'] exclude: - os: windows-latest python-version: 'pypy-3.9' From e3ceb303175ffcc4340747a0bad5ec074d270223 Mon Sep 17 00:00:00 2001 From: Peter Bieringer Date: Thu, 22 Jan 2026 08:45:30 +0100 Subject: [PATCH 276/290] test also python-oldest before matrix --- .github/workflows/test.yml | 46 +++++++++++++++++++++++++++++++++----- 1 file changed, 40 insertions(+), 6 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 20fc5c05..cd7bae74 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -3,7 +3,7 @@ on: [push, pull_request] jobs: - test-ubuntu-latest-python-latest: + test-ubuntu-python-newest: needs: lint strategy: matrix: @@ -20,8 +20,25 @@ jobs: - name: Test with latest Python on Ubuntu run: tox -c pyproject.toml -e py - test-python-latest: - needs: test-ubuntu-latest-python-latest + test-ubuntu-python-oldest: + needs: [lint, test-ubuntu-latest-python-newest] + strategy: + matrix: + os: [ubuntu-latest] + python-version: ['3.9'] + runs-on: ${{ matrix.os }} + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: ${{ matrix.python-version }} + - name: Install Test dependencies + run: pip install tox + - name: Test with oldest Python on Ubuntu + run: tox -c pyproject.toml -e py + + test-otheros-python-newest: + needs: [lint, test-ubuntu-latest-python-newest] strategy: matrix: os: [macos-latest, windows-latest] @@ -37,12 +54,29 @@ jobs: - name: Test with latest Python on other OS run: tox -c pyproject.toml -e py - test: - needs: test-python-latest + test-otheros-python-oldest: + needs: [lint, test-ubuntu-latest-python-oldest] + strategy: + matrix: + os: [macos-latest, windows-latest] + python-version: ['3.9'] + runs-on: ${{ matrix.os }} + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: ${{ matrix.python-version }} + - name: Install Test dependencies + run: pip install tox + - name: Test with oldest Python on other OS + run: tox -c pyproject.toml -e py + + test-python-versions: + needs: [lint, test-otheros-python-oldest, test-otheros-python-newest, test-ubuntu-python-oldest, test-ubuntu-python-newest] strategy: matrix: os: [ubuntu-latest, macos-latest, windows-latest] - python-version: ['3.9', '3.10', '3.11', '3.12', '3.13', 'pypy-3.9', 'pypy-3.10', 'pypy-3.11'] + python-version: ['3.10', '3.11', '3.12', '3.13', 'pypy-3.9', 'pypy-3.10', 'pypy-3.11'] exclude: - os: windows-latest python-version: 'pypy-3.9' From 0f667e137614857ecec930da605e2e6de9da1f4d Mon Sep 17 00:00:00 2001 From: Peter Bieringer Date: Thu, 22 Jan 2026 08:46:57 +0100 Subject: [PATCH 277/290] bugfix --- .github/workflows/test.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index cd7bae74..1055e774 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -21,7 +21,7 @@ jobs: run: tox -c pyproject.toml -e py test-ubuntu-python-oldest: - needs: [lint, test-ubuntu-latest-python-newest] + needs: [lint, test-ubuntu-python-newest] strategy: matrix: os: [ubuntu-latest] @@ -38,7 +38,7 @@ jobs: run: tox -c pyproject.toml -e py test-otheros-python-newest: - needs: [lint, test-ubuntu-latest-python-newest] + needs: [lint, test-ubuntu-python-newest] strategy: matrix: os: [macos-latest, windows-latest] @@ -55,7 +55,7 @@ jobs: run: tox -c pyproject.toml -e py test-otheros-python-oldest: - needs: [lint, test-ubuntu-latest-python-oldest] + needs: [lint, test-ubuntu-python-oldest] strategy: matrix: os: [macos-latest, windows-latest] From c6b97569f6170297da8b7c4c9d12e225f110b621 Mon Sep 17 00:00:00 2001 From: Peter Bieringer Date: Thu, 22 Jan 2026 17:49:45 +0100 Subject: [PATCH 278/290] fix dependency --- .github/workflows/test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 1055e774..f16e26f9 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -55,7 +55,7 @@ jobs: run: tox -c pyproject.toml -e py test-otheros-python-oldest: - needs: [lint, test-ubuntu-python-oldest] + needs: [lint, test-ubuntu-python-oldest, test-otheros-python-newest] strategy: matrix: os: [macos-latest, windows-latest] From b3ccd39d115c7f58ca61f43c48f683d5e42b3ee6 Mon Sep 17 00:00:00 2001 From: Peter Bieringer Date: Thu, 22 Jan 2026 21:33:29 +0100 Subject: [PATCH 279/290] update copyright --- radicale/item/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/radicale/item/__init__.py b/radicale/item/__init__.py index 1c9dcadb..d303ee99 100644 --- a/radicale/item/__init__.py +++ b/radicale/item/__init__.py @@ -4,7 +4,7 @@ # Copyright © 2014 Jean-Marc Martins # Copyright © 2008-2017 Guillaume Ayoub # Copyright © 2017-2022 Unrud -# Copyright © 2024-2025 Peter Bieringer +# Copyright © 2024-2026 Peter Bieringer # # This library is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by From a143c5d7651ba14744f6982a48f3a5e1561e4d43 Mon Sep 17 00:00:00 2001 From: Peter Bieringer Date: Thu, 22 Jan 2026 21:33:50 +0100 Subject: [PATCH 280/290] Workaround delete all empty lines to avoid vobject parsing errors --- radicale/item/__init__.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/radicale/item/__init__.py b/radicale/item/__init__.py index d303ee99..48c0bdaa 100644 --- a/radicale/item/__init__.py +++ b/radicale/item/__init__.py @@ -56,6 +56,8 @@ def read_components(s: str) -> List[vobject.base.Component]: # * 0x0A Line Feed # * 0x0D Carriage Return s = re.sub(r'[\x00-\x08\x0B\x0C\x0E-\x1F]', '', s) + # Workaround delete all empty lines to avoid vobject parsing errors + s = re.sub(r'(?m)^[ \t]*\r?\n', '', s) return list(vobject.readComponents(s, allowQP=True)) From fa38e25468904dd5c44e79a62416baf3fde5b070 Mon Sep 17 00:00:00 2001 From: Peter Bieringer Date: Thu, 22 Jan 2026 21:34:23 +0100 Subject: [PATCH 281/290] add files for testcase --- radicale/tests/static/event_issue1970_ok.ics | 38 ++++++++++++++++++ .../tests/static/event_issue1970_problem.ics | 39 +++++++++++++++++++ 2 files changed, 77 insertions(+) create mode 100644 radicale/tests/static/event_issue1970_ok.ics create mode 100644 radicale/tests/static/event_issue1970_problem.ics diff --git a/radicale/tests/static/event_issue1970_ok.ics b/radicale/tests/static/event_issue1970_ok.ics new file mode 100644 index 00000000..89725608 --- /dev/null +++ b/radicale/tests/static/event_issue1970_ok.ics @@ -0,0 +1,38 @@ +BEGIN:VCALENDAR +PRODID:-//Mozilla.org/NONSGML Mozilla Calendar V1.1//EN +VERSION:2.0 +BEGIN:VTIMEZONE +TZID:Europe/Paris +X-LIC-LOCATION:Europe/Paris +BEGIN:DAYLIGHT +TZOFFSETFROM:+0100 +TZOFFSETTO:+0200 +TZNAME:CEST +DTSTART:19700329T020000 +RRULE:FREQ=YEARLY;BYDAY=-1SU;BYMONTH=3 +END:DAYLIGHT +BEGIN:STANDARD +TZOFFSETFROM:+0200 +TZOFFSETTO:+0100 +TZNAME:CET +DTSTART:19701025T030000 +RRULE:FREQ=YEARLY;BYDAY=-1SU;BYMONTH=10 +END:STANDARD +END:VTIMEZONE +BEGIN:VEVENT +CREATED:20130902T150157Z +LAST-MODIFIED:20130902T150158Z +DTSTAMP:20130902T150158Z +UID:event1 +SUMMARY:Event +CATEGORIES:some_category1,another_category2 +DESCRIPTION:Line1 + Line2 + Line3 +ORGANIZER:mailto:unclesam@example.com +ATTENDEE;ROLE=REQ-PARTICIPANT;PARTSTAT=TENTATIVE;CN=Jane Doe:MAILTO:janedoe@example.com +ATTENDEE;ROLE=REQ-PARTICIPANT;DELEGATED-FROM="MAILTO:bob@host.com";PARTSTAT=ACCEPTED;CN=John Doe:MAILTO:johndoe@example.com +DTSTART;TZID=Europe/Paris:20130901T180000 +DTEND;TZID=Europe/Paris:20130901T190000 +END:VEVENT +END:VCALENDAR diff --git a/radicale/tests/static/event_issue1970_problem.ics b/radicale/tests/static/event_issue1970_problem.ics new file mode 100644 index 00000000..a1b75e6e --- /dev/null +++ b/radicale/tests/static/event_issue1970_problem.ics @@ -0,0 +1,39 @@ +BEGIN:VCALENDAR +PRODID:-//Mozilla.org/NONSGML Mozilla Calendar V1.1//EN +VERSION:2.0 +BEGIN:VTIMEZONE +TZID:Europe/Paris +X-LIC-LOCATION:Europe/Paris +BEGIN:DAYLIGHT +TZOFFSETFROM:+0100 +TZOFFSETTO:+0200 +TZNAME:CEST +DTSTART:19700329T020000 +RRULE:FREQ=YEARLY;BYDAY=-1SU;BYMONTH=3 +END:DAYLIGHT +BEGIN:STANDARD +TZOFFSETFROM:+0200 +TZOFFSETTO:+0100 +TZNAME:CET +DTSTART:19701025T030000 +RRULE:FREQ=YEARLY;BYDAY=-1SU;BYMONTH=10 +END:STANDARD +END:VTIMEZONE +BEGIN:VEVENT +CREATED:20130902T150157Z +LAST-MODIFIED:20130902T150158Z +DTSTAMP:20130902T150158Z +UID:event1 +SUMMARY:Event having description with empty line +CATEGORIES:some_category1,another_category2 +DESCRIPTION:Line1 + Line2 + + Line4 +ORGANIZER:mailto:unclesam@example.com +ATTENDEE;ROLE=REQ-PARTICIPANT;PARTSTAT=TENTATIVE;CN=Jane Doe:MAILTO:janedoe@example.com +ATTENDEE;ROLE=REQ-PARTICIPANT;DELEGATED-FROM="MAILTO:bob@host.com";PARTSTAT=ACCEPTED;CN=John Doe:MAILTO:johndoe@example.com +DTSTART;TZID=Europe/Paris:20130901T180000 +DTEND;TZID=Europe/Paris:20130901T190000 +END:VEVENT +END:VCALENDAR From 2351f49ae401c2e8f878cd007a5f5e894ef9f9af Mon Sep 17 00:00:00 2001 From: Peter Bieringer Date: Thu, 22 Jan 2026 21:34:49 +0100 Subject: [PATCH 282/290] add testcases --- radicale/tests/test_base.py | 28 ++++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/radicale/tests/test_base.py b/radicale/tests/test_base.py index ab7954bf..f50cf0be 100644 --- a/radicale/tests/test_base.py +++ b/radicale/tests/test_base.py @@ -144,6 +144,34 @@ permissions: RrWw""") assert "Event" in answer assert "UID:event" in answer + def test_add_event_with_desc_ok(self) -> None: + """Add an event.""" + self.mkcalendar("/calendar.ics/") + event = get_file_content("event_issue1970_ok.ics") + path = "/calendar.ics/event_issue1970_ok.ics" + self.put(path, event) + _, headers, answer = self.request("GET", path, check=200) + assert "ETag" in headers + assert headers["Content-Type"] == "text/calendar; charset=utf-8" + assert "DESCRIPTION" in answer + assert "VEVENT" in answer + assert "Event" in answer + assert "UID:event" in answer + + def test_add_event_with_desc_problem(self) -> None: + """Add an event.""" + self.mkcalendar("/calendar.ics/") + event = get_file_content("event_issue1970_problem.ics") + path = "/calendar.ics/event_issue1970_problem.ics" + self.put(path, event) + _, headers, answer = self.request("GET", path, check=200) + assert "ETag" in headers + assert headers["Content-Type"] == "text/calendar; charset=utf-8" + assert "DESCRIPTION" in answer + assert "VEVENT" in answer + assert "Event" in answer + assert "UID:event" in answer + def test_add_event_exceed_size(self) -> None: """Add an event which is exceeding max-resource-size.""" self.configure({"server": {"max_resource_size": 20}}) From 3d5c74ff916d4dd7385ae7ca59786f1a0b0ba2ce Mon Sep 17 00:00:00 2001 From: Peter Bieringer Date: Thu, 22 Jan 2026 21:37:19 +0100 Subject: [PATCH 283/290] changelog --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 15706e57..a1c558da 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,6 +3,7 @@ ## 3.6.1.dev * Fix: MOVE failing with URL-encoded destination header +* Workaround: remove empty lines in item to avoid reject by vobject parser ## 3.6.0 From 8613f9d6f3df6d8a27e7e828eab3d9968674e861 Mon Sep 17 00:00:00 2001 From: Peter Bieringer Date: Thu, 22 Jan 2026 21:39:40 +0100 Subject: [PATCH 284/290] cosmetics --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index a1c558da..763676ce 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,7 +3,7 @@ ## 3.6.1.dev * Fix: MOVE failing with URL-encoded destination header -* Workaround: remove empty lines in item to avoid reject by vobject parser +* Improve: add workaround to remove empty lines in item to avoid reject by vobject parser ## 3.6.0 From 0388051046e1f4bfa96750df4f7c5d661ca08980 Mon Sep 17 00:00:00 2001 From: Tobias Brox Date: Fri, 23 Jan 2026 16:44:55 +0100 Subject: [PATCH 285/290] Fix unclosed scandir iterator in path_to_filesystem Use os.scandir() as a context manager to ensure the iterator is properly closed. This fixes ResourceWarning: unclosed scandir iterator that occurs when the iterator is garbage collected without being closed. Fixes #1972 Co-Authored-By: Claude Opus 4.5 --- radicale/pathutils.py | 7 +-- radicale/tests/test_pathutils.py | 91 ++++++++++++++++++++++++++++++++ 2 files changed, 95 insertions(+), 3 deletions(-) create mode 100644 radicale/tests/test_pathutils.py diff --git a/radicale/pathutils.py b/radicale/pathutils.py index e4e65928..3193e4a2 100644 --- a/radicale/pathutils.py +++ b/radicale/pathutils.py @@ -286,9 +286,10 @@ def path_to_filesystem(root: str, sane_path: str) -> str: safe_path = os.path.join(safe_path, part) # Check for conflicting files (e.g. case-insensitive file systems # or short names on Windows file systems) - if (os.path.lexists(safe_path) and - part not in (e.name for e in os.scandir(safe_path_parent))): - raise CollidingPathError(part) + if os.path.lexists(safe_path): + with os.scandir(safe_path_parent) as entries: + if part not in (e.name for e in entries): + raise CollidingPathError(part) return safe_path diff --git a/radicale/tests/test_pathutils.py b/radicale/tests/test_pathutils.py new file mode 100644 index 00000000..ebe92de8 --- /dev/null +++ b/radicale/tests/test_pathutils.py @@ -0,0 +1,91 @@ +# This file is part of Radicale - CalDAV and CardDAV server +# Copyright © 2025 Tobias Brox +# +# This library is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This library is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with Radicale. If not, see . + +""" +Tests for pathutils module. + +""" + +import gc +import os +import tempfile + +import pytest + +from radicale import pathutils + + +class TestPathToFilesystem: + """Tests for path_to_filesystem function.""" + + @pytest.mark.filterwarnings("error::ResourceWarning") + @pytest.mark.filterwarnings("error::pytest.PytestUnraisableExceptionWarning") + def test_scandir_iterator_closed(self) -> None: + """Verify that os.scandir iterator is properly closed. + + This test catches ResourceWarning: unclosed scandir iterator + which occurs when os.scandir() is used without a context manager. + See: https://github.com/Kozea/Radicale/issues/1972 + + The ResourceWarning is emitted during garbage collection when an + unclosed scandir iterator is finalized. We use pytest.mark.filterwarnings + to convert both ResourceWarning and PytestUnraisableExceptionWarning + to errors. + """ + with tempfile.TemporaryDirectory() as tmpdir: + # Create a subdirectory so path_to_filesystem has something + # to scan (the scandir check is for case-insensitive filesystems) + subdir = os.path.join(tmpdir, "testdir") + os.makedirs(subdir) + + # Call path_to_filesystem - if scandir iterator is not closed, + # a ResourceWarning will be emitted during garbage collection + result = pathutils.path_to_filesystem(tmpdir, "testdir") + assert result == subdir + + # Force garbage collection to trigger any ResourceWarning + # from unclosed iterators + gc.collect() + + def test_path_to_filesystem_basic(self) -> None: + """Test basic path_to_filesystem functionality.""" + with tempfile.TemporaryDirectory() as tmpdir: + # Test empty path + result = pathutils.path_to_filesystem(tmpdir, "") + assert result == tmpdir + + # Test single component + subdir = os.path.join(tmpdir, "test") + os.makedirs(subdir) + result = pathutils.path_to_filesystem(tmpdir, "test") + assert result == subdir + + # Test nested path + nested = os.path.join(subdir, "nested") + os.makedirs(nested) + result = pathutils.path_to_filesystem(tmpdir, "test/nested") + assert result == nested + + def test_unsafe_path_raises(self) -> None: + """Test that unsafe path components raise UnsafePathError.""" + with tempfile.TemporaryDirectory() as tmpdir: + # Hidden files (starting with .) are not safe + with pytest.raises(pathutils.UnsafePathError): + pathutils.path_to_filesystem(tmpdir, ".hidden") + + # Backup files (ending with ~) are not safe + with pytest.raises(pathutils.UnsafePathError): + pathutils.path_to_filesystem(tmpdir, "backup~") From 732bcbf9dc669c453250d52ac2788424a6a02f6d Mon Sep 17 00:00:00 2001 From: schmijoe Date: Sat, 24 Jan 2026 09:57:31 +0100 Subject: [PATCH 286/290] fix lighttpd example --- DOCUMENTATION.md | 10 ++-------- 1 file changed, 2 insertions(+), 8 deletions(-) diff --git a/DOCUMENTATION.md b/DOCUMENTATION.md index 0b4c6636..55c5d5ad 100644 --- a/DOCUMENTATION.md +++ b/DOCUMENTATION.md @@ -476,17 +476,11 @@ RequestHeader set X-Forwarded-Proto "https" Example **lighttpd** configuration: ```lighttpd -server.modules += ( "mod_proxy" , "mod_setenv", "mod_rewrite" ) +server.modules += ( "mod_proxy" , "mod_setenv" ) $HTTP["url"] =~ "^/radicale/" { proxy.server = ( "" => (( "host" => "127.0.0.1", "port" => "5232" )) ) - proxy.header = ( "map-urlpath" => ( "/radicale/" => "/" )) - - setenv.add-request-header = ( - "X-Script-Name" => "/radicale", - "Script-Name" => "/radicale", - ) - url.rewrite-once = ( "^/radicale/radicale/(.*)" => "/radicale/$1" ) + setenv.add-request-header = ( "X-Script-Name" => "/radicale" ) } ``` From 894edfe4dc16d655a38ea69d7fa489d66218639a Mon Sep 17 00:00:00 2001 From: Steve Zabka Date: Sat, 24 Jan 2026 16:22:15 +0100 Subject: [PATCH 287/290] Refactor predefined user collections - Overhauled collection keys for consistency - Added Work Address Book - Added Birthday and Work calendars - Personal calendar and address book maintained --- config | 25 +++++++++++++++++++------ 1 file changed, 19 insertions(+), 6 deletions(-) diff --git a/config b/config index 2f85c4e0..33ab4d4e 100644 --- a/config +++ b/config @@ -268,18 +268,31 @@ # # json format: # -# { -# "def-addressbook": { +# predefined_collections = { +# "def-personal-addressbook": { # "D:displayname": "Personal Address Book", # "tag": "VADDRESSBOOK" # }, -# "def-calendar": { +# "def-work-addressbook": { +# "D:displayname": "Work Address Book", +# "tag": "VADDRESSBOOK" +# }, +# "def-personal-calendar": { # "C:supported-calendar-component-set": "VEVENT,VJOURNAL,VTODO", # "D:displayname": "Personal Calendar", # "tag": "VCALENDAR" -# } -# } -# +# }, +# "def-birthday-calendar": { +# "C:supported-calendar-component-set": "VEVENT", +# "D:displayname": "Birthday Calendar", +# "tag": "VCALENDAR" +# }, +# "def-work-calendar": { +# "C:supported-calendar-component-set": "VEVENT", +# "D:displayname": "Work Calendar", +# "tag": "VCALENDAR" +# }, +# } #predefined_collections = From f35c9de5862ea9b768a23756f925fa06762c2901 Mon Sep 17 00:00:00 2001 From: schmijoe Date: Sun, 25 Jan 2026 12:28:13 +0100 Subject: [PATCH 288/290] add lighttpd example config --- contrib/lighttpd/radicale.conf | 6 ++++++ 1 file changed, 6 insertions(+) create mode 100644 contrib/lighttpd/radicale.conf diff --git a/contrib/lighttpd/radicale.conf b/contrib/lighttpd/radicale.conf new file mode 100644 index 00000000..d3c58ced --- /dev/null +++ b/contrib/lighttpd/radicale.conf @@ -0,0 +1,6 @@ +server.modules += ( "mod_proxy" , "mod_setenv" ) + +$HTTP["url"] =~ "^/radicale/" { + proxy.server = ( "" => (( "host" => "127.0.0.1", "port" => "5232" )) ) + setenv.add-request-header = ( "X-Script-Name" => "/radicale" ) +} From f86498485abc55d379d82d8989493a2ae19053a5 Mon Sep 17 00:00:00 2001 From: kalsi-avneet <4151485+kalsi-avneet@users.noreply.github.com> Date: Mon, 26 Jan 2026 17:06:53 +0000 Subject: [PATCH 289/290] Docker publish workflow: Login and push to dockerhub as well --- .github/workflows/docker-publish.yml | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/.github/workflows/docker-publish.yml b/.github/workflows/docker-publish.yml index 4c50e211..85df03d3 100644 --- a/.github/workflows/docker-publish.yml +++ b/.github/workflows/docker-publish.yml @@ -8,7 +8,7 @@ on: workflow_dispatch: env: - REGISTRY: ghcr.io + GHCR_REGISTRY: ghcr.io IMAGE_NAME: ${{ github.repository }} jobs: @@ -22,18 +22,26 @@ jobs: - name: Checkout repository uses: actions/checkout@v4 - - name: Log in to the Container registry + - name: Log in to the ghcr container registry uses: docker/login-action@v3 with: - registry: ${{ env.REGISTRY }} + registry: ${{ env.GHCR_REGISTRY }} username: ${{ github.actor }} password: ${{ secrets.GITHUB_TOKEN }} + - name: Log in to the dockerhub container registry + uses: docker/login-action@v3 + with: + username: ${{ vars.DOCKERHUB_ORGNAME }} + password: ${{ secrets.DOCKERHUB_TOKEN }} + - name: Extract metadata for Docker build id: meta uses: docker/metadata-action@v5 with: - images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }} + images: | + name=${{ env.GHCR_REGISTRY }}/${{ env.IMAGE_NAME }} + name=${{ env.IMAGE_NAME }} flavor: latest=true tags: | type=semver,pattern={{version}} From f2f650f9227fef8be2306b4abd7c7142e30b9cef Mon Sep 17 00:00:00 2001 From: Guillaume REMBERT Date: Mon, 2 Feb 2026 18:00:40 +0100 Subject: [PATCH 290/290] Improve Dockerfile (#1962) - add curl for HTTPS healthchecks (https://github.com/Kozea/Radicale/issues/1961) - remove default TCP port EXPOSE command as it is not really usefull (https://forums.docker.com/t/what-is-the-use-of-expose-in-docker-file/37726) --- Dockerfile | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/Dockerfile b/Dockerfile index f6ac22f6..1fe5f380 100644 --- a/Dockerfile +++ b/Dockerfile @@ -19,14 +19,12 @@ WORKDIR /app RUN addgroup -g 1000 radicale \ && adduser radicale --home /var/lib/radicale --system --uid 1000 --disabled-password -G radicale \ - && apk add --no-cache ca-certificates openssl + && apk add --no-cache ca-certificates openssl curl COPY --chown=radicale:radicale --from=builder /app/venv /app # Persistent storage for data VOLUME /var/lib/radicale -# TCP port of Radicale -EXPOSE 5232 # Run Radicale ENTRYPOINT [ "/app/bin/python", "/app/bin/radicale"] CMD ["--hosts", "0.0.0.0:5232,[::]:5232"]