diff --git a/synapse/config/cas.py b/synapse/config/cas.py
index d23dcf96b2..fa59c350c1 100644
--- a/synapse/config/cas.py
+++ b/synapse/config/cas.py
@@ -66,6 +66,17 @@ class CasConfig(Config):
self.cas_enable_registration = cas_config.get("enable_registration", True)
+ self.cas_allow_numeric_ids = cas_config.get("allow_numeric_ids")
+ self.cas_numeric_ids_prefix = cas_config.get("numeric_ids_prefix")
+ if (
+ self.cas_numeric_ids_prefix is not None
+ and self.cas_numeric_ids_prefix.isalnum() is False
+ ):
+ raise ConfigError(
+ "Only alphanumeric characters are allowed for numeric IDs prefix",
+ ("cas_config", "numeric_ids_prefix"),
+ )
+
self.idp_name = cas_config.get("idp_name", "CAS")
self.idp_icon = cas_config.get("idp_icon")
self.idp_brand = cas_config.get("idp_brand")
@@ -77,6 +88,8 @@ class CasConfig(Config):
self.cas_displayname_attribute = None
self.cas_required_attributes = []
self.cas_enable_registration = False
+ self.cas_allow_numeric_ids = False
+ self.cas_numeric_ids_prefix = "u"
# CAS uses a legacy required attributes mapping, not the one provided by
diff --git a/synapse/config/federation.py b/synapse/config/federation.py
index 9032effac3..cf29fa2562 100644
--- a/synapse/config/federation.py
+++ b/synapse/config/federation.py
@@ -42,6 +42,10 @@ class FederationConfig(Config):
for domain in federation_domain_whitelist:
self.federation_domain_whitelist[domain] = True
+ self.federation_whitelist_endpoint_enabled = config.get(
+ "federation_whitelist_endpoint_enabled", False
+ )
+
federation_metrics_domains = config.get("federation_metrics_domains") or []
validate_config(
_METRICS_FOR_DOMAINS_SCHEMA,
diff --git a/synapse/federation/transport/server/_base.py b/synapse/federation/transport/server/_base.py
index 23d1254127..db0f5076a9 100644
--- a/synapse/federation/transport/server/_base.py
+++ b/synapse/federation/transport/server/_base.py
@@ -180,7 +180,11 @@ def _parse_auth_header(header_bytes: bytes) -> Tuple[str, str, str, Optional[str
"""
try:
header_str = header_bytes.decode("utf-8")
- params = re.split(" +", header_str)[1].split(",")
+ space_or_tab = "[ \t]"
+ params = re.split(
+ rf"{space_or_tab}*,{space_or_tab}*",
+ re.split(r"^X-Matrix +", header_str, maxsplit=1)[1],
+ )
param_dict: Dict[str, str] = {
k.lower(): v for k, v in [param.split("=", maxsplit=1) for param in params]
}
diff --git a/synapse/handlers/cas.py b/synapse/handlers/cas.py
index 153123ee83..cc3d641b7d 100644
--- a/synapse/handlers/cas.py
+++ b/synapse/handlers/cas.py
@@ -78,6 +78,8 @@ class CasHandler:
self._cas_displayname_attribute = hs.config.cas.cas_displayname_attribute
self._cas_required_attributes = hs.config.cas.cas_required_attributes
self._cas_enable_registration = hs.config.cas.cas_enable_registration
+ self._cas_allow_numeric_ids = hs.config.cas.cas_allow_numeric_ids
+ self._cas_numeric_ids_prefix = hs.config.cas.cas_numeric_ids_prefix
self._http_client = hs.get_proxied_http_client()
@@ -188,6 +190,9 @@ class CasHandler:
for child in root[0]:
if child.tag.endswith("user"):
user = child.text
+ # if numeric user IDs are allowed and username is numeric then we add the prefix so Synapse can handle it
+ if self._cas_allow_numeric_ids and user is not None and user.isdigit():
+ user = f"{self._cas_numeric_ids_prefix}{user}"
if child.tag.endswith("attributes"):
for attribute in child:
# ElementTree library expands the namespace in
diff --git a/synapse/handlers/profile.py b/synapse/handlers/profile.py
index e51e282a9f..6663d4b271 100644
--- a/synapse/handlers/profile.py
+++ b/synapse/handlers/profile.py
@@ -20,7 +20,7 @@
#
import logging
import random
-from typing import TYPE_CHECKING, Optional, Union
+from typing import TYPE_CHECKING, List, Optional, Union
from synapse.api.errors import (
AuthError,
@@ -64,8 +64,10 @@ class ProfileHandler:
self.user_directory_handler = hs.get_user_directory_handler()
self.request_ratelimiter = hs.get_request_ratelimiter()
- self.max_avatar_size = hs.config.server.max_avatar_size
- self.allowed_avatar_mimetypes = hs.config.server.allowed_avatar_mimetypes
+ self.max_avatar_size: Optional[int] = hs.config.server.max_avatar_size
+ self.allowed_avatar_mimetypes: Optional[List[str]] = (
+ hs.config.server.allowed_avatar_mimetypes
+ )
self._is_mine_server_name = hs.is_mine_server_name
@@ -337,6 +339,12 @@ class ProfileHandler:
return False
if self.max_avatar_size:
+ if media_info.media_length is None:
+ logger.warning(
+ "Forbidding avatar change to %s: unknown media size",
+ mxc,
+ )
+ return False
# Ensure avatar does not exceed max allowed avatar size
if media_info.media_length > self.max_avatar_size:
logger.warning(
diff --git a/synapse/handlers/sync.py b/synapse/handlers/sync.py
index 8ff45a3353..0bef58351c 100644
--- a/synapse/handlers/sync.py
+++ b/synapse/handlers/sync.py
@@ -1803,38 +1803,13 @@ class SyncHandler:
# Step 1a, check for changes in devices of users we share a room
# with
- #
- # We do this in two different ways depending on what we have cached.
- # If we already have a list of all the user that have changed since
- # the last sync then it's likely more efficient to compare the rooms
- # they're in with the rooms the syncing user is in.
- #
- # If we don't have that info cached then we get all the users that
- # share a room with our user and check if those users have changed.
- cache_result = self.store.get_cached_device_list_changes(
- since_token.device_list_key
- )
- if cache_result.hit:
- changed_users = cache_result.entities
-
- result = await self.store.get_rooms_for_users(changed_users)
-
- for changed_user_id, entries in result.items():
- # Check if the changed user shares any rooms with the user,
- # or if the changed user is the syncing user (as we always
- # want to include device list updates of their own devices).
- if user_id == changed_user_id or any(
- rid in joined_rooms for rid in entries
- ):
- users_that_have_changed.add(changed_user_id)
- else:
- users_that_have_changed = (
- await self._device_handler.get_device_changes_in_shared_rooms(
- user_id,
- sync_result_builder.joined_room_ids,
- from_token=since_token,
- )
+ users_that_have_changed = (
+ await self._device_handler.get_device_changes_in_shared_rooms(
+ user_id,
+ sync_result_builder.joined_room_ids,
+ from_token=since_token,
)
+ )
# Step 1b, check for newly joined rooms
for room_id in newly_joined_rooms:
diff --git a/synapse/replication/tcp/client.py b/synapse/replication/tcp/client.py
index ba257d34e6..5e5387fdcb 100644
--- a/synapse/replication/tcp/client.py
+++ b/synapse/replication/tcp/client.py
@@ -55,6 +55,7 @@ from synapse.replication.tcp.streams.partial_state import (
)
from synapse.types import PersistedEventPosition, ReadReceipt, StreamKeyType, UserID
from synapse.util.async_helpers import Linearizer, timeout_deferred
+from synapse.util.iterutils import batch_iter
from synapse.util.metrics import Measure
if TYPE_CHECKING:
@@ -150,9 +151,15 @@ class ReplicationDataHandler:
if row.entity.startswith("@") and not row.is_signature:
room_ids = await self.store.get_rooms_for_user(row.entity)
all_room_ids.update(room_ids)
- self.notifier.on_new_event(
- StreamKeyType.DEVICE_LIST, token, rooms=all_room_ids
- )
+
+ # `all_room_ids` can be large, so let's wake up those streams in batches
+ for batched_room_ids in batch_iter(all_room_ids, 100):
+ self.notifier.on_new_event(
+ StreamKeyType.DEVICE_LIST, token, rooms=batched_room_ids
+ )
+
+ # Yield to reactor so that we don't block.
+ await self._clock.sleep(0)
elif stream_name == PushersStream.NAME:
for row in rows:
if row.deleted:
diff --git a/synapse/rest/client/rendezvous.py b/synapse/rest/client/rendezvous.py
index 143f057651..27bf53314a 100644
--- a/synapse/rest/client/rendezvous.py
+++ b/synapse/rest/client/rendezvous.py
@@ -34,6 +34,9 @@ if TYPE_CHECKING:
logger = logging.getLogger(__name__)
+# n.b [MSC3886](https://github.com/matrix-org/matrix-spec-proposals/pull/3886) has now been closed.
+# However, we want to keep this implementation around for some time.
+# TODO: define an end-of-life date for this implementation.
class MSC3886RendezvousServlet(RestServlet):
"""
This is a placeholder implementation of [MSC3886](https://github.com/matrix-org/matrix-spec-proposals/pull/3886)
diff --git a/synapse/rest/client/sync.py b/synapse/rest/client/sync.py
index 2b103ca6a8..d19aaf0e22 100644
--- a/synapse/rest/client/sync.py
+++ b/synapse/rest/client/sync.py
@@ -47,6 +47,7 @@ from synapse.http.site import SynapseRequest
from synapse.logging.opentracing import trace_with_opname
from synapse.types import JsonDict, Requester, StreamToken
from synapse.util import json_decoder
+from synapse.util.caches.lrucache import LruCache
from ._base import client_patterns, set_timeline_upper_limit
@@ -110,6 +111,11 @@ class SyncRestServlet(RestServlet):
self._msc2654_enabled = hs.config.experimental.msc2654_enabled
self._msc3773_enabled = hs.config.experimental.msc3773_enabled
+ self._json_filter_cache: LruCache[str, bool] = LruCache(
+ max_size=1000,
+ cache_name="sync_valid_filter",
+ )
+
async def on_GET(self, request: SynapseRequest) -> Tuple[int, JsonDict]:
# This will always be set by the time Twisted calls us.
assert request.args is not None
@@ -177,7 +183,13 @@ class SyncRestServlet(RestServlet):
filter_object = json_decoder.decode(filter_id)
except Exception:
raise SynapseError(400, "Invalid filter JSON", errcode=Codes.NOT_JSON)
- self.filtering.check_valid_filter(filter_object)
+
+ # We cache the validation, as this can get quite expensive if people use
+ # a literal json blob as a query param.
+ if not self._json_filter_cache.get(filter_id):
+ self.filtering.check_valid_filter(filter_object)
+ self._json_filter_cache[filter_id] = True
+
set_timeline_upper_limit(
filter_object, self.hs.config.server.filter_timeline_limit
)
diff --git a/synapse/rest/synapse/client/__init__.py b/synapse/rest/synapse/client/__init__.py
index ba6576d4db..7b5bfc0421 100644
--- a/synapse/rest/synapse/client/__init__.py
+++ b/synapse/rest/synapse/client/__init__.py
@@ -23,6 +23,7 @@ from typing import TYPE_CHECKING, Mapping
from twisted.web.resource import Resource
+from synapse.rest.synapse.client.federation_whitelist import FederationWhitelistResource
from synapse.rest.synapse.client.new_user_consent import NewUserConsentResource
from synapse.rest.synapse.client.pick_idp import PickIdpResource
from synapse.rest.synapse.client.pick_username import pick_username_resource
@@ -77,6 +78,9 @@ def build_synapse_client_resource_tree(hs: "HomeServer") -> Mapping[str, Resourc
# To be removed in Synapse v1.32.0.
resources["/_matrix/saml2"] = res
+ if hs.config.federation.federation_whitelist_endpoint_enabled:
+ resources[FederationWhitelistResource.PATH] = FederationWhitelistResource(hs)
+
if hs.config.experimental.msc4108_enabled:
resources["/_synapse/client/rendezvous"] = MSC4108RendezvousSessionResource(hs)
diff --git a/synapse/rest/synapse/client/federation_whitelist.py b/synapse/rest/synapse/client/federation_whitelist.py
new file mode 100644
index 0000000000..2b8f0320e0
--- /dev/null
+++ b/synapse/rest/synapse/client/federation_whitelist.py
@@ -0,0 +1,66 @@
+#
+# This file is licensed under the Affero General Public License (AGPL) version 3.
+#
+# Copyright (C) 2024 New Vector, Ltd
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as
+# published by the Free Software Foundation, either version 3 of the
+# License, or (at your option) any later version.
+#
+# See the GNU Affero General Public License for more details:
+# <https://www.gnu.org/licenses/agpl-3.0.html>.
+#
+
+import logging
+from typing import TYPE_CHECKING, Tuple
+
+from synapse.http.server import DirectServeJsonResource
+from synapse.http.site import SynapseRequest
+from synapse.types import JsonDict
+
+if TYPE_CHECKING:
+ from synapse.server import HomeServer
+
+logger = logging.getLogger(__name__)
+
+
+class FederationWhitelistResource(DirectServeJsonResource):
+ """Custom endpoint (disabled by default) to fetch the federation whitelist
+ config.
+
+ Only enabled if `federation_whitelist_endpoint_enabled` feature is enabled.
+
+ Response format:
+
+ {
+ "whitelist_enabled": true, // Whether the federation whitelist is being enforced
+ "whitelist": [ // Which server names are allowed by the whitelist
+ "example.com"
+ ]
+ }
+ """
+
+ PATH = "/_synapse/client/v1/config/federation_whitelist"
+
+ def __init__(self, hs: "HomeServer"):
+ super().__init__()
+
+ self._federation_whitelist = hs.config.federation.federation_domain_whitelist
+
+ self._auth = hs.get_auth()
+
+ async def _async_render_GET(self, request: SynapseRequest) -> Tuple[int, JsonDict]:
+ await self._auth.get_user_by_req(request)
+
+ whitelist = []
+ if self._federation_whitelist:
+ # federation_whitelist is actually a dict, not a list
+ whitelist = list(self._federation_whitelist)
+
+ return_dict: JsonDict = {
+ "whitelist_enabled": self._federation_whitelist is not None,
+ "whitelist": whitelist,
+ }
+
+ return 200, return_dict
diff --git a/synapse/storage/databases/main/devices.py b/synapse/storage/databases/main/devices.py
index 8dbcb3f5a0..d98f0593bc 100644
--- a/synapse/storage/databases/main/devices.py
+++ b/synapse/storage/databases/main/devices.py
@@ -70,10 +70,7 @@ from synapse.types import (
from synapse.util import json_decoder, json_encoder
from synapse.util.caches.descriptors import cached, cachedList
from synapse.util.caches.lrucache import LruCache
-from synapse.util.caches.stream_change_cache import (
- AllEntitiesChangedResult,
- StreamChangeCache,
-)
+from synapse.util.caches.stream_change_cache import StreamChangeCache
from synapse.util.cancellation import cancellable
from synapse.util.iterutils import batch_iter
from synapse.util.stringutils import shortstr
@@ -832,16 +829,6 @@ class DeviceWorkerStore(RoomMemberWorkerStore, EndToEndKeyWorkerStore):
)
return {device[0]: db_to_json(device[1]) for device in devices}
- def get_cached_device_list_changes(
- self,
- from_key: int,
- ) -> AllEntitiesChangedResult:
- """Get set of users whose devices have changed since `from_key`, or None
- if that information is not in our cache.
- """
-
- return self._device_list_stream_cache.get_all_entities_changed(from_key)
-
@cancellable
async def get_all_devices_changed(
self,
@@ -1475,7 +1462,7 @@ class DeviceWorkerStore(RoomMemberWorkerStore, EndToEndKeyWorkerStore):
sql = """
SELECT DISTINCT user_id FROM device_lists_changes_in_room
- WHERE {clause} AND stream_id >= ?
+ WHERE {clause} AND stream_id > ?
"""
def _get_device_list_changes_in_rooms_txn(
diff --git a/synapse/storage/databases/main/room.py b/synapse/storage/databases/main/room.py
index 81c7bf3712..8205109548 100644
--- a/synapse/storage/databases/main/room.py
+++ b/synapse/storage/databases/main/room.py
@@ -21,13 +21,11 @@
#
import logging
-from abc import abstractmethod
from enum import Enum
from typing import (
TYPE_CHECKING,
AbstractSet,
Any,
- Awaitable,
Collection,
Dict,
List,
@@ -53,7 +51,7 @@ from synapse.api.room_versions import RoomVersion, RoomVersions
from synapse.config.homeserver import HomeServerConfig
from synapse.events import EventBase
from synapse.replication.tcp.streams.partial_state import UnPartialStatedRoomStream
-from synapse.storage._base import SQLBaseStore, db_to_json, make_in_list_sql_clause
+from synapse.storage._base import db_to_json, make_in_list_sql_clause
from synapse.storage.database import (
DatabasePool,
LoggingDatabaseConnection,
@@ -1684,6 +1682,58 @@ class RoomWorkerStore(CacheInvalidationWorkerStore):
return True
+ async def set_room_is_public(self, room_id: str, is_public: bool) -> None:
+ await self.db_pool.simple_update_one(
+ table="rooms",
+ keyvalues={"room_id": room_id},
+ updatevalues={"is_public": is_public},
+ desc="set_room_is_public",
+ )
+
+ async def set_room_is_public_appservice(
+ self, room_id: str, appservice_id: str, network_id: str, is_public: bool
+ ) -> None:
+ """Edit the appservice/network specific public room list.
+
+ Each appservice can have a number of published room lists associated
+ with them, keyed off of an appservice defined `network_id`, which
+ basically represents a single instance of a bridge to a third party
+ network.
+
+ Args:
+ room_id
+ appservice_id
+ network_id
+ is_public: Whether to publish or unpublish the room from the list.
+ """
+
+ if is_public:
+ await self.db_pool.simple_upsert(
+ table="appservice_room_list",
+ keyvalues={
+ "appservice_id": appservice_id,
+ "network_id": network_id,
+ "room_id": room_id,
+ },
+ values={},
+ insertion_values={
+ "appservice_id": appservice_id,
+ "network_id": network_id,
+ "room_id": room_id,
+ },
+ desc="set_room_is_public_appservice_true",
+ )
+ else:
+ await self.db_pool.simple_delete(
+ table="appservice_room_list",
+ keyvalues={
+ "appservice_id": appservice_id,
+ "network_id": network_id,
+ "room_id": room_id,
+ },
+ desc="set_room_is_public_appservice_false",
+ )
+
class _BackgroundUpdates:
REMOVE_TOMESTONED_ROOMS_BG_UPDATE = "remove_tombstoned_rooms_from_directory"
@@ -1702,7 +1752,7 @@ _REPLACE_ROOM_DEPTH_SQL_COMMANDS = (
)
-class RoomBackgroundUpdateStore(SQLBaseStore):
+class RoomBackgroundUpdateStore(RoomWorkerStore):
def __init__(
self,
database: DatabasePool,
@@ -1935,14 +1985,6 @@ class RoomBackgroundUpdateStore(SQLBaseStore):
return len(rooms)
- @abstractmethod
- def set_room_is_public(self, room_id: str, is_public: bool) -> Awaitable[None]:
- # this will need to be implemented if a background update is performed with
- # existing (tombstoned, public) rooms in the database.
- #
- # It's overridden by RoomStore for the synapse master.
- raise NotImplementedError()
-
async def has_auth_chain_index(self, room_id: str) -> bool:
"""Check if the room has (or can have) a chain cover index.
@@ -2349,62 +2391,6 @@ class RoomStore(RoomBackgroundUpdateStore, RoomWorkerStore):
},
)
- async def set_room_is_public(self, room_id: str, is_public: bool) -> None:
- await self.db_pool.simple_update_one(
- table="rooms",
- keyvalues={"room_id": room_id},
- updatevalues={"is_public": is_public},
- desc="set_room_is_public",
- )
-
- self.hs.get_notifier().on_new_replication_data()
-
- async def set_room_is_public_appservice(
- self, room_id: str, appservice_id: str, network_id: str, is_public: bool
- ) -> None:
- """Edit the appservice/network specific public room list.
-
- Each appservice can have a number of published room lists associated
- with them, keyed off of an appservice defined `network_id`, which
- basically represents a single instance of a bridge to a third party
- network.
-
- Args:
- room_id
- appservice_id
- network_id
- is_public: Whether to publish or unpublish the room from the list.
- """
-
- if is_public:
- await self.db_pool.simple_upsert(
- table="appservice_room_list",
- keyvalues={
- "appservice_id": appservice_id,
- "network_id": network_id,
- "room_id": room_id,
- },
- values={},
- insertion_values={
- "appservice_id": appservice_id,
- "network_id": network_id,
- "room_id": room_id,
- },
- desc="set_room_is_public_appservice_true",
- )
- else:
- await self.db_pool.simple_delete(
- table="appservice_room_list",
- keyvalues={
- "appservice_id": appservice_id,
- "network_id": network_id,
- "room_id": room_id,
- },
- desc="set_room_is_public_appservice_false",
- )
-
- self.hs.get_notifier().on_new_replication_data()
-
async def add_event_report(
self,
room_id: str,
|
