diff --git a/.circleci/config.yml b/.circleci/config.yml
index 98c217dd1d..34a6c87b35 100644
--- a/.circleci/config.yml
+++ b/.circleci/config.yml
@@ -8,6 +8,13 @@ jobs:
- run: docker login --username $DOCKER_HUB_USERNAME --password $DOCKER_HUB_PASSWORD
- run: docker push matrixdotorg/synapse:${CIRCLE_TAG}
- run: docker push matrixdotorg/synapse:${CIRCLE_TAG}-py3
+ dockerhubuploadreleaseshhs:
+ machine: true
+ steps:
+ - checkout
+ - run: docker build -f docker/Dockerfile --label gitsha1=${CIRCLE_SHA1} -t matrixdotorg/synapse:${CIRCLE_TAG} --build-arg PYTHON_VERSION=3.7 .
+ - run: docker login --username $DOCKER_HUB_USERNAME --password $DOCKER_HUB_PASSWORD
+ - run: docker push matrixdotorg/synapse:${CIRCLE_TAG}
dockerhubuploadlatest:
machine: true
steps:
@@ -27,6 +34,12 @@ workflows:
only: /v[0-9].[0-9]+.[0-9]+.*/
branches:
ignore: /.*/
+ - dockerhubuploadreleaseshhs:
+ filters:
+ tags:
+ only: /shhs-v[0-9]+.[0-9]+.*/
+ branches:
+ ignore: /.*/
- dockerhubuploadlatest:
filters:
branches:
diff --git a/.gitignore b/.gitignore
index a84c41b0c9..702dd33eb1 100644
--- a/.gitignore
+++ b/.gitignore
@@ -19,6 +19,7 @@ _trial_temp*/
/*.signing.key
/env/
/homeserver*.yaml
+/logs
/media_store/
/uploads
@@ -37,4 +38,3 @@ _trial_temp*/
/docs/build/
/htmlcov
/pip-wheel-metadata/
-
diff --git a/changelog.d/5072.feature b/changelog.d/5072.feature
new file mode 100644
index 0000000000..99fda5616a
--- /dev/null
+++ b/changelog.d/5072.feature
@@ -0,0 +1 @@
+Synapse can now be configured to not join remote rooms of a given "complexity" (currently, state events). This option can be used to prevent adverse performance on resource-constrained homeservers.
\ No newline at end of file
diff --git a/changelog.d/5099.misc b/changelog.d/5099.misc
new file mode 100644
index 0000000000..21863e4e3e
--- /dev/null
+++ b/changelog.d/5099.misc
@@ -0,0 +1 @@
+Python 2 has been removed from the CI.
diff --git a/docs/sample_config.yaml b/docs/sample_config.yaml
index bf9cd88b15..768a003638 100644
--- a/docs/sample_config.yaml
+++ b/docs/sample_config.yaml
@@ -278,6 +278,17 @@ listeners:
# Used by phonehome stats to group together related servers.
#server_context: context
+# Resource-constrained Homeserver Settings
+#
+# If limit_large_remote_room_joins is True, the room complexity will be
+# checked before a user joins a new remote room. If it is above
+# limit_large_remote_room_complexity, it will disallow joining or
+# instantly leave.
+#
+# Uncomment the below lines to enable:
+#limit_large_remote_room_joins: True
+#limit_large_remote_room_complexity: 1.0
+
# Whether to require a user to be in the room to add an alias to it.
# Defaults to 'true'.
#
diff --git a/synapse/config/repository.py b/synapse/config/repository.py
index 80a628d9b0..feff90cfbb 100644
--- a/synapse/config/repository.py
+++ b/synapse/config/repository.py
@@ -87,13 +87,18 @@ def parse_thumbnail_requirements(thumbnail_sizes):
class ContentRepositoryConfig(Config):
def read_config(self, config, **kwargs):
+ self.enable_media_repo = config.get("enable_media_repo", True)
+
self.max_upload_size = self.parse_size(config.get("max_upload_size", "10M"))
self.max_image_pixels = self.parse_size(config.get("max_image_pixels", "32M"))
self.max_spider_size = self.parse_size(config.get("max_spider_size", "10M"))
- self.media_store_path = self.ensure_directory(
- config.get("media_store_path", "media_store")
- )
+ if self.enable_media_repo:
+ self.media_store_path = self.ensure_directory(config.get("media_store_path", "media_store"))
+ self.uploads_path = self.ensure_directory(config.get("uploads_path", "uploads"))
+ else:
+ self.media_store_path = None
+ self.uploads_path = None
backup_media_store_path = config.get("backup_media_store_path")
@@ -150,7 +155,6 @@ class ContentRepositoryConfig(Config):
(provider_class, parsed_config, wrapper_config)
)
- self.uploads_path = self.ensure_directory(config.get("uploads_path", "uploads"))
self.dynamic_thumbnails = config.get("dynamic_thumbnails", False)
self.thumbnail_requirements = parse_thumbnail_requirements(
config.get("thumbnail_sizes", DEFAULT_THUMBNAIL_SIZES)
diff --git a/synapse/config/server.py b/synapse/config/server.py
index 2a74dea2ea..d00bb970fa 100644
--- a/synapse/config/server.py
+++ b/synapse/config/server.py
@@ -247,6 +247,12 @@ class ServerConfig(Config):
self.gc_thresholds = read_gc_thresholds(config.get("gc_thresholds", None))
+ # Resource-constrained Homeserver Configuration
+ self.limit_large_room_joins = config.get("limit_large_remote_room_joins", False)
+ self.limit_large_room_complexity = config.get(
+ "limit_large_remote_room_complexity", 1.0
+ )
+
bind_port = config.get("bind_port")
if bind_port:
if config.get("no_tls", False):
@@ -617,6 +623,17 @@ class ServerConfig(Config):
# Used by phonehome stats to group together related servers.
#server_context: context
+ # Resource-constrained Homeserver Settings
+ #
+ # If limit_large_remote_room_joins is True, the room complexity will be
+ # checked before a user joins a new remote room. If it is above
+ # limit_large_remote_room_complexity, it will disallow joining or
+ # instantly leave.
+ #
+ # Uncomment the below lines to enable:
+ #limit_large_remote_room_joins: True
+ #limit_large_remote_room_complexity: 1.0
+
# Whether to require a user to be in the room to add an alias to it.
# Defaults to 'true'.
#
diff --git a/synapse/federation/federation_client.py b/synapse/federation/federation_client.py
index 3883eb525e..4e62fbf321 100644
--- a/synapse/federation/federation_client.py
+++ b/synapse/federation/federation_client.py
@@ -995,3 +995,39 @@ class FederationClient(FederationBase):
)
raise RuntimeError("Failed to send to any server.")
+
+ @defer.inlineCallbacks
+ def get_room_complexity(self, destination, room_id):
+ """
+ Fetch the complexity of a remote room from another server.
+
+ Args:
+ destination (str): The remote server
+ room_id (str): The room ID to ask about.
+
+ Returns:
+ Deferred[dict] or Deferred[None]: Dict contains the complexity
+ metric versions, while None means we could not fetch the complexity.
+ """
+ try:
+ complexity = yield self.transport_layer.get_room_complexity(
+ destination=destination,
+ room_id=room_id
+ )
+ defer.returnValue(complexity)
+ except CodeMessageException as e:
+ # We didn't manage to get it -- probably a 404. We are okay if other
+ # servers don't give it to us.
+ logger.debug(
+ "Failed to fetch room complexity via %s for %s, got a %d",
+ destination, room_id, e.code
+ )
+ except Exception:
+ logger.exception(
+ "Failed to fetch room complexity via %s for %s",
+ destination, room_id
+ )
+
+ # If we don't manage to find it, return None. It's not an error if a
+ # server doesn't give it to us.
+ defer.returnValue(None)
diff --git a/synapse/federation/transport/client.py b/synapse/federation/transport/client.py
index aecd142309..8ba893ab35 100644
--- a/synapse/federation/transport/client.py
+++ b/synapse/federation/transport/client.py
@@ -21,7 +21,11 @@ from six.moves import urllib
from twisted.internet import defer
from synapse.api.constants import Membership
-from synapse.api.urls import FEDERATION_V1_PREFIX, FEDERATION_V2_PREFIX
+from synapse.api.urls import (
+ FEDERATION_UNSTABLE_PREFIX,
+ FEDERATION_V1_PREFIX,
+ FEDERATION_V2_PREFIX,
+)
from synapse.util.logutils import log_function
logger = logging.getLogger(__name__)
@@ -935,6 +939,28 @@ class TransportLayerClient(object):
destination=destination, path=path, data=content, ignore_backoff=True
)
+ def get_room_complexity(self, destination, room_id):
+ """
+ Args:
+ destination (str): The remote server
+ room_id (str): The room ID to ask about.
+ """
+ path = _create_path(
+ FEDERATION_UNSTABLE_PREFIX, "/rooms/%s/complexity", room_id
+ )
+
+ return self.client.get_json(
+ destination=destination,
+ path=path
+ )
+
+
+def _create_path(federation_prefix, path, *args):
+ """
+ Ensures that all args are url encoded.
+ """
+ return federation_prefix + path % tuple(urllib.parse.quote(arg, "") for arg in args)
+
def _create_v1_path(path, *args):
"""Creates a path against V1 federation API from the path template and
@@ -951,9 +977,7 @@ def _create_v1_path(path, *args):
Returns:
str
"""
- return FEDERATION_V1_PREFIX + path % tuple(
- urllib.parse.quote(arg, "") for arg in args
- )
+ return _create_path(FEDERATION_V1_PREFIX, path, *args)
def _create_v2_path(path, *args):
@@ -971,6 +995,4 @@ def _create_v2_path(path, *args):
Returns:
str
"""
- return FEDERATION_V2_PREFIX + path % tuple(
- urllib.parse.quote(arg, "") for arg in args
- )
+ return _create_path(FEDERATION_V2_PREFIX, path, *args)
diff --git a/synapse/handlers/federation.py b/synapse/handlers/federation.py
index 02d397c498..e7db501d98 100644
--- a/synapse/handlers/federation.py
+++ b/synapse/handlers/federation.py
@@ -2764,3 +2764,28 @@ class FederationHandler(BaseHandler):
)
else:
return user_joined_room(self.distributor, user, room_id)
+
+ @defer.inlineCallbacks
+ def get_room_complexity(self, remote_room_hosts, room_id):
+ """
+ Fetch the complexity of a remote room over federation.
+
+ Args:
+ remote_room_hosts (list[str]): The remote servers to ask.
+ room_id (str): The room ID to ask about.
+
+ Returns:
+ Deferred[dict] or Deferred[None]: Dict contains the complexity
+ metric versions, while None means we could not fetch the complexity.
+ """
+
+ for host in remote_room_hosts:
+ res = yield self.federation_client.get_room_complexity(host, room_id)
+
+ # We got a result, return it.
+ if res:
+ defer.returnValue(res)
+
+ # We fell off the bottom, couldn't get the complexity from anyone. Oh
+ # well.
+ defer.returnValue(None)
diff --git a/synapse/handlers/room_member.py b/synapse/handlers/room_member.py
index 66b05b4732..cef520392f 100644
--- a/synapse/handlers/room_member.py
+++ b/synapse/handlers/room_member.py
@@ -26,8 +26,7 @@ from unpaddedbase64 import decode_base64
from twisted.internet import defer
-import synapse.server
-import synapse.types
+from synapse import types
from synapse.api.constants import EventTypes, Membership
from synapse.api.errors import AuthError, Codes, SynapseError
from synapse.types import RoomID, UserID
@@ -39,6 +38,11 @@ from ._base import BaseHandler
logger = logging.getLogger(__name__)
id_server_scheme = "https://"
+ROOM_COMPLEXITY_TOO_GREAT = (
+ "Your homeserver is unable to join rooms this large or complex. "
+ "Please speak to your server administrator, or upgrade your instance "
+ "to join this room."
+)
class RoomMemberHandler(object):
@@ -561,7 +565,7 @@ class RoomMemberHandler(object):
), "Sender (%s) must be same as requester (%s)" % (sender, requester.user)
assert self.hs.is_mine(sender), "Sender must be our own: %s" % (sender,)
else:
- requester = synapse.types.create_requester(target_user)
+ requester = types.create_requester(target_user)
prev_event = yield self.event_creation_handler.deduplicate_state_event(
event, context
@@ -965,13 +969,53 @@ class RoomMemberMasterHandler(RoomMemberHandler):
self.distributor.declare("user_left_room")
@defer.inlineCallbacks
+ def _is_remote_room_too_complex(self, room_id, remote_room_hosts):
+ """
+ Check if complexity of a remote room is too great.
+
+ Args:
+ room_id (str)
+ remote_room_hosts (list[str])
+
+ Returns: bool of whether the complexity is too great, or None
+ if unable to be fetched
+ """
+ max_complexity = self.hs.config.limit_large_room_complexity
+ complexity = yield self.federation_handler.get_room_complexity(
+ remote_room_hosts, room_id
+ )
+
+ if complexity:
+ if complexity["v1"] > max_complexity:
+ return True
+ return False
+ return None
+
+ @defer.inlineCallbacks
+ def _is_local_room_too_complex(self, room_id):
+ """
+ Check if the complexity of a local room is too great.
+
+ Args:
+ room_id (str)
+
+ Returns: bool
+ """
+ max_complexity = self.hs.config.limit_large_room_complexity
+ complexity = yield self.store.get_room_complexity(room_id)
+
+ if complexity["v1"] > max_complexity:
+ return True
+
+ return False
+
+ @defer.inlineCallbacks
def _remote_join(self, requester, remote_room_hosts, room_id, user, content):
"""Implements RoomMemberHandler._remote_join
"""
# filter ourselves out of remote_room_hosts: do_invite_join ignores it
# and if it is the only entry we'd like to return a 404 rather than a
# 500.
-
remote_room_hosts = [
host for host in remote_room_hosts if host != self.hs.hostname
]
@@ -979,6 +1023,17 @@ class RoomMemberMasterHandler(RoomMemberHandler):
if len(remote_room_hosts) == 0:
raise SynapseError(404, "No known servers")
+ if self.hs.config.limit_large_room_joins:
+ # Fetch the room complexity
+ too_complex = yield self._is_remote_room_too_complex(
+ room_id, remote_room_hosts
+ )
+ if too_complex is True:
+ raise SynapseError(
+ code=400, msg=ROOM_COMPLEXITY_TOO_GREAT,
+ errcode=Codes.RESOURCE_LIMIT_EXCEEDED
+ )
+
# We don't do an auth check if we are doing an invite
# join dance for now, since we're kinda implicitly checking
# that we are allowed to join when we decide whether or not we
@@ -988,6 +1043,35 @@ class RoomMemberMasterHandler(RoomMemberHandler):
)
yield self._user_joined_room(user, room_id)
+ # Check the room we just joined wasn't too large, if we didn't fetch the
+ # complexity of it before.
+ if self.hs.config.limit_large_room_joins:
+ if too_complex is False:
+ # We checked, and we're under the limit.
+ return
+
+ # Check again, but with the local state events
+ too_complex = yield self._is_local_room_too_complex(room_id)
+
+ if too_complex is False:
+ # We're under the limit.
+ return
+
+ # The room is too large. Leave.
+ requester = types.create_requester(
+ user, None, False, None
+ )
+ yield self.update_membership(
+ requester=requester,
+ target=user,
+ room_id=room_id,
+ action="leave"
+ )
+ raise SynapseError(
+ code=400, msg=ROOM_COMPLEXITY_TOO_GREAT,
+ errcode=Codes.RESOURCE_LIMIT_EXCEEDED
+ )
+
@defer.inlineCallbacks
def _remote_reject_invite(self, requester, remote_room_hosts, room_id, target):
"""Implements RoomMemberHandler._remote_reject_invite
diff --git a/synapse/storage/events_worker.py b/synapse/storage/events_worker.py
index 6d680d405a..c7dddb2bbe 100644
--- a/synapse/storage/events_worker.py
+++ b/synapse/storage/events_worker.py
@@ -649,6 +649,15 @@ class EventsWorkerStore(SQLBaseStore):
return self.runInteraction("get_seen_events_with_rejections", f)
+ def _get_current_state_event_counts_txn(self, txn, room_id):
+ """
+ See get_current_state_event_counts.
+ """
+ sql = "SELECT COUNT(*) FROM current_state_events WHERE room_id=?"
+ txn.execute(sql, (room_id,))
+ row = txn.fetchone()
+ return row[0] if row else 0
+
def _get_total_state_event_counts_txn(self, txn, room_id):
"""
See get_total_state_event_counts.
@@ -663,9 +672,9 @@ class EventsWorkerStore(SQLBaseStore):
row = txn.fetchone()
return row[0] if row else 0
- def get_total_state_event_counts(self, room_id):
+ def get_current_state_event_counts(self, room_id):
"""
- Gets the total number of state events in a room.
+ Gets the current number of state events in a room.
Args:
room_id (str)
@@ -674,9 +683,7 @@ class EventsWorkerStore(SQLBaseStore):
Deferred[int]
"""
return self.runInteraction(
- "get_total_state_event_counts",
- self._get_total_state_event_counts_txn,
- room_id,
+ "get_current_state_event_counts", self._get_current_state_event_counts_txn, room_id
)
def _get_current_state_event_counts_txn(self, txn, room_id):
diff --git a/tests/federation/test_complexity.py b/tests/federation/test_complexity.py
index a5b03005d7..c499d9d9d6 100644
--- a/tests/federation/test_complexity.py
+++ b/tests/federation/test_complexity.py
@@ -13,12 +13,16 @@
# See the License for the specific language governing permissions and
# limitations under the License.
+from mock import Mock
+
from twisted.internet import defer
+from synapse.api.errors import Codes, SynapseError
from synapse.config.ratelimiting import FederationRateLimitConfig
from synapse.federation.transport import server
from synapse.rest import admin
from synapse.rest.client.v1 import login, room
+from synapse.types import UserID
from synapse.util.ratelimitutils import FederationRateLimiter
from tests import unittest
@@ -32,8 +36,8 @@ class RoomComplexityTests(unittest.HomeserverTestCase):
login.register_servlets,
]
- def default_config(self, name="test"):
- config = super(RoomComplexityTests, self).default_config(name=name)
+ def default_config(self, name='test'):
+ config = super().default_config(name=name)
config["limit_large_remote_room_joins"] = True
config["limit_large_remote_room_complexity"] = 0.05
return config
@@ -88,3 +92,71 @@ class RoomComplexityTests(unittest.HomeserverTestCase):
self.assertEquals(200, channel.code)
complexity = channel.json_body["v1"]
self.assertEqual(complexity, 1.23)
+
+ def test_join_too_large(self):
+
+ u1 = self.register_user("u1", "pass")
+
+ handler = self.hs.get_room_member_handler()
+ fed_transport = self.hs.get_federation_transport_client()
+
+ # Mock out some things, because we don't want to test the whole join
+ fed_transport.client.get_json = Mock(return_value=defer.succeed({"v1": 9999}))
+ handler.federation_handler.do_invite_join = Mock(return_value=defer.succeed(1))
+
+ d = handler._remote_join(
+ None,
+ ["otherserver.example"],
+ "roomid",
+ UserID.from_string(u1),
+ {"membership": "join"},
+ )
+
+ self.pump()
+
+ # The request failed with a SynapseError saying the resource limit was
+ # exceeded.
+ f = self.get_failure(d, SynapseError)
+ self.assertEqual(f.value.code, 400, f.value)
+ self.assertEqual(f.value.errcode, Codes.RESOURCE_LIMIT_EXCEEDED)
+
+ def test_join_too_large_once_joined(self):
+
+ u1 = self.register_user("u1", "pass")
+ u1_token = self.login("u1", "pass")
+
+ # Ok, this might seem a bit weird -- I want to test that we actually
+ # leave the room, but I don't want to simulate two servers. So, we make
+ # a local room, which we say we're joining remotely, even if there's no
+ # remote, because we mock that out. Then, we'll leave the (actually
+ # local) room, which will be propagated over federation in a real
+ # scenario.
+ room_1 = self.helper.create_room_as(u1, tok=u1_token)
+
+ handler = self.hs.get_room_member_handler()
+ fed_transport = self.hs.get_federation_transport_client()
+
+ # Mock out some things, because we don't want to test the whole join
+ fed_transport.client.get_json = Mock(return_value=defer.succeed(None))
+ handler.federation_handler.do_invite_join = Mock(return_value=defer.succeed(1))
+
+ # Artificially raise the complexity
+ self.hs.get_datastore().get_current_state_event_counts = lambda x: defer.succeed(
+ 600
+ )
+
+ d = handler._remote_join(
+ None,
+ ["otherserver.example"],
+ room_1,
+ UserID.from_string(u1),
+ {"membership": "join"},
+ )
+
+ self.pump()
+
+ # The request failed with a SynapseError saying the resource limit was
+ # exceeded.
+ f = self.get_failure(d, SynapseError)
+ self.assertEqual(f.value.code, 400)
+ self.assertEqual(f.value.errcode, Codes.RESOURCE_LIMIT_EXCEEDED)
diff --git a/tests/unittest.py b/tests/unittest.py
index d26804b5b5..1815c22d27 100644
--- a/tests/unittest.py
+++ b/tests/unittest.py
@@ -80,10 +80,6 @@ class TestCase(unittest.TestCase):
@around(self)
def setUp(orig):
- # enable debugging of delayed calls - this means that we get a
- # traceback when a unit test exits leaving things on the reactor.
- twisted.internet.base.DelayedCall.debug = True
-
# if we're not starting in the sentinel logcontext, then to be honest
# all future bets are off.
if LoggingContext.current_context() is not LoggingContext.sentinel:
|
