diff --git a/AUTHORS.rst b/AUTHORS.rst
index d599aec74c..3ea18eefcb 100644
--- a/AUTHORS.rst
+++ b/AUTHORS.rst
@@ -69,3 +69,6 @@ Serban Constantin <serban.constantin at gmail dot com>
Jason Robinson <jasonr at matrix.org>
* Minor fixes
+
+Joseph Weston <joseph at weston.cloud>
+ + Add admin API for querying HS version
diff --git a/changelog.d/4772.feature b/changelog.d/4772.feature
new file mode 100644
index 0000000000..19bb91f1e8
--- /dev/null
+++ b/changelog.d/4772.feature
@@ -0,0 +1 @@
+Add an endpoint to the admin API for querying the server version. Contributed by Joseph Weston.
diff --git a/changelog.d/4792.bugfix b/changelog.d/4792.bugfix
new file mode 100644
index 0000000000..b127b6254f
--- /dev/null
+++ b/changelog.d/4792.bugfix
@@ -0,0 +1 @@
+Handle batch updates in worker replication protocol.
\ No newline at end of file
diff --git a/changelog.d/4804.feature b/changelog.d/4804.feature
new file mode 100644
index 0000000000..a4c0b196f6
--- /dev/null
+++ b/changelog.d/4804.feature
@@ -0,0 +1 @@
+Add configurable rate limiting to the /register endpoint.
diff --git a/changelog.d/4815.misc b/changelog.d/4815.misc
new file mode 100644
index 0000000000..b123b36f7f
--- /dev/null
+++ b/changelog.d/4815.misc
@@ -0,0 +1 @@
+Add some docstrings.
diff --git a/changelog.d/4816.misc b/changelog.d/4816.misc
new file mode 100644
index 0000000000..43d94251f7
--- /dev/null
+++ b/changelog.d/4816.misc
@@ -0,0 +1 @@
+Add debug logger to try and track down #4422.
diff --git a/changelog.d/4817.misc b/changelog.d/4817.misc
new file mode 100644
index 0000000000..438a51dc63
--- /dev/null
+++ b/changelog.d/4817.misc
@@ -0,0 +1 @@
+Make shutdown API send explanation message to room after users have been forced joined.
diff --git a/changelog.d/4818.bugfix b/changelog.d/4818.bugfix
new file mode 100644
index 0000000000..ebbc27a433
--- /dev/null
+++ b/changelog.d/4818.bugfix
@@ -0,0 +1 @@
+Fix bug where we didn't correctly throttle sending of USER_IP commands over replication.
diff --git a/docs/admin_api/version_api.rst b/docs/admin_api/version_api.rst
new file mode 100644
index 0000000000..30a91b5f43
--- /dev/null
+++ b/docs/admin_api/version_api.rst
@@ -0,0 +1,22 @@
+Version API
+===========
+
+This API returns the running Synapse version and the Python version
+on which Synapse is being run. This is useful when a Synapse instance
+is behind a proxy that does not forward the 'Server' header (which also
+contains Synapse version information).
+
+The api is::
+
+ GET /_matrix/client/r0/admin/server_version
+
+including an ``access_token`` of a server admin.
+
+It returns a JSON body like the following:
+
+.. code:: json
+
+ {
+ "server_version": "0.99.2rc1 (b=develop, abcdef123)",
+ "python_version": "3.6.8"
+ }
diff --git a/docs/sample_config.yaml b/docs/sample_config.yaml
index e0140003fd..3dd0b4a1a8 100644
--- a/docs/sample_config.yaml
+++ b/docs/sample_config.yaml
@@ -393,6 +393,17 @@ federation_rc_reject_limit: 50
#
federation_rc_concurrent: 3
+# Number of registration requests a client can send per second.
+# Defaults to 1/minute (0.17).
+#
+#rc_registration_requests_per_second: 0.17
+
+# Number of registration requests a client can send before being
+# throttled.
+# Defaults to 3.
+#
+#rc_registration_request_burst_count: 3.0
+
# Directory where uploaded images and attachments are stored.
@@ -580,6 +591,8 @@ turn_allow_guests: True
## Registration ##
+# Registration can be rate-limited using the parameters in the "Ratelimiting"
+# section of this file.
# Enable registration for new users.
enable_registration: False
@@ -657,17 +670,6 @@ trusted_third_party_id_servers:
#
autocreate_auto_join_rooms: true
-# Number of registration requests a client can send per second.
-# Defaults to 1/minute (0.17).
-#
-#rc_registration_requests_per_second: 0.17
-
-# Number of registration requests a client can send before being
-# throttled.
-# Defaults to 3.
-#
-#rc_registration_request_burst_count: 3.0
-
## Metrics ###
diff --git a/synapse/config/ratelimiting.py b/synapse/config/ratelimiting.py
index 54b71e6841..093042fdb9 100644
--- a/synapse/config/ratelimiting.py
+++ b/synapse/config/ratelimiting.py
@@ -27,6 +27,13 @@ class RatelimitConfig(Config):
self.federation_rc_reject_limit = config["federation_rc_reject_limit"]
self.federation_rc_concurrent = config["federation_rc_concurrent"]
+ self.rc_registration_requests_per_second = config.get(
+ "rc_registration_requests_per_second", 0.17,
+ )
+ self.rc_registration_request_burst_count = config.get(
+ "rc_registration_request_burst_count", 3,
+ )
+
def default_config(self, **kwargs):
return """\
## Ratelimiting ##
@@ -62,4 +69,15 @@ class RatelimitConfig(Config):
# single server
#
federation_rc_concurrent: 3
+
+ # Number of registration requests a client can send per second.
+ # Defaults to 1/minute (0.17).
+ #
+ #rc_registration_requests_per_second: 0.17
+
+ # Number of registration requests a client can send before being
+ # throttled.
+ # Defaults to 3.
+ #
+ #rc_registration_request_burst_count: 3.0
"""
diff --git a/synapse/config/registration.py b/synapse/config/registration.py
index d32f6fff73..d34dc9e456 100644
--- a/synapse/config/registration.py
+++ b/synapse/config/registration.py
@@ -54,13 +54,6 @@ class RegistrationConfig(Config):
config.get("disable_msisdn_registration", False)
)
- self.rc_registration_requests_per_second = config.get(
- "rc_registration_requests_per_second", 0.17,
- )
- self.rc_registration_request_burst_count = config.get(
- "rc_registration_request_burst_count", 3,
- )
-
def default_config(self, generate_secrets=False, **kwargs):
if generate_secrets:
registration_shared_secret = 'registration_shared_secret: "%s"' % (
@@ -71,6 +64,8 @@ class RegistrationConfig(Config):
return """\
## Registration ##
+ # Registration can be rate-limited using the parameters in the "Ratelimiting"
+ # section of this file.
# Enable registration for new users.
enable_registration: False
@@ -147,17 +142,6 @@ class RegistrationConfig(Config):
# users cannot be auto-joined since they do not exist.
#
autocreate_auto_join_rooms: true
-
- # Number of registration requests a client can send per second.
- # Defaults to 1/minute (0.17).
- #
- #rc_registration_requests_per_second: 0.17
-
- # Number of registration requests a client can send before being
- # throttled.
- # Defaults to 3.
- #
- #rc_registration_request_burst_count: 3.0
""" % locals()
def add_arguments(self, parser):
diff --git a/synapse/handlers/register.py b/synapse/handlers/register.py
index 47d5e276f8..03130edc54 100644
--- a/synapse/handlers/register.py
+++ b/synapse/handlers/register.py
@@ -61,7 +61,7 @@ class RegistrationHandler(BaseHandler):
self.user_directory_handler = hs.get_user_directory_handler()
self.captcha_client = CaptchaServerHttpClient(hs)
self.identity_handler = self.hs.get_handlers().identity_handler
- self.ratelimiter = hs.get_ratelimiter()
+ self.ratelimiter = hs.get_registration_ratelimiter()
self._next_generated_user_id = None
diff --git a/synapse/handlers/sync.py b/synapse/handlers/sync.py
index 0e504f945d..b7916bce00 100644
--- a/synapse/handlers/sync.py
+++ b/synapse/handlers/sync.py
@@ -39,6 +39,9 @@ from synapse.visibility import filter_events_for_client
logger = logging.getLogger(__name__)
+# Debug logger for https://github.com/matrix-org/synapse/issues/4422
+issue4422_logger = logging.getLogger("synapse.handler.sync.4422_debug")
+
SYNC_RESPONSE_CACHE_MS = 2 * 60 * 1000
# Counts the number of times we returned a non-empty sync. `type` is one of
@@ -969,7 +972,7 @@ class SyncHandler(object):
for joined_room in sync_result_builder.joined:
room_id = joined_room.room_id
if room_id in newly_joined_rooms:
- logger.info(
+ issue4422_logger.debug(
"Sync result for newly joined room %s: %r",
room_id, joined_room,
)
@@ -1443,7 +1446,7 @@ class SyncHandler(object):
prev_membership = None
if old_mem_ev:
prev_membership = old_mem_ev.membership
- logger.info(
+ issue4422_logger.debug(
"Previous membership for room %s with join: %s (event %s)",
room_id, prev_membership, old_mem_ev_id,
)
@@ -1570,7 +1573,7 @@ class SyncHandler(object):
if newly_joined:
# debugging for https://github.com/matrix-org/synapse/issues/4422
- logger.info(
+ issue4422_logger.debug(
"RoomSyncResultBuilder events for newly joined room %s: %r",
room_id, entry.events,
)
@@ -1697,7 +1700,7 @@ class SyncHandler(object):
if newly_joined:
# debug for https://github.com/matrix-org/synapse/issues/4422
- logger.info(
+ issue4422_logger.debug(
"Timeline events after filtering in newly-joined room %s: %r",
room_id, batch,
)
@@ -1936,18 +1939,31 @@ class SyncResultBuilder(object):
"""Used to help build up a new SyncResult for a user
Attributes:
- joined (list[JoinedSyncResult]):
- archived (list[ArchivedSyncResult]):
+ sync_config (SyncConfig)
+ full_state (bool)
+ since_token (StreamToken)
+ now_token (StreamToken)
+ joined_room_ids (list[str])
+
+ # The following mirror the fields in a sync response
+ presence (list)
+ account_data (list)
+ joined (list[JoinedSyncResult])
+ invited (list[InvitedSyncResult])
+ archived (list[ArchivedSyncResult])
+ device (list)
+ groups (GroupsSyncResult|None)
+ to_device (list)
"""
def __init__(self, sync_config, full_state, since_token, now_token,
joined_room_ids):
"""
Args:
- sync_config(SyncConfig)
- full_state(bool): The full_state flag as specified by user
- since_token(StreamToken): The token supplied by user, or None.
- now_token(StreamToken): The token to sync up to.
-
+ sync_config (SyncConfig)
+ full_state (bool): The full_state flag as specified by user
+ since_token (StreamToken): The token supplied by user, or None.
+ now_token (StreamToken): The token to sync up to.
+ joined_room_ids (list[str]): List of rooms the user is joined to
"""
self.sync_config = sync_config
self.full_state = full_state
@@ -1975,8 +1991,8 @@ class RoomSyncResultBuilder(object):
Args:
room_id(str)
rtype(str): One of `"joined"` or `"archived"`
- events(list[FrozenEvent]): List of events to include in the room (more events
- may be added when generating result).
+ events(list[FrozenEvent]): List of events to include in the room
+ (more events may be added when generating result).
newly_joined(bool): If the user has newly joined the room
full_state(bool): Whether the full state should be sent in result
since_token(StreamToken): Earliest point to return events from, or None
diff --git a/synapse/replication/tcp/protocol.py b/synapse/replication/tcp/protocol.py
index 49ae5b3355..55630ba9a7 100644
--- a/synapse/replication/tcp/protocol.py
+++ b/synapse/replication/tcp/protocol.py
@@ -451,7 +451,7 @@ class ServerReplicationStreamProtocol(BaseReplicationStreamProtocol):
@defer.inlineCallbacks
def subscribe_to_stream(self, stream_name, token):
- """Subscribe the remote to a streams.
+ """Subscribe the remote to a stream.
This invloves checking if they've missed anything and sending those
updates down if they have. During that time new updates for the stream
@@ -478,11 +478,36 @@ class ServerReplicationStreamProtocol(BaseReplicationStreamProtocol):
# Now we can send any updates that came in while we were subscribing
pending_rdata = self.pending_rdata.pop(stream_name, [])
+ updates = []
for token, update in pending_rdata:
- # Only send updates newer than the current token
- if token > current_token:
+ # If the token is null, it is part of a batch update. Batches
+ # are multiple updates that share a single token. To denote
+ # this, the token is set to None for all tokens in the batch
+ # except for the last. If we find a None token, we keep looking
+ # through tokens until we find one that is not None and then
+ # process all previous updates in the batch as if they had the
+ # final token.
+ if token is None:
+ # Store this update as part of a batch
+ updates.append(update)
+ continue
+
+ if token <= current_token:
+ # This update or batch of updates is older than
+ # current_token, dismiss it
+ updates = []
+ continue
+
+ updates.append(update)
+
+ # Send all updates that are part of this batch with the
+ # found token
+ for update in updates:
self.send_command(RdataCommand(stream_name, token, update))
+ # Clear stored updates
+ updates = []
+
# They're now fully subscribed
self.replication_streams.add(stream_name)
except Exception as e:
diff --git a/synapse/rest/client/v1/admin.py b/synapse/rest/client/v1/admin.py
index 2e303264f6..2a29f0c2af 100644
--- a/synapse/rest/client/v1/admin.py
+++ b/synapse/rest/client/v1/admin.py
@@ -17,12 +17,14 @@
import hashlib
import hmac
import logging
+import platform
from six import text_type
from six.moves import http_client
from twisted.internet import defer
+import synapse
from synapse.api.constants import Membership, UserTypes
from synapse.api.errors import AuthError, Codes, NotFoundError, SynapseError
from synapse.http.servlet import (
@@ -32,6 +34,7 @@ from synapse.http.servlet import (
parse_string,
)
from synapse.types import UserID, create_requester
+from synapse.util.versionstring import get_version_string
from .base import ClientV1RestServlet, client_path_patterns
@@ -66,6 +69,25 @@ class UsersRestServlet(ClientV1RestServlet):
defer.returnValue((200, ret))
+class VersionServlet(ClientV1RestServlet):
+ PATTERNS = client_path_patterns("/admin/server_version")
+
+ @defer.inlineCallbacks
+ def on_GET(self, request):
+ requester = yield self.auth.get_user_by_req(request)
+ is_admin = yield self.auth.is_server_admin(requester.user)
+
+ if not is_admin:
+ raise AuthError(403, "You are not a server admin")
+
+ ret = {
+ 'server_version': get_version_string(synapse),
+ 'python_version': platform.python_version(),
+ }
+
+ defer.returnValue((200, ret))
+
+
class UserRegisterServlet(ClientV1RestServlet):
"""
Attributes:
@@ -763,3 +785,4 @@ def register_servlets(hs, http_server):
QuarantineMediaInRoom(hs).register(http_server)
ListMediaInRoom(hs).register(http_server)
UserRegisterServlet(hs).register(http_server)
+ VersionServlet(hs).register(http_server)
diff --git a/synapse/rest/client/v2_alpha/register.py b/synapse/rest/client/v2_alpha/register.py
index b7f354570c..6f34029431 100644
--- a/synapse/rest/client/v2_alpha/register.py
+++ b/synapse/rest/client/v2_alpha/register.py
@@ -196,7 +196,7 @@ class RegisterRestServlet(RestServlet):
self.identity_handler = hs.get_handlers().identity_handler
self.room_member_handler = hs.get_room_member_handler()
self.macaroon_gen = hs.get_macaroon_generator()
- self.ratelimiter = hs.get_ratelimiter()
+ self.ratelimiter = hs.get_registration_ratelimiter()
self.clock = hs.get_clock()
@interactive_auth_handler
diff --git a/synapse/server.py b/synapse/server.py
index 4323e7ff12..72835e8c86 100644
--- a/synapse/server.py
+++ b/synapse/server.py
@@ -206,6 +206,7 @@ class HomeServer(object):
self.clock = Clock(reactor)
self.distributor = Distributor()
self.ratelimiter = Ratelimiter()
+ self.registration_ratelimiter = Ratelimiter()
self.datastore = None
@@ -251,6 +252,9 @@ class HomeServer(object):
def get_ratelimiter(self):
return self.ratelimiter
+ def get_registration_ratelimiter(self):
+ return self.registration_ratelimiter
+
def build_federation_client(self):
return FederationClient(self)
diff --git a/synapse/storage/stream.py b/synapse/storage/stream.py
index b5aa849f4c..580fafeb3a 100644
--- a/synapse/storage/stream.py
+++ b/synapse/storage/stream.py
@@ -191,14 +191,18 @@ class StreamWorkerStore(EventsWorkerStore, SQLBaseStore):
@defer.inlineCallbacks
def get_room_events_stream_for_rooms(self, room_ids, from_key, to_key, limit=0,
order='DESC'):
- """
+ """Get new room events in stream ordering since `from_key`.
Args:
- room_ids:
- from_key:
- to_key:
- limit:
- order:
+ room_id (str)
+ from_key (str): Token from which no events are returned before
+ to_key (str): Token from which no events are returned after. (This
+ is typically the current stream token)
+ limit (int): Maximum number of events to return
+ order (str): Either "DESC" or "ASC". Determines which events are
+ returned when the result is limited. If "DESC" then the most
+ recent `limit` events are returned, otherwise returns the
+ oldest `limit` events.
Returns:
Deferred[dict[str,tuple[list[FrozenEvent], str]]]
diff --git a/tests/rest/client/v1/test_admin.py b/tests/rest/client/v1/test_admin.py
index 407bf0ac4c..ea03b7e523 100644
--- a/tests/rest/client/v1/test_admin.py
+++ b/tests/rest/client/v1/test_admin.py
@@ -20,14 +20,48 @@ import json
from mock import Mock
from synapse.api.constants import UserTypes
-from synapse.rest.client.v1.admin import register_servlets
+from synapse.rest.client.v1 import admin, login
from tests import unittest
+class VersionTestCase(unittest.HomeserverTestCase):
+
+ servlets = [
+ admin.register_servlets,
+ login.register_servlets,
+ ]
+
+ url = '/_matrix/client/r0/admin/server_version'
+
+ def test_version_string(self):
+ self.register_user("admin", "pass", admin=True)
+ self.admin_token = self.login("admin", "pass")
+
+ request, channel = self.make_request("GET", self.url,
+ access_token=self.admin_token)
+ self.render(request)
+
+ self.assertEqual(200, int(channel.result["code"]),
+ msg=channel.result["body"])
+ self.assertEqual({'server_version', 'python_version'},
+ set(channel.json_body.keys()))
+
+ def test_inaccessible_to_non_admins(self):
+ self.register_user("unprivileged-user", "pass", admin=False)
+ user_token = self.login("unprivileged-user", "pass")
+
+ request, channel = self.make_request("GET", self.url,
+ access_token=user_token)
+ self.render(request)
+
+ self.assertEqual(403, int(channel.result['code']),
+ msg=channel.result['body'])
+
+
class UserRegisterTestCase(unittest.HomeserverTestCase):
- servlets = [register_servlets]
+ servlets = [admin.register_servlets]
def make_homeserver(self, reactor, clock):
|
