diff --git a/README.rst b/README.rst
index 759197f5ff..9503ef510a 100644
--- a/README.rst
+++ b/README.rst
@@ -245,6 +245,25 @@ Setting up a TURN server
For reliable VoIP calls to be routed via this homeserver, you MUST configure
a TURN server. See `<docs/turn-howto.rst>`_ for details.
+IPv6
+----
+
+As of Synapse 0.19 we finally support IPv6, many thanks to @kyrias and @glyph
+for providing PR #1696.
+
+However, for federation to work on hosts with IPv6 DNS servers you **must**
+be running Twisted 17.1.0 or later - see https://github.com/matrix-org/synapse/issues/1002
+for details. We can't make Synapse depend on Twisted 17.1 by default
+yet as it will break most older distributions (see https://github.com/matrix-org/synapse/pull/1909)
+so if you are using operating system dependencies you'll have to install your
+own Twisted 17.1 package via pip or backports etc.
+
+If you're running in a virtualenv then pip should have installed the newest
+Twisted automatically, but if your virtualenv is old you will need to manually
+upgrade to a newer Twisted dependency via:
+
+ pip install Twisted>=17.1.0
+
Running Synapse
===============
@@ -335,8 +354,11 @@ ArchLinux
---------
The quickest way to get up and running with ArchLinux is probably with the community package
-https://www.archlinux.org/packages/community/any/matrix-synapse/, which should pull in all
-the necessary dependencies.
+https://www.archlinux.org/packages/community/any/matrix-synapse/, which should pull in most of
+the necessary dependencies. If the default web client is to be served (enabled by default in
+the generated config),
+https://www.archlinux.org/packages/community/any/python2-matrix-angular-sdk/ will also need to
+be installed.
Alternatively, to install using pip a few changes may be needed as ArchLinux
defaults to python 3, but synapse currently assumes python 2.7 by default:
@@ -373,7 +395,7 @@ FreeBSD
Synapse can be installed via FreeBSD Ports or Packages contributed by Brendan Molloy from:
- - Ports: ``cd /usr/ports/net/py-matrix-synapse && make install clean``
+ - Ports: ``cd /usr/ports/net-im/py-matrix-synapse && make install clean``
- Packages: ``pkg install py27-matrix-synapse``
diff --git a/contrib/cmdclient/http.py b/contrib/cmdclient/http.py
index 4186897316..c833f3f318 100644
--- a/contrib/cmdclient/http.py
+++ b/contrib/cmdclient/http.py
@@ -36,15 +36,13 @@ class HttpClient(object):
the request body. This will be encoded as JSON.
Returns:
- Deferred: Succeeds when we get *any* HTTP response.
-
- The result of the deferred is a tuple of `(code, response)`,
- where `response` is a dict representing the decoded JSON body.
+ Deferred: Succeeds when we get a 2xx HTTP response. The result
+ will be the decoded JSON body.
"""
pass
def get_json(self, url, args=None):
- """ Get's some json from the given host homeserver and path
+ """ Gets some json from the given host homeserver and path
Args:
url (str): The URL to GET data from.
@@ -54,10 +52,8 @@ class HttpClient(object):
and *not* a string.
Returns:
- Deferred: Succeeds when we get *any* HTTP response.
-
- The result of the deferred is a tuple of `(code, response)`,
- where `response` is a dict representing the decoded JSON body.
+ Deferred: Succeeds when we get a 2xx HTTP response. The result
+ will be the decoded JSON body.
"""
pass
@@ -214,4 +210,4 @@ class _JsonProducer(object):
pass
def stopProducing(self):
- pass
\ No newline at end of file
+ pass
diff --git a/docs/admin_api/user_admin_api.rst b/docs/admin_api/user_admin_api.rst
new file mode 100644
index 0000000000..1c9c5a6bde
--- /dev/null
+++ b/docs/admin_api/user_admin_api.rst
@@ -0,0 +1,73 @@
+Query Account
+=============
+
+This API returns information about a specific user account.
+
+The api is::
+
+ GET /_matrix/client/r0/admin/whois/<user_id>
+
+including an ``access_token`` of a server admin.
+
+It returns a JSON body like the following:
+
+.. code:: json
+
+ {
+ "user_id": "<user_id>",
+ "devices": {
+ "": {
+ "sessions": [
+ {
+ "connections": [
+ {
+ "ip": "1.2.3.4",
+ "last_seen": 1417222374433,
+ "user_agent": "Mozilla/5.0 ..."
+ },
+ {
+ "ip": "1.2.3.10",
+ "last_seen": 1417222374500,
+ "user_agent": "Dalvik/2.1.0 ..."
+ }
+ ]
+ }
+ ]
+ }
+ }
+ }
+
+``last_seen`` is measured in milliseconds since the Unix epoch.
+
+Deactivate Account
+==================
+
+This API deactivates an account. It removes active access tokens, resets the
+password, and deletes third-party IDs (to prevent the user requesting a
+password reset).
+
+The api is::
+
+ POST /_matrix/client/r0/admin/deactivate/<user_id>
+
+including an ``access_token`` of a server admin, and an empty request body.
+
+
+Reset password
+==============
+
+Changes the password of another user.
+
+The api is::
+
+ POST /_matrix/client/r0/admin/reset_password/<user_id>
+
+with a body of:
+
+.. code:: json
+
+ {
+ "new_password": "<secret>"
+ }
+
+including an ``access_token`` of a server admin.
diff --git a/docs/metrics-howto.rst b/docs/metrics-howto.rst
index 7390ab85c9..143cd0f42f 100644
--- a/docs/metrics-howto.rst
+++ b/docs/metrics-howto.rst
@@ -21,13 +21,12 @@ How to monitor Synapse metrics using Prometheus
3. Add a prometheus target for synapse.
- It needs to set the ``metrics_path`` to a non-default value::
+ It needs to set the ``metrics_path`` to a non-default value (under ``scrape_configs``)::
- job_name: "synapse"
metrics_path: "/_synapse/metrics"
static_configs:
- - targets:
- "my.server.here:9092"
+ - targets: ["my.server.here:9092"]
If your prometheus is older than 1.5.2, you will need to replace
``static_configs`` in the above with ``target_groups``.
diff --git a/synapse/app/homeserver.py b/synapse/app/homeserver.py
index 6f5924d2c7..3457402596 100755
--- a/synapse/app/homeserver.py
+++ b/synapse/app/homeserver.py
@@ -25,7 +25,7 @@ import synapse.config.logger
from synapse.config._base import ConfigError
from synapse.python_dependencies import (
- check_requirements, DEPENDENCY_LINKS
+ check_requirements, CONDITIONAL_REQUIREMENTS
)
from synapse.rest import ClientRestResource
@@ -92,7 +92,7 @@ def build_resource_for_web_client(hs):
"\n"
"You can also disable hosting of the webclient via the\n"
"configuration option `web_client`\n"
- % {"dep": DEPENDENCY_LINKS["matrix-angular-sdk"]}
+ % {"dep": CONDITIONAL_REQUIREMENTS["web_client"].keys()[0]}
)
syweb_path = os.path.dirname(syweb.__file__)
webclient_path = os.path.join(syweb_path, "webclient")
diff --git a/synapse/config/server.py b/synapse/config/server.py
index 1f9999d57a..25e6666238 100644
--- a/synapse/config/server.py
+++ b/synapse/config/server.py
@@ -144,6 +144,12 @@ class ServerConfig(Config):
# Whether to serve a web client from the HTTP/HTTPS root resource.
web_client: True
+ # The root directory to server for the above web client.
+ # If left undefined, synapse will serve the matrix-angular-sdk web client.
+ # Make sure matrix-angular-sdk is installed with pip if web_client is True
+ # and web_client_location is undefined
+ # web_client_location: "/path/to/web/root"
+
# The public-facing base URL for the client API (not including _matrix/...)
# public_baseurl: https://example.com:8448/
diff --git a/synapse/federation/federation_client.py b/synapse/federation/federation_client.py
index deee0f4904..861441708b 100644
--- a/synapse/federation/federation_client.py
+++ b/synapse/federation/federation_client.py
@@ -474,8 +474,13 @@ class FederationClient(FederationBase):
content (object): Any additional data to put into the content field
of the event.
Return:
- A tuple of (origin (str), event (object)) where origin is the remote
- homeserver which generated the event.
+ Deferred: resolves to a tuple of (origin (str), event (object))
+ where origin is the remote homeserver which generated the event.
+
+ Fails with a ``CodeMessageException`` if the chosen remote server
+ returns a 300/400 code.
+
+ Fails with a ``RuntimeError`` if no servers were reachable.
"""
valid_memberships = {Membership.JOIN, Membership.LEAVE}
if membership not in valid_memberships:
@@ -528,6 +533,27 @@ class FederationClient(FederationBase):
@defer.inlineCallbacks
def send_join(self, destinations, pdu):
+ """Sends a join event to one of a list of homeservers.
+
+ Doing so will cause the remote server to add the event to the graph,
+ and send the event out to the rest of the federation.
+
+ Args:
+ destinations (str): Candidate homeservers which are probably
+ participating in the room.
+ pdu (BaseEvent): event to be sent
+
+ Return:
+ Deferred: resolves to a dict with members ``origin`` (a string
+ giving the serer the event was sent to, ``state`` (?) and
+ ``auth_chain``.
+
+ Fails with a ``CodeMessageException`` if the chosen remote server
+ returns a 300/400 code.
+
+ Fails with a ``RuntimeError`` if no servers were reachable.
+ """
+
for destination in destinations:
if destination == self.server_name:
continue
@@ -635,6 +661,26 @@ class FederationClient(FederationBase):
@defer.inlineCallbacks
def send_leave(self, destinations, pdu):
+ """Sends a leave event to one of a list of homeservers.
+
+ Doing so will cause the remote server to add the event to the graph,
+ and send the event out to the rest of the federation.
+
+ This is mostly useful to reject received invites.
+
+ Args:
+ destinations (str): Candidate homeservers which are probably
+ participating in the room.
+ pdu (BaseEvent): event to be sent
+
+ Return:
+ Deferred: resolves to None.
+
+ Fails with a ``CodeMessageException`` if the chosen remote server
+ returns a non-200 code.
+
+ Fails with a ``RuntimeError`` if no servers were reachable.
+ """
for destination in destinations:
if destination == self.server_name:
continue
diff --git a/synapse/federation/transport/client.py b/synapse/federation/transport/client.py
index 15a03378f5..52b2a717d2 100644
--- a/synapse/federation/transport/client.py
+++ b/synapse/federation/transport/client.py
@@ -193,6 +193,26 @@ class TransportLayerClient(object):
@defer.inlineCallbacks
@log_function
def make_membership_event(self, destination, room_id, user_id, membership):
+ """Asks a remote server to build and sign us a membership event
+
+ Note that this does not append any events to any graphs.
+
+ Args:
+ destination (str): address of remote homeserver
+ room_id (str): room to join/leave
+ user_id (str): user to be joined/left
+ membership (str): one of join/leave
+
+ Returns:
+ Deferred: Succeeds when we get a 2xx HTTP response. The result
+ will be the decoded JSON body (ie, the new event).
+
+ Fails with ``HTTPRequestException`` if we get an HTTP response
+ code >= 300.
+
+ Fails with ``NotRetryingDestination`` if we are not yet ready
+ to retry this server.
+ """
valid_memberships = {Membership.JOIN, Membership.LEAVE}
if membership not in valid_memberships:
raise RuntimeError(
@@ -201,11 +221,23 @@ class TransportLayerClient(object):
)
path = PREFIX + "/make_%s/%s/%s" % (membership, room_id, user_id)
+ ignore_backoff = False
+ retry_on_dns_fail = False
+
+ if membership == Membership.LEAVE:
+ # we particularly want to do our best to send leave events. The
+ # problem is that if it fails, we won't retry it later, so if the
+ # remote server was just having a momentary blip, the room will be
+ # out of sync.
+ ignore_backoff = True
+ retry_on_dns_fail = True
+
content = yield self.client.get_json(
destination=destination,
path=path,
- retry_on_dns_fail=False,
+ retry_on_dns_fail=retry_on_dns_fail,
timeout=20000,
+ ignore_backoff=ignore_backoff,
)
defer.returnValue(content)
@@ -232,6 +264,12 @@ class TransportLayerClient(object):
destination=destination,
path=path,
data=content,
+
+ # we want to do our best to send this through. The problem is
+ # that if it fails, we won't retry it later, so if the remote
+ # server was just having a momentary blip, the room will be out of
+ # sync.
+ ignore_backoff=True,
)
defer.returnValue(response)
diff --git a/synapse/handlers/federation.py b/synapse/handlers/federation.py
index 2d9126dd86..ebbf844489 100644
--- a/synapse/handlers/federation.py
+++ b/synapse/handlers/federation.py
@@ -171,6 +171,12 @@ class FederationHandler(BaseHandler):
yield self._get_missing_events_for_pdu(
origin, pdu, prevs, min_depth
)
+ elif prevs - seen:
+ logger.info(
+ "Not fetching %d missing events for room %r,event %s: %r...",
+ len(prevs - seen), pdu.room_id, pdu.event_id,
+ list(prevs - seen)[:5],
+ )
prevs = {e_id for e_id, _ in pdu.prev_events}
seen = set(have_seen.keys())
@@ -232,8 +238,8 @@ class FederationHandler(BaseHandler):
latest |= seen
logger.info(
- "Missing %d events for room %r: %r...",
- len(prevs - seen), pdu.room_id, list(prevs - seen)[:5]
+ "Missing %d events for room %r pdu %s: %r...",
+ len(prevs - seen), pdu.room_id, pdu.event_id, list(prevs - seen)[:5]
)
# XXX: we set timeout to 10s to help workaround
@@ -265,11 +271,17 @@ class FederationHandler(BaseHandler):
timeout=10000,
)
+ logger.info(
+ "Got %d events: %r...",
+ len(missing_events), [e.event_id for e in missing_events[:5]]
+ )
+
# We want to sort these by depth so we process them and
# tell clients about them in order.
missing_events.sort(key=lambda x: x.depth)
for e in missing_events:
+ logger.info("Handling found event %s", e.event_id)
yield self.on_receive_pdu(
origin,
e,
@@ -279,6 +291,14 @@ class FederationHandler(BaseHandler):
have_seen = yield self.store.have_events(
[ev for ev, _ in pdu.prev_events]
)
+ seen = set(have_seen.keys())
+ if prevs - seen:
+ logger.info(
+ "Still missing %d prev events for %s: %r...",
+ len(prevs - seen), pdu.event_id, list(prevs - seen)[:5]
+ )
+ else:
+ logger.info("Found all missing prev events for %s", pdu.event_id)
defer.returnValue(have_seen)
@log_function
@@ -1090,19 +1110,13 @@ class FederationHandler(BaseHandler):
@defer.inlineCallbacks
def do_remotely_reject_invite(self, target_hosts, room_id, user_id):
- try:
- origin, event = yield self._make_and_verify_event(
- target_hosts,
- room_id,
- user_id,
- "leave"
- )
- event = self._sign_event(event)
- except SynapseError:
- raise
- except CodeMessageException as e:
- logger.warn("Failed to reject invite: %s", e)
- raise SynapseError(500, "Failed to reject invite")
+ origin, event = yield self._make_and_verify_event(
+ target_hosts,
+ room_id,
+ user_id,
+ "leave"
+ )
+ event = self._sign_event(event)
# Try the host that we succesfully called /make_leave/ on first for
# the /send_leave/ request.
@@ -1112,16 +1126,10 @@ class FederationHandler(BaseHandler):
except ValueError:
pass
- try:
- yield self.replication_layer.send_leave(
- target_hosts,
- event
- )
- except SynapseError:
- raise
- except CodeMessageException as e:
- logger.warn("Failed to reject invite: %s", e)
- raise SynapseError(500, "Failed to reject invite")
+ yield self.replication_layer.send_leave(
+ target_hosts,
+ event
+ )
context = yield self.state_handler.compute_event_context(event)
diff --git a/synapse/handlers/read_marker.py b/synapse/handlers/read_marker.py
new file mode 100644
index 0000000000..b5b0303d54
--- /dev/null
+++ b/synapse/handlers/read_marker.py
@@ -0,0 +1,64 @@
+# -*- coding: utf-8 -*-
+# Copyright 2017 Vector Creations Ltd
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from ._base import BaseHandler
+
+from twisted.internet import defer
+
+from synapse.util.async import Linearizer
+
+import logging
+logger = logging.getLogger(__name__)
+
+
+class ReadMarkerHandler(BaseHandler):
+ def __init__(self, hs):
+ super(ReadMarkerHandler, self).__init__(hs)
+ self.server_name = hs.config.server_name
+ self.store = hs.get_datastore()
+ self.read_marker_linearizer = Linearizer(name="read_marker")
+ self.notifier = hs.get_notifier()
+
+ @defer.inlineCallbacks
+ def received_client_read_marker(self, room_id, user_id, event_id):
+ """Updates the read marker for a given user in a given room if the event ID given
+ is ahead in the stream relative to the current read marker.
+
+ This uses a notifier to indicate that account data should be sent down /sync if
+ the read marker has changed.
+ """
+
+ with (yield self.read_marker_linearizer.queue((room_id, user_id))):
+ account_data = yield self.store.get_account_data_for_room(user_id, room_id)
+
+ existing_read_marker = account_data.get("m.fully_read", None)
+
+ should_update = True
+
+ if existing_read_marker:
+ # Only update if the new marker is ahead in the stream
+ should_update = yield self.store.is_event_after(
+ event_id,
+ existing_read_marker['event_id']
+ )
+
+ if should_update:
+ content = {
+ "event_id": event_id
+ }
+ max_id = yield self.store.add_account_data_to_room(
+ user_id, room_id, "m.fully_read", content
+ )
+ self.notifier.on_new_event("account_data_key", max_id, users=[user_id])
diff --git a/synapse/handlers/room_member.py b/synapse/handlers/room_member.py
index 2052d6d05f..28b2c80a93 100644
--- a/synapse/handlers/room_member.py
+++ b/synapse/handlers/room_member.py
@@ -139,13 +139,6 @@ class RoomMemberHandler(BaseHandler):
)
yield user_joined_room(self.distributor, user, room_id)
- def reject_remote_invite(self, user_id, room_id, remote_room_hosts):
- return self.hs.get_handlers().federation_handler.do_remotely_reject_invite(
- remote_room_hosts,
- room_id,
- user_id
- )
-
@defer.inlineCallbacks
def update_membership(
self,
@@ -286,13 +279,21 @@ class RoomMemberHandler(BaseHandler):
else:
# send the rejection to the inviter's HS.
remote_room_hosts = remote_room_hosts + [inviter.domain]
-
+ fed_handler = self.hs.get_handlers().federation_handler
try:
- ret = yield self.reject_remote_invite(
- target.to_string(), room_id, remote_room_hosts
+ ret = yield fed_handler.do_remotely_reject_invite(
+ remote_room_hosts,
+ room_id,
+ target.to_string(),
)
defer.returnValue(ret)
- except SynapseError as e:
+ except Exception as e:
+ # if we were unable to reject the exception, just mark
+ # it as rejected on our end and plough ahead.
+ #
+ # The 'except' clause is very broad, but we need to
+ # capture everything from DNS failures upwards
+ #
logger.warn("Failed to reject invite: %s", e)
yield self.store.locally_reject_invite(
diff --git a/synapse/http/matrixfederationclient.py b/synapse/http/matrixfederationclient.py
index 62b4d7e93d..747a791f83 100644
--- a/synapse/http/matrixfederationclient.py
+++ b/synapse/http/matrixfederationclient.py
@@ -125,6 +125,8 @@ class MatrixFederationHttpClient(object):
code >= 300.
Fails with ``NotRetryingDestination`` if we are not yet ready
to retry this server.
+ (May also fail with plenty of other Exceptions for things like DNS
+ failures, connection failures, SSL failures.)
"""
limiter = yield synapse.util.retryutils.get_retry_limiter(
destination,
@@ -302,8 +304,10 @@ class MatrixFederationHttpClient(object):
Returns:
Deferred: Succeeds when we get a 2xx HTTP response. The result
- will be the decoded JSON body. On a 4xx or 5xx error response a
- CodeMessageException is raised.
+ will be the decoded JSON body.
+
+ Fails with ``HTTPRequestException`` if we get an HTTP response
+ code >= 300.
Fails with ``NotRetryingDestination`` if we are not yet ready
to retry this server.
@@ -360,8 +364,10 @@ class MatrixFederationHttpClient(object):
try the request anyway.
Returns:
Deferred: Succeeds when we get a 2xx HTTP response. The result
- will be the decoded JSON body. On a 4xx or 5xx error response a
- CodeMessageException is raised.
+ will be the decoded JSON body.
+
+ Fails with ``HTTPRequestException`` if we get an HTTP response
+ code >= 300.
Fails with ``NotRetryingDestination`` if we are not yet ready
to retry this server.
@@ -410,10 +416,11 @@ class MatrixFederationHttpClient(object):
ignore_backoff (bool): true to ignore the historical backoff data
and try the request anyway.
Returns:
- Deferred: Succeeds when we get *any* HTTP response.
+ Deferred: Succeeds when we get a 2xx HTTP response. The result
+ will be the decoded JSON body.
- The result of the deferred is a tuple of `(code, response)`,
- where `response` is a dict representing the decoded JSON body.
+ Fails with ``HTTPRequestException`` if we get an HTTP response
+ code >= 300.
Fails with ``NotRetryingDestination`` if we are not yet ready
to retry this server.
diff --git a/synapse/push/bulk_push_rule_evaluator.py b/synapse/push/bulk_push_rule_evaluator.py
index 78b095c903..f943ff640f 100644
--- a/synapse/push/bulk_push_rule_evaluator.py
+++ b/synapse/push/bulk_push_rule_evaluator.py
@@ -87,7 +87,11 @@ class BulkPushRuleEvaluator:
condition_cache = {}
for uid, rules in self.rules_by_user.items():
- display_name = room_members.get(uid, {}).get("display_name", None)
+ display_name = None
+ profile_info = room_members.get(uid)
+ if profile_info:
+ display_name = profile_info.display_name
+
if not display_name:
# Handle the case where we are pushing a membership event to
# that user, as they might not be already joined.
diff --git a/synapse/replication/slave/storage/events.py b/synapse/replication/slave/storage/events.py
index 4ca1e5aa8c..ab48ff925e 100644
--- a/synapse/replication/slave/storage/events.py
+++ b/synapse/replication/slave/storage/events.py
@@ -102,9 +102,6 @@ class SlavedEventStore(BaseSlavedStore):
_get_state_groups_from_groups_txn = (
DataStore._get_state_groups_from_groups_txn.__func__
)
- _get_state_group_from_group = (
- StateStore.__dict__["_get_state_group_from_group"]
- )
get_recent_event_ids_for_room = (
StreamStore.__dict__["get_recent_event_ids_for_room"]
)
diff --git a/synapse/rest/__init__.py b/synapse/rest/__init__.py
index f9f5a3e077..aa8d874f96 100644
--- a/synapse/rest/__init__.py
+++ b/synapse/rest/__init__.py
@@ -40,6 +40,7 @@ from synapse.rest.client.v2_alpha import (
register,
auth,
receipts,
+ read_marker,
keys,
tokenrefresh,
tags,
@@ -88,6 +89,7 @@ class ClientRestResource(JsonResource):
register.register_servlets(hs, client_resource)
auth.register_servlets(hs, client_resource)
receipts.register_servlets(hs, client_resource)
+ read_marker.register_servlets(hs, client_resource)
keys.register_servlets(hs, client_resource)
tokenrefresh.register_servlets(hs, client_resource)
tags.register_servlets(hs, client_resource)
diff --git a/synapse/rest/client/v1/directory.py b/synapse/rest/client/v1/directory.py
index 8930f1826f..f15aa5c13f 100644
--- a/synapse/rest/client/v1/directory.py
+++ b/synapse/rest/client/v1/directory.py
@@ -39,6 +39,7 @@ class ClientDirectoryServer(ClientV1RestServlet):
def __init__(self, hs):
super(ClientDirectoryServer, self).__init__(hs)
+ self.store = hs.get_datastore()
self.handlers = hs.get_handlers()
@defer.inlineCallbacks
@@ -70,7 +71,10 @@ class ClientDirectoryServer(ClientV1RestServlet):
logger.debug("Got servers: %s", servers)
# TODO(erikj): Check types.
- # TODO(erikj): Check that room exists
+
+ room = yield self.store.get_room(room_id)
+ if room is None:
+ raise SynapseError(400, "Room does not exist")
dir_handler = self.handlers.directory_handler
diff --git a/synapse/rest/client/v1/room.py b/synapse/rest/client/v1/room.py
index 0bdd6b5b36..c376ab8fd7 100644
--- a/synapse/rest/client/v1/room.py
+++ b/synapse/rest/client/v1/room.py
@@ -406,7 +406,13 @@ class JoinedRoomMemberListRestServlet(ClientV1RestServlet):
users_with_profile = yield self.state.get_current_user_in_room(room_id)
defer.returnValue((200, {
- "joined": users_with_profile
+ "joined": {
+ user_id: {
+ "avatar_url": profile.avatar_url,
+ "display_name": profile.display_name,
+ }
+ for user_id, profile in users_with_profile.iteritems()
+ }
}))
diff --git a/synapse/rest/client/v2_alpha/account_data.py b/synapse/rest/client/v2_alpha/account_data.py
index b16079cece..0e0a187efd 100644
--- a/synapse/rest/client/v2_alpha/account_data.py
+++ b/synapse/rest/client/v2_alpha/account_data.py
@@ -16,7 +16,7 @@
from ._base import client_v2_patterns
from synapse.http.servlet import RestServlet, parse_json_object_from_request
-from synapse.api.errors import AuthError
+from synapse.api.errors import AuthError, SynapseError
from twisted.internet import defer
@@ -82,6 +82,13 @@ class RoomAccountDataServlet(RestServlet):
body = parse_json_object_from_request(request)
+ if account_data_type == "m.fully_read":
+ raise SynapseError(
+ 405,
+ "Cannot set m.fully_read through this API."
+ " Use /rooms/!roomId:server.name/read_markers"
+ )
+
max_id = yield self.store.add_account_data_to_room(
user_id, room_id, account_data_type, body
)
diff --git a/synapse/rest/client/v2_alpha/read_marker.py b/synapse/rest/client/v2_alpha/read_marker.py
new file mode 100644
index 0000000000..2f8784fe06
--- /dev/null
+++ b/synapse/rest/client/v2_alpha/read_marker.py
@@ -0,0 +1,66 @@
+# -*- coding: utf-8 -*-
+# Copyright 2017 Vector Creations Ltd
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from twisted.internet import defer
+
+from synapse.http.servlet import RestServlet, parse_json_object_from_request
+from ._base import client_v2_patterns
+
+import logging
+
+
+logger = logging.getLogger(__name__)
+
+
+class ReadMarkerRestServlet(RestServlet):
+ PATTERNS = client_v2_patterns("/rooms/(?P<room_id>[^/]*)/read_markers$")
+
+ def __init__(self, hs):
+ super(ReadMarkerRestServlet, self).__init__()
+ self.auth = hs.get_auth()
+ self.receipts_handler = hs.get_receipts_handler()
+ self.read_marker_handler = hs.get_read_marker_handler()
+ self.presence_handler = hs.get_presence_handler()
+
+ @defer.inlineCallbacks
+ def on_POST(self, request, room_id):
+ requester = yield self.auth.get_user_by_req(request)
+
+ yield self.presence_handler.bump_presence_active_time(requester.user)
+
+ body = parse_json_object_from_request(request)
+
+ read_event_id = body.get("m.read", None)
+ if read_event_id:
+ yield self.receipts_handler.received_client_receipt(
+ room_id,
+ "m.read",
+ user_id=requester.user.to_string(),
+ event_id=read_event_id
+ )
+
+ read_marker_event_id = body.get("m.fully_read", None)
+ if read_marker_event_id:
+ yield self.read_marker_handler.received_client_read_marker(
+ room_id,
+ user_id=requester.user.to_string(),
+ event_id=read_marker_event_id
+ )
+
+ defer.returnValue((200, {}))
+
+
+def register_servlets(hs, http_server):
+ ReadMarkerRestServlet(hs).register(http_server)
diff --git a/synapse/rest/key/v2/local_key_resource.py b/synapse/rest/key/v2/local_key_resource.py
index ff95269ba8..be68d9a096 100644
--- a/synapse/rest/key/v2/local_key_resource.py
+++ b/synapse/rest/key/v2/local_key_resource.py
@@ -84,12 +84,11 @@ class LocalKey(Resource):
}
old_verify_keys = {}
- for key in self.config.old_signing_keys:
- key_id = "%s:%s" % (key.alg, key.version)
+ for key_id, key in self.config.old_signing_keys.items():
verify_key_bytes = key.encode()
old_verify_keys[key_id] = {
u"key": encode_base64(verify_key_bytes),
- u"expired_ts": key.expired,
+ u"expired_ts": key.expired_ts,
}
tls_fingerprints = self.config.tls_fingerprints
diff --git a/synapse/server.py b/synapse/server.py
index 6310152560..12754c89ae 100644
--- a/synapse/server.py
+++ b/synapse/server.py
@@ -48,6 +48,7 @@ from synapse.handlers.typing import TypingHandler
from synapse.handlers.events import EventHandler, EventStreamHandler
from synapse.handlers.initial_sync import InitialSyncHandler
from synapse.handlers.receipts import ReceiptsHandler
+from synapse.handlers.read_marker import ReadMarkerHandler
from synapse.http.client import SimpleHttpClient, InsecureInterceptableContextFactory
from synapse.http.matrixfederationclient import MatrixFederationHttpClient
from synapse.notifier import Notifier
@@ -133,6 +134,7 @@ class HomeServer(object):
'receipts_handler',
'macaroon_generator',
'tcp_replication',
+ 'read_marker_handler',
]
def __init__(self, hostname, **kwargs):
@@ -291,6 +293,9 @@ class HomeServer(object):
def build_receipts_handler(self):
return ReceiptsHandler(self)
+ def build_read_marker_handler(self):
+ return ReadMarkerHandler(self)
+
def build_tcp_replication(self):
raise NotImplementedError()
diff --git a/synapse/storage/background_updates.py b/synapse/storage/background_updates.py
index 813ad59e56..d4cf0fc59b 100644
--- a/synapse/storage/background_updates.py
+++ b/synapse/storage/background_updates.py
@@ -228,46 +228,69 @@ class BackgroundUpdateStore(SQLBaseStore):
columns (list[str]): columns/expressions to include in index
"""
- # if this is postgres, we add the indexes concurrently. Otherwise
- # we fall back to doing it inline
- if isinstance(self.database_engine, engines.PostgresEngine):
- conc = True
- else:
- conc = False
- # We don't use partial indices on SQLite as it wasn't introduced
- # until 3.8, and wheezy has 3.7
- where_clause = None
-
- sql = (
- "CREATE INDEX %(conc)s %(name)s ON %(table)s (%(columns)s)"
- " %(where_clause)s"
- ) % {
- "conc": "CONCURRENTLY" if conc else "",
- "name": index_name,
- "table": table,
- "columns": ", ".join(columns),
- "where_clause": "WHERE " + where_clause if where_clause else ""
- }
-
- def create_index_concurrently(conn):
+ def create_index_psql(conn):
conn.rollback()
# postgres insists on autocommit for the index
conn.set_session(autocommit=True)
- c = conn.cursor()
- c.execute(sql)
- conn.set_session(autocommit=False)
- def create_index(conn):
+ try:
+ c = conn.cursor()
+
+ # If a previous attempt to create the index was interrupted,
+ # we may already have a half-built index. Let's just drop it
+ # before trying to create it again.
+
+ sql = "DROP INDEX IF EXISTS %s" % (index_name,)
+ logger.debug("[SQL] %s", sql)
+ c.execute(sql)
+
+ sql = (
+ "CREATE INDEX CONCURRENTLY %(name)s ON %(table)s"
+ " (%(columns)s) %(where_clause)s"
+ ) % {
+ "name": index_name,
+ "table": table,
+ "columns": ", ".join(columns),
+ "where_clause": "WHERE " + where_clause if where_clause else ""
+ }
+ logger.debug("[SQL] %s", sql)
+ c.execute(sql)
+ finally:
+ conn.set_session(autocommit=False)
+
+ def create_index_sqlite(conn):
+ # Sqlite doesn't support concurrent creation of indexes.
+ #
+ # We don't use partial indices on SQLite as it wasn't introduced
+ # until 3.8, and wheezy has 3.7
+ #
+ # We assume that sqlite doesn't give us invalid indices; however
+ # we may still end up with the index existing but the
+ # background_updates not having been recorded if synapse got shut
+ # down at the wrong moment - hance we use IF NOT EXISTS. (SQLite
+ # has supported CREATE TABLE|INDEX IF NOT EXISTS since 3.3.0.)
+ sql = (
+ "CREATE INDEX IF NOT EXISTS %(name)s ON %(table)s"
+ " (%(columns)s)"
+ ) % {
+ "name": index_name,
+ "table": table,
+ "columns": ", ".join(columns),
+ }
+
c = conn.cursor()
+ logger.debug("[SQL] %s", sql)
c.execute(sql)
+ if isinstance(self.database_engine, engines.PostgresEngine):
+ runner = create_index_psql
+ else:
+ runner = create_index_sqlite
+
@defer.inlineCallbacks
def updater(progress, batch_size):
logger.info("Adding index %s to %s", index_name, table)
- if conc:
- yield self.runWithConnection(create_index_concurrently)
- else:
- yield self.runWithConnection(create_index)
+ yield self.runWithConnection(runner)
yield self._end_background_update(update_name)
defer.returnValue(1)
diff --git a/synapse/storage/events.py b/synapse/storage/events.py
index 64fe937bdc..a3790419dd 100644
--- a/synapse/storage/events.py
+++ b/synapse/storage/events.py
@@ -2159,6 +2159,28 @@ class EventsStore(SQLBaseStore):
]
)
+ @defer.inlineCallbacks
+ def is_event_after(self, event_id1, event_id2):
+ """Returns True if event_id1 is after event_id2 in the stream
+ """
+ to_1, so_1 = yield self._get_event_ordering(event_id1)
+ to_2, so_2 = yield self._get_event_ordering(event_id2)
+ defer.returnValue((to_1, so_1) > (to_2, so_2))
+
+ @defer.inlineCallbacks
+ def _get_event_ordering(self, event_id):
+ res = yield self._simple_select_one(
+ table="events",
+ retcols=["topological_ordering", "stream_ordering"],
+ keyvalues={"event_id": event_id},
+ allow_none=True
+ )
+
+ if not res:
+ raise SynapseError(404, "Could not find event %s" % (event_id,))
+
+ defer.returnValue((int(res["topological_ordering"]), int(res["stream_ordering"])))
+
AllNewEventsResult = namedtuple("AllNewEventsResult", [
"new_forward_events", "new_backfill_events",
diff --git a/synapse/storage/receipts.py b/synapse/storage/receipts.py
index 6b0f8c2787..efb90c3c91 100644
--- a/synapse/storage/receipts.py
+++ b/synapse/storage/receipts.py
@@ -47,10 +47,13 @@ class ReceiptsStore(SQLBaseStore):
# Returns an ObservableDeferred
res = self.get_users_with_read_receipts_in_room.cache.get((room_id,), None)
- if res and res.called and user_id in res.result:
- # We'd only be adding to the set, so no point invalidating if the
- # user is already there
- return
+ if res:
+ if isinstance(res, defer.Deferred) and res.called:
+ res = res.result
+ if user_id in res:
+ # We'd only be adding to the set, so no point invalidating if the
+ # user is already there
+ return
self.get_users_with_read_receipts_in_room.invalidate((room_id,))
diff --git a/synapse/storage/roommember.py b/synapse/storage/roommember.py
index 367dbbbcf6..7ad2198d96 100644
--- a/synapse/storage/roommember.py
+++ b/synapse/storage/roommember.py
@@ -19,6 +19,7 @@ from collections import namedtuple
from ._base import SQLBaseStore
from synapse.util.caches.descriptors import cached, cachedInlineCallbacks
+from synapse.util.stringutils import to_ascii
from synapse.api.constants import Membership, EventTypes
from synapse.types import get_domain_from_id
@@ -35,6 +36,13 @@ RoomsForUser = namedtuple(
)
+# We store this using a namedtuple so that we save about 3x space over using a
+# dict.
+ProfileInfo = namedtuple(
+ "ProfileInfo", ("avatar_url", "display_name")
+)
+
+
_MEMBERSHIP_PROFILE_UPDATE_NAME = "room_membership_profile_update"
@@ -422,20 +430,20 @@ class RoomMemberStore(SQLBaseStore):
)
users_in_room = {
- row["user_id"]: {
- "display_name": row["display_name"],
- "avatar_url": row["avatar_url"],
- }
+ to_ascii(row["user_id"]): ProfileInfo(
+ avatar_url=to_ascii(row["avatar_url"]),
+ display_name=to_ascii(row["display_name"]),
+ )
for row in rows
}
if event is not None and event.type == EventTypes.Member:
if event.membership == Membership.JOIN:
if event.event_id in member_event_ids:
- users_in_room[event.state_key] = {
- "display_name": event.content.get("displayname", None),
- "avatar_url": event.content.get("avatar_url", None),
- }
+ users_in_room[to_ascii(event.state_key)] = ProfileInfo(
+ display_name=to_ascii(event.content.get("displayname", None)),
+ avatar_url=to_ascii(event.content.get("avatar_url", None)),
+ )
defer.returnValue(users_in_room)
diff --git a/synapse/storage/state.py b/synapse/storage/state.py
index acd69944c4..a16afa8df5 100644
--- a/synapse/storage/state.py
+++ b/synapse/storage/state.py
@@ -16,6 +16,7 @@
from ._base import SQLBaseStore
from synapse.util.caches.descriptors import cached, cachedList
from synapse.util.caches import intern_string
+from synapse.util.stringutils import to_ascii
from synapse.storage.engines import PostgresEngine
from twisted.internet import defer
@@ -89,7 +90,7 @@ class StateStore(SQLBaseStore):
)
return {
- (r[0], r[1]): r[2] for r in txn
+ (intern_string(r[0]), intern_string(r[1])): to_ascii(r[2]) for r in txn
}
return self.runInteraction(
@@ -279,12 +280,7 @@ class StateStore(SQLBaseStore):
return count
- @cached(num_args=2, max_entries=100000, iterable=True)
- def _get_state_group_from_group(self, group, types):
- raise NotImplementedError()
-
- @cachedList(cached_method_name="_get_state_group_from_group",
- list_name="groups", num_args=2, inlineCallbacks=True)
+ @defer.inlineCallbacks
def _get_state_groups_from_groups(self, groups, types):
"""Returns dictionary state_group -> (dict of (type, state_key) -> event id)
"""
@@ -512,7 +508,7 @@ class StateStore(SQLBaseStore):
state_map = yield self.get_state_ids_for_events([event_id], types)
defer.returnValue(state_map[event_id])
- @cached(num_args=2, max_entries=100000)
+ @cached(num_args=2, max_entries=50000)
def _get_state_group_for_event(self, room_id, event_id):
return self._simple_select_one_onecol(
table="event_to_state_groups",
@@ -660,7 +656,7 @@ class StateStore(SQLBaseStore):
state_dict = results[group]
state_dict.update(
- ((intern_string(k[0]), intern_string(k[1])), v)
+ ((intern_string(k[0]), intern_string(k[1])), to_ascii(v))
for k, v in group_state_dict.iteritems()
)
diff --git a/synapse/util/caches/__init__.py b/synapse/util/caches/__init__.py
index 8a7774a88e..4a83c46d98 100644
--- a/synapse/util/caches/__init__.py
+++ b/synapse/util/caches/__init__.py
@@ -14,13 +14,10 @@
# limitations under the License.
import synapse.metrics
-from lrucache import LruCache
import os
CACHE_SIZE_FACTOR = float(os.environ.get("SYNAPSE_CACHE_FACTOR", 0.1))
-DEBUG_CACHES = False
-
metrics = synapse.metrics.get_metrics_for("synapse.util.caches")
caches_by_name = {}
@@ -40,10 +37,6 @@ def register_cache(name, cache):
)
-_string_cache = LruCache(int(100000 * CACHE_SIZE_FACTOR))
-_stirng_cache_metrics = register_cache("string_cache", _string_cache)
-
-
KNOWN_KEYS = {
key: key for key in
(
@@ -67,14 +60,16 @@ KNOWN_KEYS = {
def intern_string(string):
- """Takes a (potentially) unicode string and interns using custom cache
+ """Takes a (potentially) unicode string and interns it if it's ascii
"""
- new_str = _string_cache.setdefault(string, string)
- if new_str is string:
- _stirng_cache_metrics.inc_hits()
- else:
- _stirng_cache_metrics.inc_misses()
- return new_str
+ if string is None:
+ return None
+
+ try:
+ string = string.encode("ascii")
+ return intern(string)
+ except UnicodeEncodeError:
+ return string
def intern_dict(dictionary):
@@ -87,13 +82,9 @@ def intern_dict(dictionary):
def _intern_known_values(key, value):
- intern_str_keys = ("event_id", "room_id")
- intern_unicode_keys = ("sender", "user_id", "type", "state_key")
-
- if key in intern_str_keys:
- return intern(value.encode('ascii'))
+ intern_keys = ("event_id", "room_id", "sender", "user_id", "type", "state_key",)
- if key in intern_unicode_keys:
+ if key in intern_keys:
return intern_string(value)
return value
diff --git a/synapse/util/caches/descriptors.py b/synapse/util/caches/descriptors.py
index 9d0d0be1f9..807e147657 100644
--- a/synapse/util/caches/descriptors.py
+++ b/synapse/util/caches/descriptors.py
@@ -19,7 +19,7 @@ from synapse.util import unwrapFirstError, logcontext
from synapse.util.caches.lrucache import LruCache
from synapse.util.caches.treecache import TreeCache, iterate_tree_cache_entry
-from . import DEBUG_CACHES, register_cache
+from . import register_cache
from twisted.internet import defer
from collections import namedtuple
@@ -76,7 +76,7 @@ class Cache(object):
self.cache = LruCache(
max_size=max_entries, keylen=keylen, cache_type=cache_type,
- size_callback=(lambda d: len(d.result)) if iterable else None,
+ size_callback=(lambda d: len(d)) if iterable else None,
)
self.name = name
@@ -96,6 +96,17 @@ class Cache(object):
)
def get(self, key, default=_CacheSentinel, callback=None):
+ """Looks the key up in the caches.
+
+ Args:
+ key(tuple)
+ default: What is returned if key is not in the caches. If not
+ specified then function throws KeyError instead
+ callback(fn): Gets called when the entry in the cache is invalidated
+
+ Returns:
+ Either a Deferred or the raw result
+ """
callbacks = [callback] if callback else []
val = self._pending_deferred_cache.get(key, _CacheSentinel)
if val is not _CacheSentinel:
@@ -137,7 +148,7 @@ class Cache(object):
if self.sequence == entry.sequence:
existing_entry = self._pending_deferred_cache.pop(key, None)
if existing_entry is entry:
- self.cache.set(key, entry.deferred, entry.callbacks)
+ self.cache.set(key, result, entry.callbacks)
else:
entry.invalidate()
else:
@@ -335,20 +346,10 @@ class CacheDescriptor(_CacheDescriptorBase):
try:
cached_result_d = cache.get(cache_key, callback=invalidate_callback)
- observer = cached_result_d.observe()
- if DEBUG_CACHES:
- @defer.inlineCallbacks
- def check_result(cached_result):
- actual_result = yield self.function_to_call(obj, *args, **kwargs)
- if actual_result != cached_result:
- logger.error(
- "Stale cache entry %s%r: cached: %r, actual %r",
- self.orig.__name__, cache_key,
- cached_result, actual_result,
- )
- raise ValueError("Stale cache entry")
- defer.returnValue(cached_result)
- observer.addCallback(check_result)
+ if isinstance(cached_result_d, ObservableDeferred):
+ observer = cached_result_d.observe()
+ else:
+ observer = cached_result_d
except KeyError:
ret = defer.maybeDeferred(
@@ -447,7 +448,9 @@ class CacheListDescriptor(_CacheDescriptorBase):
try:
res = cache.get(tuple(key), callback=invalidate_callback)
- if not res.has_succeeded():
+ if not isinstance(res, ObservableDeferred):
+ results[arg] = res
+ elif not res.has_succeeded():
res = res.observe()
res.addCallback(lambda r, arg: (arg, r), arg)
cached_defers[arg] = res
diff --git a/synapse/util/stringutils.py b/synapse/util/stringutils.py
index a100f151d4..95a6168e16 100644
--- a/synapse/util/stringutils.py
+++ b/synapse/util/stringutils.py
@@ -40,3 +40,17 @@ def is_ascii(s):
return False
else:
return True
+
+
+def to_ascii(s):
+ """Converts a string to ascii if it is ascii, otherwise leave it alone.
+
+ If given None then will return None.
+ """
+ if s is None:
+ return None
+
+ try:
+ return s.encode("ascii")
+ except UnicodeEncodeError:
+ return s
|
