From 53ace904b2b8e2444bc518b2498947c869c8c13b Mon Sep 17 00:00:00 2001 From: Matthew Hodgson Date: Tue, 5 Dec 2017 01:29:25 +0000 Subject: total WIP skeleton for /room_keys API --- synapse/rest/client/v2_alpha/room_keys.py | 56 +++++++++++++++++++++++++++++++ 1 file changed, 56 insertions(+) create mode 100644 synapse/rest/client/v2_alpha/room_keys.py (limited to 'synapse/rest/client/v2_alpha') diff --git a/synapse/rest/client/v2_alpha/room_keys.py b/synapse/rest/client/v2_alpha/room_keys.py new file mode 100644 index 0000000000..9b93001919 --- /dev/null +++ b/synapse/rest/client/v2_alpha/room_keys.py @@ -0,0 +1,56 @@ +# -*- coding: utf-8 -*- +# Copyright 2017 New Vector Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import logging + +from twisted.internet import defer + +from synapse.api.errors import SynapseError +from synapse.http.servlet import ( + RestServlet, parse_json_object_from_request, parse_integer +) +from synapse.http.servlet import parse_string +from synapse.types import StreamToken +from ._base import client_v2_patterns + +logger = logging.getLogger(__name__) + + +class RoomKeysUploadServlet(RestServlet): + PATTERNS = client_v2_patterns("/room_keys/keys(/(?P[^/]+))?(/(?P[^/]+))?$") + + def __init__(self, hs): + """ + Args: + hs (synapse.server.HomeServer): server + """ + super(RoomKeysUploadServlet, self).__init__() + self.auth = hs.get_auth() + self.e2e_room_keys_handler = hs.get_e2e_room_keys_handler() + + @defer.inlineCallbacks + def on_POST(self, request, room_id, session_id): + requester = yield self.auth.get_user_by_req(request, allow_guest=True) + user_id = requester.user.to_string() + body = parse_json_object_from_request(request) + + result = yield self.e2e_room_keys_handler.upload_room_keys( + user_id, version, body + ) + defer.returnValue((200, result)) + + +def register_servlets(hs, http_server): + RoomKeysUploadServlet(hs).register(http_server) -- cgit 1.5.1 From 0bc4627a731d0edd437905a5b07e85421c7553d8 Mon Sep 17 00:00:00 2001 From: Matthew Hodgson Date: Tue, 5 Dec 2017 17:54:48 +0000 Subject: interim WIP checkin; doesn't build yet --- synapse/handlers/e2e_room_keys.py | 46 +++++++++++++++++++++---------- synapse/rest/client/v2_alpha/room_keys.py | 37 ++++++++++++++++++++++--- synapse/storage/e2e_room_keys.py | 20 ++++++++++++++ 3 files changed, 84 insertions(+), 19 deletions(-) (limited to 'synapse/rest/client/v2_alpha') diff --git a/synapse/handlers/e2e_room_keys.py b/synapse/handlers/e2e_room_keys.py index 78c838a829..15e3beb5ed 100644 --- a/synapse/handlers/e2e_room_keys.py +++ b/synapse/handlers/e2e_room_keys.py @@ -20,8 +20,7 @@ from canonicaljson import encode_canonical_json from twisted.internet import defer from synapse.api.errors import SynapseError, CodeMessageException -from synapse.types import get_domain_from_id -from synapse.util.logcontext import preserve_fn, make_deferred_yieldable +from synapse.util.async import Linearizer from synapse.util.retryutils import NotRetryingDestination logger = logging.getLogger(__name__) @@ -30,31 +29,48 @@ logger = logging.getLogger(__name__) class E2eRoomKeysHandler(object): def __init__(self, hs): self.store = hs.get_datastore() + self._upload_linearizer = async.Linearizer("upload_room_keys_lock") @defer.inlineCallbacks def get_room_keys(self, user_id, version, room_id, session_id): results = yield self.store.get_e2e_room_keys(user_id, version, room_id, session_id) defer.returnValue(results) + @defer.inlineCallbacks + def delete_room_keys(self, user_id, version, room_id, session_id): + yield self.store.delete_e2e_room_keys(user_id, version, room_id, session_id) + @defer.inlineCallbacks def upload_room_keys(self, user_id, version, room_keys): # TODO: Validate the JSON to make sure it has the right keys. - # go through the room_keys - for room_id in room_keys['rooms']: - for session_id in room_keys['rooms'][room_id]['sessions']: - session = room_keys['rooms'][room_id]['sessions'][session_id] - - # get a lock + # XXX: perhaps we should use a finer grained lock here? + with (yield self._upload_linearizer.queue(user_id): - # get the room_key for this particular row - yield self.store.get_e2e_room_key() + # go through the room_keys + for room_id in room_keys['rooms']: + for session_id in room_keys['rooms'][room_id]['sessions']: + room_key = room_keys['rooms'][room_id]['sessions'][session_id] - # check whether we merge or not - if() + # get the room_key for this particular row + current_room_key = yield self.store.get_e2e_room_key( + user_id, version, room_id, session_id + ) - # if so, we set it - yield self.store.set_e2e_room_key() + # check whether we merge or not. spelling it out with if/elifs rather than + # lots of booleans for legibility. + replace = False + if current_room_key: + if room_key['is_verified'] and not current_room_key['is_verified']: + replace = True + elif room_key['first_message_index'] < current_room_key['first_message_index']: + replace = True + elif room_key['forwarded_count'] < room_key['forwarded_count']: + replace = True - # release the lock + # if so, we set the new room_key + if replace: + yield self.store.set_e2e_room_key( + user_id, version, room_id, session_id, room_key + ) diff --git a/synapse/rest/client/v2_alpha/room_keys.py b/synapse/rest/client/v2_alpha/room_keys.py index 9b93001919..7291018a48 100644 --- a/synapse/rest/client/v2_alpha/room_keys.py +++ b/synapse/rest/client/v2_alpha/room_keys.py @@ -28,7 +28,7 @@ from ._base import client_v2_patterns logger = logging.getLogger(__name__) -class RoomKeysUploadServlet(RestServlet): +class RoomKeysServlet(RestServlet): PATTERNS = client_v2_patterns("/room_keys/keys(/(?P[^/]+))?(/(?P[^/]+))?$") def __init__(self, hs): @@ -41,16 +41,45 @@ class RoomKeysUploadServlet(RestServlet): self.e2e_room_keys_handler = hs.get_e2e_room_keys_handler() @defer.inlineCallbacks - def on_POST(self, request, room_id, session_id): - requester = yield self.auth.get_user_by_req(request, allow_guest=True) + def on_PUT(self, request, room_id, session_id): + requester = yield self.auth.get_user_by_req(request, allow_guest=False) user_id = requester.user.to_string() body = parse_json_object_from_request(request) + version = request.args.get("version", None) + + if session_id: + body = { "sessions": { session_id : body } } + + if room_id: + body = { "rooms": { room_id : body } } result = yield self.e2e_room_keys_handler.upload_room_keys( user_id, version, body ) defer.returnValue((200, result)) + @defer.inlineCallbacks + def on_GET(self, request, room_id, session_id): + requester = yield self.auth.get_user_by_req(request, allow_guest=False) + user_id = requester.user.to_string() + version = request.args.get("version", None) + + room_keys = yield self.e2e_room_keys_handler.get_room_keys( + user_id, version, room_id, session_id + ) + defer.returnValue((200, room_keys)) + + @defer.inlineCallbacks + def on_DELETE(self, request, room_id, session_id): + requester = yield self.auth.get_user_by_req(request, allow_guest=False) + user_id = requester.user.to_string() + version = request.args.get("version", None) + + yield self.e2e_room_keys_handler.delete_room_keys( + user_id, version, room_id, session_id + ) + defer.returnValue((200, {})) + def register_servlets(hs, http_server): - RoomKeysUploadServlet(hs).register(http_server) + RoomKeysServlet(hs).register(http_server) diff --git a/synapse/storage/e2e_room_keys.py b/synapse/storage/e2e_room_keys.py index 9f6d47e1b6..903dc083f8 100644 --- a/synapse/storage/e2e_room_keys.py +++ b/synapse/storage/e2e_room_keys.py @@ -12,6 +12,7 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. + from twisted.internet import defer from synapse.util.caches.descriptors import cached @@ -77,6 +78,9 @@ class EndToEndRoomKeyStore(SQLBaseStore): ) + # XXX: this isn't currently used and isn't tested anywhere + # it could be used in future for bulk-uploading new versions of room_keys + # for a user or something though. def set_e2e_room_keys(self, user_id, version, room_keys): def _set_e2e_room_keys_txn(txn): @@ -131,3 +135,19 @@ class EndToEndRoomKeyStore(SQLBaseStore): sessions = {} sessions['rooms'][roomId]['sessions'][session_id] = row for row in rows; defer.returnValue(sessions); + + @defer.inlineCallbacks + def delete_e2e_room_keys(self, user_id, version, room_id, session_id): + + keyvalues={ + "user_id": user_id, + "version": version, + } + if room_id: keyvalues['room_id'] = room_id + if session_id: keyvalues['session_id'] = session_id + + yield self._simple_delete( + table="e2e_room_keys", + keyvalues=keyvalues, + desc="delete_e2e_room_keys", + ) -- cgit 1.5.1 From 6b8c07abc293bd222051e769550753bc0fd6f667 Mon Sep 17 00:00:00 2001 From: Matthew Hodgson Date: Tue, 5 Dec 2017 21:44:25 +0000 Subject: make it work and fix pep8 --- synapse/handlers/e2e_room_keys.py | 69 +++++++++------ synapse/rest/__init__.py | 2 + synapse/rest/client/v2_alpha/room_keys.py | 33 ++++--- synapse/server.py | 5 ++ synapse/storage/__init__.py | 2 + synapse/storage/e2e_room_keys.py | 103 +++++++++++++--------- synapse/storage/schema/delta/46/e2e_room_keys.sql | 2 +- 7 files changed, 134 insertions(+), 82 deletions(-) (limited to 'synapse/rest/client/v2_alpha') diff --git a/synapse/handlers/e2e_room_keys.py b/synapse/handlers/e2e_room_keys.py index 15e3beb5ed..93f4ad5194 100644 --- a/synapse/handlers/e2e_room_keys.py +++ b/synapse/handlers/e2e_room_keys.py @@ -13,15 +13,12 @@ # See the License for the specific language governing permissions and # limitations under the License. -import ujson as json import logging -from canonicaljson import encode_canonical_json from twisted.internet import defer -from synapse.api.errors import SynapseError, CodeMessageException +from synapse.api.errors import StoreError from synapse.util.async import Linearizer -from synapse.util.retryutils import NotRetryingDestination logger = logging.getLogger(__name__) @@ -29,11 +26,13 @@ logger = logging.getLogger(__name__) class E2eRoomKeysHandler(object): def __init__(self, hs): self.store = hs.get_datastore() - self._upload_linearizer = async.Linearizer("upload_room_keys_lock") + self._upload_linearizer = Linearizer("upload_room_keys_lock") @defer.inlineCallbacks def get_room_keys(self, user_id, version, room_id, session_id): - results = yield self.store.get_e2e_room_keys(user_id, version, room_id, session_id) + results = yield self.store.get_e2e_room_keys( + user_id, version, room_id, session_id + ) defer.returnValue(results) @defer.inlineCallbacks @@ -46,31 +45,49 @@ class E2eRoomKeysHandler(object): # TODO: Validate the JSON to make sure it has the right keys. # XXX: perhaps we should use a finer grained lock here? - with (yield self._upload_linearizer.queue(user_id): + with (yield self._upload_linearizer.queue(user_id)): # go through the room_keys for room_id in room_keys['rooms']: for session_id in room_keys['rooms'][room_id]['sessions']: room_key = room_keys['rooms'][room_id]['sessions'][session_id] - # get the room_key for this particular row - current_room_key = yield self.store.get_e2e_room_key( - user_id, version, room_id, session_id + yield self._upload_room_key( + user_id, version, room_id, session_id, room_key ) - # check whether we merge or not. spelling it out with if/elifs rather than - # lots of booleans for legibility. - replace = False - if current_room_key: - if room_key['is_verified'] and not current_room_key['is_verified']: - replace = True - elif room_key['first_message_index'] < current_room_key['first_message_index']: - replace = True - elif room_key['forwarded_count'] < room_key['forwarded_count']: - replace = True - - # if so, we set the new room_key - if replace: - yield self.store.set_e2e_room_key( - user_id, version, room_id, session_id, room_key - ) + @defer.inlineCallbacks + def _upload_room_key(self, user_id, version, room_id, session_id, room_key): + # get the room_key for this particular row + current_room_key = None + try: + current_room_key = yield self.store.get_e2e_room_key( + user_id, version, room_id, session_id + ) + except StoreError as e: + if e.code == 404: + pass + else: + raise + + # check whether we merge or not. spelling it out with if/elifs rather + # than lots of booleans for legibility. + upsert = True + if current_room_key: + if room_key['is_verified'] and not current_room_key['is_verified']: + pass + elif ( + room_key['first_message_index'] < + current_room_key['first_message_index'] + ): + pass + elif room_key['forwarded_count'] < room_key['forwarded_count']: + pass + else: + upsert = False + + # if so, we set the new room_key + if upsert: + yield self.store.set_e2e_room_key( + user_id, version, room_id, session_id, room_key + ) diff --git a/synapse/rest/__init__.py b/synapse/rest/__init__.py index 3418f06fd6..4856822a5d 100644 --- a/synapse/rest/__init__.py +++ b/synapse/rest/__init__.py @@ -46,6 +46,7 @@ from synapse.rest.client.v2_alpha import ( receipts, register, report_event, + room_keys, sendtodevice, sync, tags, @@ -102,6 +103,7 @@ class ClientRestResource(JsonResource): auth.register_servlets(hs, client_resource) receipts.register_servlets(hs, client_resource) read_marker.register_servlets(hs, client_resource) + room_keys.register_servlets(hs, client_resource) keys.register_servlets(hs, client_resource) tokenrefresh.register_servlets(hs, client_resource) tags.register_servlets(hs, client_resource) diff --git a/synapse/rest/client/v2_alpha/room_keys.py b/synapse/rest/client/v2_alpha/room_keys.py index 7291018a48..010aed98f9 100644 --- a/synapse/rest/client/v2_alpha/room_keys.py +++ b/synapse/rest/client/v2_alpha/room_keys.py @@ -17,26 +17,25 @@ import logging from twisted.internet import defer -from synapse.api.errors import SynapseError from synapse.http.servlet import ( - RestServlet, parse_json_object_from_request, parse_integer + RestServlet, parse_json_object_from_request ) -from synapse.http.servlet import parse_string -from synapse.types import StreamToken from ._base import client_v2_patterns logger = logging.getLogger(__name__) class RoomKeysServlet(RestServlet): - PATTERNS = client_v2_patterns("/room_keys/keys(/(?P[^/]+))?(/(?P[^/]+))?$") + PATTERNS = client_v2_patterns( + "/room_keys/keys(/(?P[^/]+))?(/(?P[^/]+))?$" + ) def __init__(self, hs): """ Args: hs (synapse.server.HomeServer): server """ - super(RoomKeysUploadServlet, self).__init__() + super(RoomKeysServlet, self).__init__() self.auth = hs.get_auth() self.e2e_room_keys_handler = hs.get_e2e_room_keys_handler() @@ -45,24 +44,32 @@ class RoomKeysServlet(RestServlet): requester = yield self.auth.get_user_by_req(request, allow_guest=False) user_id = requester.user.to_string() body = parse_json_object_from_request(request) - version = request.args.get("version", None) + version = request.args.get("version")[0] if session_id: - body = { "sessions": { session_id : body } } + body = { + "sessions": { + session_id: body + } + } if room_id: - body = { "rooms": { room_id : body } } + body = { + "rooms": { + room_id: body + } + } - result = yield self.e2e_room_keys_handler.upload_room_keys( + yield self.e2e_room_keys_handler.upload_room_keys( user_id, version, body ) - defer.returnValue((200, result)) + defer.returnValue((200, {})) @defer.inlineCallbacks def on_GET(self, request, room_id, session_id): requester = yield self.auth.get_user_by_req(request, allow_guest=False) user_id = requester.user.to_string() - version = request.args.get("version", None) + version = request.args.get("version")[0] room_keys = yield self.e2e_room_keys_handler.get_room_keys( user_id, version, room_id, session_id @@ -73,7 +80,7 @@ class RoomKeysServlet(RestServlet): def on_DELETE(self, request, room_id, session_id): requester = yield self.auth.get_user_by_req(request, allow_guest=False) user_id = requester.user.to_string() - version = request.args.get("version", None) + version = request.args.get("version")[0] yield self.e2e_room_keys_handler.delete_room_keys( user_id, version, room_id, session_id diff --git a/synapse/server.py b/synapse/server.py index 140be9ebe8..706cb1361f 100644 --- a/synapse/server.py +++ b/synapse/server.py @@ -49,6 +49,7 @@ from synapse.handlers.deactivate_account import DeactivateAccountHandler from synapse.handlers.device import DeviceHandler from synapse.handlers.devicemessage import DeviceMessageHandler from synapse.handlers.e2e_keys import E2eKeysHandler +from synapse.handlers.e2e_room_keys import E2eRoomKeysHandler from synapse.handlers.events import EventHandler, EventStreamHandler from synapse.handlers.groups_local import GroupsLocalHandler from synapse.handlers.initial_sync import InitialSyncHandler @@ -127,6 +128,7 @@ class HomeServer(object): 'auth_handler', 'device_handler', 'e2e_keys_handler', + 'e2e_room_keys_handler', 'event_handler', 'event_stream_handler', 'initial_sync_handler', @@ -288,6 +290,9 @@ class HomeServer(object): def build_e2e_keys_handler(self): return E2eKeysHandler(self) + def build_e2e_room_keys_handler(self): + return E2eRoomKeysHandler(self) + def build_application_service_api(self): return ApplicationServiceApi(self) diff --git a/synapse/storage/__init__.py b/synapse/storage/__init__.py index 134e4a80f1..69cb28268a 100644 --- a/synapse/storage/__init__.py +++ b/synapse/storage/__init__.py @@ -30,6 +30,7 @@ from .appservice import ApplicationServiceStore, ApplicationServiceTransactionSt from .client_ips import ClientIpStore from .deviceinbox import DeviceInboxStore from .directory import DirectoryStore +from .e2e_room_keys import EndToEndRoomKeyStore from .end_to_end_keys import EndToEndKeyStore from .engines import PostgresEngine from .event_federation import EventFederationStore @@ -76,6 +77,7 @@ class DataStore(RoomMemberStore, RoomStore, ApplicationServiceTransactionStore, ReceiptsStore, EndToEndKeyStore, + EndToEndRoomKeyStore, SearchStore, TagsStore, AccountDataStore, diff --git a/synapse/storage/e2e_room_keys.py b/synapse/storage/e2e_room_keys.py index 903dc083f8..5982710bd5 100644 --- a/synapse/storage/e2e_room_keys.py +++ b/synapse/storage/e2e_room_keys.py @@ -15,11 +15,6 @@ from twisted.internet import defer -from synapse.util.caches.descriptors import cached - -from canonicaljson import encode_canonical_json -import ujson as json - from ._base import SQLBaseStore @@ -45,29 +40,27 @@ class EndToEndRoomKeyStore(SQLBaseStore): desc="get_e2e_room_key", ) - defer.returnValue(row); + defer.returnValue(row) def set_e2e_room_key(self, user_id, version, room_id, session_id, room_key): def _set_e2e_room_key_txn(txn): - self._simple_upsert( + self._simple_upsert_txn( txn, table="e2e_room_keys", keyvalues={ "user_id": user_id, "room_id": room_id, - "session_id": session_id, - } - values=[ - { - "version": version, - "first_message_index": room_key['first_message_index'], - "forwarded_count": room_key['forwarded_count'], - "is_verified": room_key['is_verified'], - "session_data": room_key['session_data'], - } - ], + "session_id": session_id, + }, + values={ + "version": version, + "first_message_index": room_key['first_message_index'], + "forwarded_count": room_key['forwarded_count'], + "is_verified": room_key['is_verified'], + "session_data": room_key['session_data'], + }, lock=False, ) @@ -77,7 +70,6 @@ class EndToEndRoomKeyStore(SQLBaseStore): "set_e2e_room_key", _set_e2e_room_key_txn ) - # XXX: this isn't currently used and isn't tested anywhere # it could be used in future for bulk-uploading new versions of room_keys # for a user or something though. @@ -85,23 +77,27 @@ class EndToEndRoomKeyStore(SQLBaseStore): def _set_e2e_room_keys_txn(txn): + values = [] + for room_id in room_keys['rooms']: + for session_id in room_keys['rooms'][room_id]['sessions']: + session = room_keys['rooms'][room_id]['sessions'][session_id] + values.append( + { + "user_id": user_id, + "room_id": room_id, + "session_id": session_id, + "version": version, + "first_message_index": session['first_message_index'], + "forwarded_count": session['forwarded_count'], + "is_verified": session['is_verified'], + "session_data": session['session_data'], + } + ) + self._simple_insert_many_txn( txn, table="e2e_room_keys", - values=[ - { - "user_id": user_id, - "room_id": room_id, - "session_id": session_id, - "version": version, - "first_message_index": room_keys['rooms'][room_id]['sessions'][session_id]['first_message_index'], - "forwarded_count": room_keys['rooms'][room_id]['sessions'][session_id]['forwarded_count'], - "is_verified": room_keys['rooms'][room_id]['sessions'][session_id]['is_verified'], - "session_data": room_keys['rooms'][room_id]['sessions'][session_id]['session_data'], - } - for session_id in room_keys['rooms'][room_id]['sessions'] - for room_id in room_keys['rooms'] - ] + values=values ) return True @@ -113,17 +109,22 @@ class EndToEndRoomKeyStore(SQLBaseStore): @defer.inlineCallbacks def get_e2e_room_keys(self, user_id, version, room_id, session_id): - keyvalues={ + keyvalues = { "user_id": user_id, "version": version, } - if room_id: keyvalues['room_id'] = room_id - if session_id: keyvalues['session_id'] = session_id + if room_id: + keyvalues['room_id'] = room_id + if session_id: + keyvalues['session_id'] = session_id rows = yield self._simple_select_list( table="e2e_room_keys", keyvalues=keyvalues, retcols=( + "user_id", + "room_id", + "session_id", "first_message_index", "forwarded_count", "is_verified", @@ -132,19 +133,37 @@ class EndToEndRoomKeyStore(SQLBaseStore): desc="get_e2e_room_keys", ) - sessions = {} - sessions['rooms'][roomId]['sessions'][session_id] = row for row in rows; - defer.returnValue(sessions); + # perlesque autovivification from https://stackoverflow.com/a/19829714/6764493 + class AutoVivification(dict): + def __getitem__(self, item): + try: + return dict.__getitem__(self, item) + except KeyError: + value = self[item] = type(self)() + return value + + sessions = AutoVivification() + for row in rows: + sessions['rooms'][row['room_id']]['sessions'][row['session_id']] = { + "first_message_index": row["first_message_index"], + "forwarded_count": row["forwarded_count"], + "is_verified": row["is_verified"], + "session_data": row["session_data"], + } + + defer.returnValue(sessions) @defer.inlineCallbacks def delete_e2e_room_keys(self, user_id, version, room_id, session_id): - keyvalues={ + keyvalues = { "user_id": user_id, "version": version, } - if room_id: keyvalues['room_id'] = room_id - if session_id: keyvalues['session_id'] = session_id + if room_id: + keyvalues['room_id'] = room_id + if session_id: + keyvalues['session_id'] = session_id yield self._simple_delete( table="e2e_room_keys", diff --git a/synapse/storage/schema/delta/46/e2e_room_keys.sql b/synapse/storage/schema/delta/46/e2e_room_keys.sql index 51b826e8b3..6b344c5ad7 100644 --- a/synapse/storage/schema/delta/46/e2e_room_keys.sql +++ b/synapse/storage/schema/delta/46/e2e_room_keys.sql @@ -29,7 +29,7 @@ CREATE UNIQUE INDEX e2e_room_keys_user_idx ON e2e_room_keys(user_id); CREATE UNIQUE INDEX e2e_room_keys_room_idx ON e2e_room_keys(room_id); CREATE UNIQUE INDEX e2e_room_keys_session_idx ON e2e_room_keys(session_id); --- the versioning metadata about versions of users' encrypted e2e session backups +-- the metadata for each generation of encrypted e2e session backups CREATE TABLE e2e_room_key_versions ( user_id TEXT NOT NULL, version INT NOT NULL, -- cgit 1.5.1 From cf1e2000f623afea8f3afb58e4a7659288c45777 Mon Sep 17 00:00:00 2001 From: Matthew Hodgson Date: Tue, 5 Dec 2017 23:06:43 +0000 Subject: document the API --- synapse/rest/client/v2_alpha/room_keys.py | 133 ++++++++++++++++++++++++++++++ 1 file changed, 133 insertions(+) (limited to 'synapse/rest/client/v2_alpha') diff --git a/synapse/rest/client/v2_alpha/room_keys.py b/synapse/rest/client/v2_alpha/room_keys.py index 010aed98f9..be82eccb2b 100644 --- a/synapse/rest/client/v2_alpha/room_keys.py +++ b/synapse/rest/client/v2_alpha/room_keys.py @@ -41,6 +41,79 @@ class RoomKeysServlet(RestServlet): @defer.inlineCallbacks def on_PUT(self, request, room_id, session_id): + """ + Uploads one or more encrypted E2E room keys for backup purposes. + room_id: the ID of the room the keys are for (optional) + session_id: the ID for the E2E room keys for the room (optional) + version: the version of the user's backup which this data is for. + the version must already have been created via the /change_secret API. + + Each session has: + * first_message_index: a numeric index indicating the oldest message + encrypted by this session. + * forwarded_count: how many times the uploading client claims this key + has been shared (forwarded) + * is_verified: whether the client that uploaded the keys claims they + were sent by a device which they've verified + * session_data: base64-encrypted data describing the session. + + Returns 200 OK on success with body {} + + The API is designed to be otherwise agnostic to the room_key encryption + algorithm being used. Sessions are merged with existing ones in the + backup using the heuristics: + * is_verified sessions always win over unverified sessions + * older first_message_index always win over newer sessions + * lower forwarded_count always wins over higher forwarded_count + + We trust the clients not to lie and corrupt their own backups. + + POST /room_keys/keys/!abc:matrix.org/c0ff33?version=1 HTTP/1.1 + Content-Type: application/json + + { + "first_message_index": 1, + "forwarded_count": 1, + "is_verified": false, + "session_data": "SSBBTSBBIEZJU0gK" + } + + Or... + + POST /room_keys/keys/!abc:matrix.org?version=1 HTTP/1.1 + Content-Type: application/json + + { + "sessions": { + "c0ff33": { + "first_message_index": 1, + "forwarded_count": 1, + "is_verified": false, + "session_data": "SSBBTSBBIEZJU0gK" + } + } + } + + Or... + + POST /room_keys/keys?version=1 HTTP/1.1 + Content-Type: application/json + + { + "rooms": { + "!abc:matrix.org": { + "sessions": { + "c0ff33": { + "first_message_index": 1, + "forwarded_count": 1, + "is_verified": false, + "session_data": "SSBBTSBBIEZJU0gK" + } + } + } + } + } + """ requester = yield self.auth.get_user_by_req(request, allow_guest=False) user_id = requester.user.to_string() body = parse_json_object_from_request(request) @@ -67,6 +140,57 @@ class RoomKeysServlet(RestServlet): @defer.inlineCallbacks def on_GET(self, request, room_id, session_id): + """ + Retrieves one or more encrypted E2E room keys for backup purposes. + Symmetric with the PUT version of the API. + + room_id: the ID of the room to retrieve the keys for (optional) + session_id: the ID for the E2E room keys to retrieve the keys for (optional) + version: the version of the user's backup which this data is for. + the version must already have been created via the /change_secret API. + + Returns as follows: + + GET /room_keys/keys/!abc:matrix.org/c0ff33?version=1 HTTP/1.1 + { + "first_message_index": 1, + "forwarded_count": 1, + "is_verified": false, + "session_data": "SSBBTSBBIEZJU0gK" + } + + Or... + + GET /room_keys/keys/!abc:matrix.org?version=1 HTTP/1.1 + { + "sessions": { + "c0ff33": { + "first_message_index": 1, + "forwarded_count": 1, + "is_verified": false, + "session_data": "SSBBTSBBIEZJU0gK" + } + } + } + + Or... + + GET /room_keys/keys?version=1 HTTP/1.1 + { + "rooms": { + "!abc:matrix.org": { + "sessions": { + "c0ff33": { + "first_message_index": 1, + "forwarded_count": 1, + "is_verified": false, + "session_data": "SSBBTSBBIEZJU0gK" + } + } + } + } + } + """ requester = yield self.auth.get_user_by_req(request, allow_guest=False) user_id = requester.user.to_string() version = request.args.get("version")[0] @@ -78,6 +202,15 @@ class RoomKeysServlet(RestServlet): @defer.inlineCallbacks def on_DELETE(self, request, room_id, session_id): + """ + Deletes one or more encrypted E2E room keys for a user for backup purposes. + + room_id: the ID of the room whose keys to delete (optional) + session_id: the ID for the E2E session to delete (optional) + version: the version of the user's backup which this data is for. + the version must already have been created via the /change_secret API. + """ + requester = yield self.auth.get_user_by_req(request, allow_guest=False) user_id = requester.user.to_string() version = request.args.get("version")[0] -- cgit 1.5.1 From 8ae64b270f7742abfe4cf0b8d140c81993464ea4 Mon Sep 17 00:00:00 2001 From: Matthew Hodgson Date: Wed, 6 Dec 2017 01:02:57 +0000 Subject: implement /room_keys/version too (untested) --- synapse/api/errors.py | 25 ++++++++++ synapse/handlers/e2e_room_keys.py | 47 +++++++++++++++++-- synapse/rest/client/v2_alpha/room_keys.py | 47 +++++++++++++++++++ synapse/storage/e2e_room_keys.py | 56 +++++++++++++++++++++++ synapse/storage/schema/delta/46/e2e_room_keys.sql | 2 +- 5 files changed, 171 insertions(+), 6 deletions(-) (limited to 'synapse/rest/client/v2_alpha') diff --git a/synapse/api/errors.py b/synapse/api/errors.py index b41d595059..8c97e91ba1 100644 --- a/synapse/api/errors.py +++ b/synapse/api/errors.py @@ -56,6 +56,7 @@ class Codes(object): CONSENT_NOT_GIVEN = "M_CONSENT_NOT_GIVEN" CANNOT_LEAVE_SERVER_NOTICE_ROOM = "M_CANNOT_LEAVE_SERVER_NOTICE_ROOM" MAU_LIMIT_EXCEEDED = "M_MAU_LIMIT_EXCEEDED" + WRONG_ROOM_KEYS_VERSION = "M_WRONG_ROOM_KEYS_VERSION" class CodeMessageException(RuntimeError): @@ -285,6 +286,30 @@ class LimitExceededError(SynapseError): ) +class RoomKeysVersionError(SynapseError): + """A client has tried to upload to a non-current version of the room_keys store + """ + def __init__(self, code=403, msg="Wrong room_keys version", current_version=None, + errcode=Codes.WRONG_ROOM_KEYS_VERSION): + super(RoomKeysVersionError, self).__init__(code, msg, errcode) + self.current_version = current_version + + def error_dict(self): + return cs_error( + self.msg, + self.errcode, + current_version=self.current_version, + ) + + +def cs_exception(exception): + if isinstance(exception, CodeMessageException): + return exception.error_dict() + else: + logger.error("Unknown exception type: %s", type(exception)) + return {} + + def cs_error(msg, code=Codes.UNKNOWN, **kwargs): """ Utility method for constructing an error response for client-server interactions. diff --git a/synapse/handlers/e2e_room_keys.py b/synapse/handlers/e2e_room_keys.py index 93f4ad5194..4333ca610c 100644 --- a/synapse/handlers/e2e_room_keys.py +++ b/synapse/handlers/e2e_room_keys.py @@ -17,7 +17,7 @@ import logging from twisted.internet import defer -from synapse.api.errors import StoreError +from synapse.api.errors import StoreError, SynapseError, RoomKeysVersionError from synapse.util.async import Linearizer logger = logging.getLogger(__name__) @@ -30,10 +30,13 @@ class E2eRoomKeysHandler(object): @defer.inlineCallbacks def get_room_keys(self, user_id, version, room_id, session_id): - results = yield self.store.get_e2e_room_keys( - user_id, version, room_id, session_id - ) - defer.returnValue(results) + # we deliberately take the lock to get keys so that changing the version + # works atomically + with (yield self._upload_linearizer.queue(user_id)): + results = yield self.store.get_e2e_room_keys( + user_id, version, room_id, session_id + ) + defer.returnValue(results) @defer.inlineCallbacks def delete_room_keys(self, user_id, version, room_id, session_id): @@ -44,6 +47,16 @@ class E2eRoomKeysHandler(object): # TODO: Validate the JSON to make sure it has the right keys. + # Check that the version we're trying to upload is the current version + try: + version_info = yield self.get_version_info(user_id, version) + except StoreError as e: + if e.code == 404: + raise SynapseError(404, "Version '%d' not found" % (version,)) + + if version_info.version != version: + raise RoomKeysVersionError(current_version=version_info.version) + # XXX: perhaps we should use a finer grained lock here? with (yield self._upload_linearizer.queue(user_id)): @@ -91,3 +104,27 @@ class E2eRoomKeysHandler(object): yield self.store.set_e2e_room_key( user_id, version, room_id, session_id, room_key ) + + @defer.inlineCallbacks + def create_version(self, user_id, version, version_info): + + # TODO: Validate the JSON to make sure it has the right keys. + + # lock everyone out until we've switched version + with (yield self._upload_linearizer.queue(user_id)): + yield self.store.create_version( + user_id, version, version_info + ) + + @defer.inlineCallbacks + def get_version_info(self, user_id, version): + with (yield self._upload_linearizer.queue(user_id)): + results = yield self.store.get_e2e_room_key_version( + user_id, version + ) + defer.returnValue(results) + + @defer.inlineCallbacks + def delete_version(self, user_id, version): + with (yield self._upload_linearizer.queue(user_id)): + yield self.store.delete_e2e_room_key_version(user_id, version) diff --git a/synapse/rest/client/v2_alpha/room_keys.py b/synapse/rest/client/v2_alpha/room_keys.py index be82eccb2b..4d76e1d824 100644 --- a/synapse/rest/client/v2_alpha/room_keys.py +++ b/synapse/rest/client/v2_alpha/room_keys.py @@ -221,5 +221,52 @@ class RoomKeysServlet(RestServlet): defer.returnValue((200, {})) +class RoomKeysVersionServlet(RestServlet): + PATTERNS = client_v2_patterns( + "/room_keys/version(/(?P[^/]+))?$" + ) + + def __init__(self, hs): + """ + Args: + hs (synapse.server.HomeServer): server + """ + super(RoomKeysVersionServlet, self).__init__() + self.auth = hs.get_auth() + self.e2e_room_keys_handler = hs.get_e2e_room_keys_handler() + + @defer.inlineCallbacks + def on_POST(self, request, version): + requester = yield self.auth.get_user_by_req(request, allow_guest=False) + user_id = requester.user.to_string() + info = parse_json_object_from_request(request) + + new_version = yield self.e2e_room_keys_handler.create_version( + user_id, version, info + ) + defer.returnValue((200, {"version": new_version})) + + @defer.inlineCallbacks + def on_GET(self, request, version): + requester = yield self.auth.get_user_by_req(request, allow_guest=False) + user_id = requester.user.to_string() + + info = yield self.e2e_room_keys_handler.get_version_info( + user_id, version + ) + defer.returnValue((200, info)) + + @defer.inlineCallbacks + def on_DELETE(self, request, version): + requester = yield self.auth.get_user_by_req(request, allow_guest=False) + user_id = requester.user.to_string() + + yield self.e2e_room_keys_handler.delete_version( + user_id, version + ) + defer.returnValue((200, {})) + + def register_servlets(hs, http_server): RoomKeysServlet(hs).register(http_server) + RoomKeysVersionServlet(hs).register(http_server) diff --git a/synapse/storage/e2e_room_keys.py b/synapse/storage/e2e_room_keys.py index 5982710bd5..994878acf6 100644 --- a/synapse/storage/e2e_room_keys.py +++ b/synapse/storage/e2e_room_keys.py @@ -170,3 +170,59 @@ class EndToEndRoomKeyStore(SQLBaseStore): keyvalues=keyvalues, desc="delete_e2e_room_keys", ) + + @defer.inlineCallbacks + def get_e2e_room_key_version(self, user_id, version): + + row = yield self._simple_select_one( + table="e2e_room_key_versions", + keyvalues={ + "user_id": user_id, + "version": version, + }, + retcols=( + "user_id", + "version", + "algorithm", + "auth_data", + ), + desc="get_e2e_room_key_version_info", + ) + + defer.returnValue(row) + + def create_e2e_room_key_version(self, user_id, version, info): + + def _create_e2e_room_key_version_txn(txn): + + self._simple_insert_txn( + txn, + table="e2e_room_key_versions", + values={ + "user_id": user_id, + "version": version, + "algorithm": info["algorithm"], + "auth_data": info["auth_data"], + }, + lock=False, + ) + + return True + + return self.runInteraction( + "create_e2e_room_key_version_txn", _create_e2e_room_key_version_txn + ) + + @defer.inlineCallbacks + def delete_e2e_room_key_version(self, user_id, version): + + keyvalues = { + "user_id": user_id, + "version": version, + } + + yield self._simple_delete( + table="e2e_room_key_versions", + keyvalues=keyvalues, + desc="delete_e2e_room_key_version", + ) diff --git a/synapse/storage/schema/delta/46/e2e_room_keys.sql b/synapse/storage/schema/delta/46/e2e_room_keys.sql index 6b344c5ad7..463f828c66 100644 --- a/synapse/storage/schema/delta/46/e2e_room_keys.sql +++ b/synapse/storage/schema/delta/46/e2e_room_keys.sql @@ -34,7 +34,7 @@ CREATE TABLE e2e_room_key_versions ( user_id TEXT NOT NULL, version INT NOT NULL, algorithm TEXT NOT NULL, - dummy_session_data TEXT NOT NULL + auth_data TEXT NOT NULL ); CREATE UNIQUE INDEX e2e_room_key_user_idx ON e2e_room_keys(user_id); -- cgit 1.5.1 From 69e51c7ba48a84b48ab64c8c290a232d14193a18 Mon Sep 17 00:00:00 2001 From: Matthew Hodgson Date: Wed, 6 Dec 2017 10:02:49 +0100 Subject: make /room_keys/version work --- synapse/handlers/e2e_room_keys.py | 18 +++++++++++------- synapse/rest/client/v2_alpha/room_keys.py | 9 ++++++++- synapse/storage/e2e_room_keys.py | 22 +++++++++++++++++----- synapse/storage/schema/delta/46/e2e_room_keys.sql | 4 ++-- 4 files changed, 38 insertions(+), 15 deletions(-) (limited to 'synapse/rest/client/v2_alpha') diff --git a/synapse/handlers/e2e_room_keys.py b/synapse/handlers/e2e_room_keys.py index 4333ca610c..bd58be6558 100644 --- a/synapse/handlers/e2e_room_keys.py +++ b/synapse/handlers/e2e_room_keys.py @@ -48,13 +48,16 @@ class E2eRoomKeysHandler(object): # TODO: Validate the JSON to make sure it has the right keys. # Check that the version we're trying to upload is the current version + try: version_info = yield self.get_version_info(user_id, version) except StoreError as e: if e.code == 404: - raise SynapseError(404, "Version '%d' not found" % (version,)) + raise SynapseError(404, "Version '%s' not found" % (version,)) + else: + raise e - if version_info.version != version: + if version_info['version'] != version: raise RoomKeysVersionError(current_version=version_info.version) # XXX: perhaps we should use a finer grained lock here? @@ -81,7 +84,7 @@ class E2eRoomKeysHandler(object): if e.code == 404: pass else: - raise + raise e # check whether we merge or not. spelling it out with if/elifs rather # than lots of booleans for legibility. @@ -106,20 +109,21 @@ class E2eRoomKeysHandler(object): ) @defer.inlineCallbacks - def create_version(self, user_id, version, version_info): + def create_version(self, user_id, version_info): # TODO: Validate the JSON to make sure it has the right keys. # lock everyone out until we've switched version with (yield self._upload_linearizer.queue(user_id)): - yield self.store.create_version( - user_id, version, version_info + new_version = yield self.store.create_e2e_room_key_version( + user_id, version_info ) + defer.returnValue(new_version) @defer.inlineCallbacks def get_version_info(self, user_id, version): with (yield self._upload_linearizer.queue(user_id)): - results = yield self.store.get_e2e_room_key_version( + results = yield self.store.get_e2e_room_key_version_info( user_id, version ) defer.returnValue(results) diff --git a/synapse/rest/client/v2_alpha/room_keys.py b/synapse/rest/client/v2_alpha/room_keys.py index 4d76e1d824..128b732fb1 100644 --- a/synapse/rest/client/v2_alpha/room_keys.py +++ b/synapse/rest/client/v2_alpha/room_keys.py @@ -17,6 +17,7 @@ import logging from twisted.internet import defer +from synapse.api.errors import SynapseError from synapse.http.servlet import ( RestServlet, parse_json_object_from_request ) @@ -237,15 +238,21 @@ class RoomKeysVersionServlet(RestServlet): @defer.inlineCallbacks def on_POST(self, request, version): + if version: + raise SynapseError(405, "Cannot POST to a specific version") + requester = yield self.auth.get_user_by_req(request, allow_guest=False) user_id = requester.user.to_string() info = parse_json_object_from_request(request) new_version = yield self.e2e_room_keys_handler.create_version( - user_id, version, info + user_id, info ) defer.returnValue((200, {"version": new_version})) + # we deliberately don't have a PUT /version, as these things really should + # be immutable to avoid people footgunning + @defer.inlineCallbacks def on_GET(self, request, version): requester = yield self.auth.get_user_by_req(request, allow_guest=False) diff --git a/synapse/storage/e2e_room_keys.py b/synapse/storage/e2e_room_keys.py index 994878acf6..8efca11a8c 100644 --- a/synapse/storage/e2e_room_keys.py +++ b/synapse/storage/e2e_room_keys.py @@ -172,7 +172,7 @@ class EndToEndRoomKeyStore(SQLBaseStore): ) @defer.inlineCallbacks - def get_e2e_room_key_version(self, user_id, version): + def get_e2e_room_key_version_info(self, user_id, version): row = yield self._simple_select_one( table="e2e_room_key_versions", @@ -191,23 +191,35 @@ class EndToEndRoomKeyStore(SQLBaseStore): defer.returnValue(row) - def create_e2e_room_key_version(self, user_id, version, info): + def create_e2e_room_key_version(self, user_id, info): + """Atomically creates a new version of this user's e2e_room_keys store + with the given version info. + """ def _create_e2e_room_key_version_txn(txn): + txn.execute( + "SELECT MAX(version) FROM e2e_room_key_versions WHERE user_id=?", + (user_id,) + ) + current_version = txn.fetchone()[0] + if current_version is None: + current_version = 0 + + new_version = current_version + 1 + self._simple_insert_txn( txn, table="e2e_room_key_versions", values={ "user_id": user_id, - "version": version, + "version": new_version, "algorithm": info["algorithm"], "auth_data": info["auth_data"], }, - lock=False, ) - return True + return new_version return self.runInteraction( "create_e2e_room_key_version_txn", _create_e2e_room_key_version_txn diff --git a/synapse/storage/schema/delta/46/e2e_room_keys.sql b/synapse/storage/schema/delta/46/e2e_room_keys.sql index 463f828c66..0d2a85fbe6 100644 --- a/synapse/storage/schema/delta/46/e2e_room_keys.sql +++ b/synapse/storage/schema/delta/46/e2e_room_keys.sql @@ -18,7 +18,7 @@ CREATE TABLE e2e_room_keys ( user_id TEXT NOT NULL, room_id TEXT NOT NULL, session_id TEXT NOT NULL, - version INT NOT NULL, + version TEXT NOT NULL, first_message_index INT, forwarded_count INT, is_verified BOOLEAN, @@ -32,7 +32,7 @@ CREATE UNIQUE INDEX e2e_room_keys_session_idx ON e2e_room_keys(session_id); -- the metadata for each generation of encrypted e2e session backups CREATE TABLE e2e_room_key_versions ( user_id TEXT NOT NULL, - version INT NOT NULL, + version TEXT NOT NULL, algorithm TEXT NOT NULL, auth_data TEXT NOT NULL ); -- cgit 1.5.1 From 0abb205b47158a4160ddceb317c0245d640b6e3f Mon Sep 17 00:00:00 2001 From: Matthew Hodgson Date: Mon, 18 Dec 2017 01:52:46 +0000 Subject: blindly incorporate PR review - needs testing & fixing --- synapse/api/errors.py | 11 ++- synapse/handlers/e2e_room_keys.py | 88 +++++++++++++++-------- synapse/rest/client/v2_alpha/room_keys.py | 2 + synapse/storage/e2e_room_keys.py | 69 ++++++++---------- synapse/storage/schema/delta/46/e2e_room_keys.sql | 8 +-- 5 files changed, 99 insertions(+), 79 deletions(-) (limited to 'synapse/rest/client/v2_alpha') diff --git a/synapse/api/errors.py b/synapse/api/errors.py index 8c97e91ba1..d37bcb4082 100644 --- a/synapse/api/errors.py +++ b/synapse/api/errors.py @@ -289,9 +289,14 @@ class LimitExceededError(SynapseError): class RoomKeysVersionError(SynapseError): """A client has tried to upload to a non-current version of the room_keys store """ - def __init__(self, code=403, msg="Wrong room_keys version", current_version=None, - errcode=Codes.WRONG_ROOM_KEYS_VERSION): - super(RoomKeysVersionError, self).__init__(code, msg, errcode) + def __init__(self, current_version): + """ + Args: + current_version (str): the current version of the store they should have used + """ + super(RoomKeysVersionError, self).__init__( + 403, "Wrong room_keys version", Codes.WRONG_ROOM_KEYS_VERSION + ) self.current_version = current_version def error_dict(self): diff --git a/synapse/handlers/e2e_room_keys.py b/synapse/handlers/e2e_room_keys.py index bd58be6558..dda31fdd24 100644 --- a/synapse/handlers/e2e_room_keys.py +++ b/synapse/handlers/e2e_room_keys.py @@ -24,8 +24,21 @@ logger = logging.getLogger(__name__) class E2eRoomKeysHandler(object): + """ + Implements an optional realtime backup mechanism for encrypted E2E megolm room keys. + This gives a way for users to store and recover their megolm keys if they lose all + their clients. It should also extend easily to future room key mechanisms. + The actual payload of the encrypted keys is completely opaque to the handler. + """ + def __init__(self, hs): self.store = hs.get_datastore() + + # Used to lock whenever a client is uploading key data. This prevents collisions + # between clients trying to upload the details of a new session, given all + # clients belonging to a user will receive and try to upload a new session at + # roughly the same time. Also used to lock out uploads when the key is being + # changed. self._upload_linearizer = Linearizer("upload_room_keys_lock") @defer.inlineCallbacks @@ -40,33 +53,34 @@ class E2eRoomKeysHandler(object): @defer.inlineCallbacks def delete_room_keys(self, user_id, version, room_id, session_id): - yield self.store.delete_e2e_room_keys(user_id, version, room_id, session_id) + # lock for consistency with uploading + with (yield self._upload_linearizer.queue(user_id)): + yield self.store.delete_e2e_room_keys(user_id, version, room_id, session_id) @defer.inlineCallbacks def upload_room_keys(self, user_id, version, room_keys): # TODO: Validate the JSON to make sure it has the right keys. - # Check that the version we're trying to upload is the current version - - try: - version_info = yield self.get_version_info(user_id, version) - except StoreError as e: - if e.code == 404: - raise SynapseError(404, "Version '%s' not found" % (version,)) - else: - raise e - - if version_info['version'] != version: - raise RoomKeysVersionError(current_version=version_info.version) - # XXX: perhaps we should use a finer grained lock here? with (yield self._upload_linearizer.queue(user_id)): - - # go through the room_keys - for room_id in room_keys['rooms']: - for session_id in room_keys['rooms'][room_id]['sessions']: - room_key = room_keys['rooms'][room_id]['sessions'][session_id] + # Check that the version we're trying to upload is the current version + try: + version_info = yield self.get_version_info(user_id, version) + except StoreError as e: + if e.code == 404: + raise SynapseError(404, "Version '%s' not found" % (version,)) + else: + raise e + + if version_info['version'] != version: + raise RoomKeysVersionError(current_version=version_info.version) + + # go through the room_keys. + # XXX: this should/could be done concurrently, given we're in a lock. + for room_id, room in room_keys['rooms'].iteritems(): + for session_id, session in room['sessions'].iteritems(): + room_key = session[session_id] yield self._upload_room_key( user_id, version, room_id, session_id, room_key @@ -86,10 +100,29 @@ class E2eRoomKeysHandler(object): else: raise e - # check whether we merge or not. spelling it out with if/elifs rather - # than lots of booleans for legibility. - upsert = True + if _should_replace_room_key(current_room_key, room_key): + yield self.store.set_e2e_room_key( + user_id, version, room_id, session_id, room_key + ) + + def _should_replace_room_key(current_room_key, room_key): + """ + Determine whether to replace the current_room_key in our backup for this + session (if any) with a new room_key that has been uploaded. + + Args: + current_room_key (dict): Optional, the current room_key dict if any + room_key (dict): The new room_key dict which may or may not be fit to + replace the current_room_key + + Returns: + True if current_room_key should be replaced by room_key in the backup + """ + if current_room_key: + # spelt out with if/elifs rather than nested boolean expressions + # purely for legibility. + if room_key['is_verified'] and not current_room_key['is_verified']: pass elif ( @@ -97,16 +130,11 @@ class E2eRoomKeysHandler(object): current_room_key['first_message_index'] ): pass - elif room_key['forwarded_count'] < room_key['forwarded_count']: + elif room_key['forwarded_count'] < current_room_key['forwarded_count']: pass else: - upsert = False - - # if so, we set the new room_key - if upsert: - yield self.store.set_e2e_room_key( - user_id, version, room_id, session_id, room_key - ) + return False + return True @defer.inlineCallbacks def create_version(self, user_id, version_info): diff --git a/synapse/rest/client/v2_alpha/room_keys.py b/synapse/rest/client/v2_alpha/room_keys.py index 128b732fb1..70b7b4573f 100644 --- a/synapse/rest/client/v2_alpha/room_keys.py +++ b/synapse/rest/client/v2_alpha/room_keys.py @@ -68,6 +68,8 @@ class RoomKeysServlet(RestServlet): * lower forwarded_count always wins over higher forwarded_count We trust the clients not to lie and corrupt their own backups. + It also means that if your access_token is stolen, the attacker could + delete your backup. POST /room_keys/keys/!abc:matrix.org/c0ff33?version=1 HTTP/1.1 Content-Type: application/json diff --git a/synapse/storage/e2e_room_keys.py b/synapse/storage/e2e_room_keys.py index 8efca11a8c..c11417c415 100644 --- a/synapse/storage/e2e_room_keys.py +++ b/synapse/storage/e2e_room_keys.py @@ -44,30 +44,21 @@ class EndToEndRoomKeyStore(SQLBaseStore): def set_e2e_room_key(self, user_id, version, room_id, session_id, room_key): - def _set_e2e_room_key_txn(txn): - - self._simple_upsert_txn( - txn, - table="e2e_room_keys", - keyvalues={ - "user_id": user_id, - "room_id": room_id, - "session_id": session_id, - }, - values={ - "version": version, - "first_message_index": room_key['first_message_index'], - "forwarded_count": room_key['forwarded_count'], - "is_verified": room_key['is_verified'], - "session_data": room_key['session_data'], - }, - lock=False, - ) - - return True - - return self.runInteraction( - "set_e2e_room_key", _set_e2e_room_key_txn + yield self._simple_upsert( + table="e2e_room_keys", + keyvalues={ + "user_id": user_id, + "room_id": room_id, + "session_id": session_id, + }, + values={ + "version": version, + "first_message_index": room_key['first_message_index'], + "forwarded_count": room_key['forwarded_count'], + "is_verified": room_key['is_verified'], + "session_data": room_key['session_data'], + }, + lock=False, ) # XXX: this isn't currently used and isn't tested anywhere @@ -107,7 +98,9 @@ class EndToEndRoomKeyStore(SQLBaseStore): ) @defer.inlineCallbacks - def get_e2e_room_keys(self, user_id, version, room_id, session_id): + def get_e2e_room_keys( + self, user_id, version, room_id=room_id, session_id=session_id + ): keyvalues = { "user_id": user_id, @@ -115,8 +108,8 @@ class EndToEndRoomKeyStore(SQLBaseStore): } if room_id: keyvalues['room_id'] = room_id - if session_id: - keyvalues['session_id'] = session_id + if session_id: + keyvalues['session_id'] = session_id rows = yield self._simple_select_list( table="e2e_room_keys", @@ -133,18 +126,10 @@ class EndToEndRoomKeyStore(SQLBaseStore): desc="get_e2e_room_keys", ) - # perlesque autovivification from https://stackoverflow.com/a/19829714/6764493 - class AutoVivification(dict): - def __getitem__(self, item): - try: - return dict.__getitem__(self, item) - except KeyError: - value = self[item] = type(self)() - return value - - sessions = AutoVivification() + sessions = {} for row in rows: - sessions['rooms'][row['room_id']]['sessions'][row['session_id']] = { + room_entry = sessions['rooms'].setdefault(row['room_id'], {"sessions": {}}) + room_entry['sessions'][row['session_id']] = { "first_message_index": row["first_message_index"], "forwarded_count": row["forwarded_count"], "is_verified": row["is_verified"], @@ -154,7 +139,9 @@ class EndToEndRoomKeyStore(SQLBaseStore): defer.returnValue(sessions) @defer.inlineCallbacks - def delete_e2e_room_keys(self, user_id, version, room_id, session_id): + def delete_e2e_room_keys( + self, user_id, version, room_id=room_id, session_id=session_id + ): keyvalues = { "user_id": user_id, @@ -162,8 +149,8 @@ class EndToEndRoomKeyStore(SQLBaseStore): } if room_id: keyvalues['room_id'] = room_id - if session_id: - keyvalues['session_id'] = session_id + if session_id: + keyvalues['session_id'] = session_id yield self._simple_delete( table="e2e_room_keys", diff --git a/synapse/storage/schema/delta/46/e2e_room_keys.sql b/synapse/storage/schema/delta/46/e2e_room_keys.sql index 0d2a85fbe6..16499ac34c 100644 --- a/synapse/storage/schema/delta/46/e2e_room_keys.sql +++ b/synapse/storage/schema/delta/46/e2e_room_keys.sql @@ -25,16 +25,14 @@ CREATE TABLE e2e_room_keys ( session_data TEXT NOT NULL ); -CREATE UNIQUE INDEX e2e_room_keys_user_idx ON e2e_room_keys(user_id); -CREATE UNIQUE INDEX e2e_room_keys_room_idx ON e2e_room_keys(room_id); -CREATE UNIQUE INDEX e2e_room_keys_session_idx ON e2e_room_keys(session_id); +CREATE UNIQUE INDEX e2e_room_keys_idx ON e2e_room_keys(user_id, room_id, session_id); -- the metadata for each generation of encrypted e2e session backups -CREATE TABLE e2e_room_key_versions ( +CREATE TABLE e2e_room_keys_versions ( user_id TEXT NOT NULL, version TEXT NOT NULL, algorithm TEXT NOT NULL, auth_data TEXT NOT NULL ); -CREATE UNIQUE INDEX e2e_room_key_user_idx ON e2e_room_keys(user_id); +CREATE UNIQUE INDEX e2e_room_keys_versions_user_idx ON e2e_room_keys_versions(user_id); -- cgit 1.5.1 From 9f500cb39efa608c02f212711a5ad2177757bf4b Mon Sep 17 00:00:00 2001 From: Matthew Hodgson Date: Sun, 24 Dec 2017 17:42:17 +0000 Subject: more docstring for the e2e_room_keys rest --- synapse/handlers/e2e_room_keys.py | 2 -- synapse/rest/client/v2_alpha/room_keys.py | 51 +++++++++++++++++++++++++++++++ 2 files changed, 51 insertions(+), 2 deletions(-) (limited to 'synapse/rest/client/v2_alpha') diff --git a/synapse/handlers/e2e_room_keys.py b/synapse/handlers/e2e_room_keys.py index b67d6a2a7e..7a940d1c21 100644 --- a/synapse/handlers/e2e_room_keys.py +++ b/synapse/handlers/e2e_room_keys.py @@ -59,7 +59,6 @@ class E2eRoomKeysHandler(object): @defer.inlineCallbacks def upload_room_keys(self, user_id, version, room_keys): - # TODO: Validate the JSON to make sure it has the right keys. # XXX: perhaps we should use a finer grained lock here? @@ -139,7 +138,6 @@ class E2eRoomKeysHandler(object): @defer.inlineCallbacks def create_version(self, user_id, version_info): - # TODO: Validate the JSON to make sure it has the right keys. # lock everyone out until we've switched version diff --git a/synapse/rest/client/v2_alpha/room_keys.py b/synapse/rest/client/v2_alpha/room_keys.py index 70b7b4573f..04547c7d43 100644 --- a/synapse/rest/client/v2_alpha/room_keys.py +++ b/synapse/rest/client/v2_alpha/room_keys.py @@ -208,6 +208,10 @@ class RoomKeysServlet(RestServlet): """ Deletes one or more encrypted E2E room keys for a user for backup purposes. + DELETE /room_keys/keys/!abc:matrix.org/c0ff33?version=1 + HTTP/1.1 200 OK + {} + room_id: the ID of the room whose keys to delete (optional) session_id: the ID for the E2E session to delete (optional) version: the version of the user's backup which this data is for. @@ -240,6 +244,33 @@ class RoomKeysVersionServlet(RestServlet): @defer.inlineCallbacks def on_POST(self, request, version): + """ + Create a new backup version for this user's room_keys with the given + info. The version is allocated by the server and returned to the user + in the response. This API is intended to be used whenever the user + changes the encryption key for their backups, ensuring that backups + encrypted with different keys don't collide. + + The algorithm passed in the version info is a reverse-DNS namespaced + identifier to describe the format of the encrypted backupped keys. + + The auth_data is { user_id: "user_id", nonce: } + encrypted using the algorithm and current encryption key described above. + + POST /room_keys/version + Content-Type: application/json + { + "algorithm": "m.megolm_backup.v1", + "auth_data": "dGhpcyBzaG91bGQgYWN0dWFsbHkgYmUgZW5jcnlwdGVkIGpzb24K" + } + + HTTP/1.1 200 OK + Content-Type: application/json + { + "version": 12345 + } + """ + if version: raise SynapseError(405, "Cannot POST to a specific version") @@ -257,6 +288,17 @@ class RoomKeysVersionServlet(RestServlet): @defer.inlineCallbacks def on_GET(self, request, version): + """ + Retrieve the version information about a given version of the user's + room_keys backup. + + GET /room_keys/version/12345 HTTP/1.1 + { + "algorithm": "m.megolm_backup.v1", + "auth_data": "dGhpcyBzaG91bGQgYWN0dWFsbHkgYmUgZW5jcnlwdGVkIGpzb24K" + } + """ + requester = yield self.auth.get_user_by_req(request, allow_guest=False) user_id = requester.user.to_string() @@ -267,6 +309,15 @@ class RoomKeysVersionServlet(RestServlet): @defer.inlineCallbacks def on_DELETE(self, request, version): + """ + Delete the information about a given version of the user's + room_keys backup. Doesn't delete the actual room data. + + DELETE /room_keys/version/12345 HTTP/1.1 + HTTP/1.1 200 OK + {} + """ + requester = yield self.auth.get_user_by_req(request, allow_guest=False) user_id = requester.user.to_string() -- cgit 1.5.1 From 14b3da63a339333292a83410c0ba3148bcb644ba Mon Sep 17 00:00:00 2001 From: Matthew Hodgson Date: Wed, 27 Dec 2017 23:37:44 +0000 Subject: add a tonne of docstring; make upload_room_keys properly assert version --- synapse/handlers/e2e_room_keys.py | 111 +++++++++++++++++++++++++++--- synapse/rest/client/v2_alpha/room_keys.py | 11 ++- 2 files changed, 113 insertions(+), 9 deletions(-) (limited to 'synapse/rest/client/v2_alpha') diff --git a/synapse/handlers/e2e_room_keys.py b/synapse/handlers/e2e_room_keys.py index 7a940d1c21..2fa025bfc7 100644 --- a/synapse/handlers/e2e_room_keys.py +++ b/synapse/handlers/e2e_room_keys.py @@ -42,7 +42,16 @@ class E2eRoomKeysHandler(object): self._upload_linearizer = Linearizer("upload_room_keys_lock") @defer.inlineCallbacks - def get_room_keys(self, user_id, version, room_id, session_id): + def get_room_keys(self, user_id, version, room_id=None, session_id=None): + """Bulk get the E2E room keys for a given backup, optionally filtered to a given + room, or a given session. + See EndToEndRoomKeyStore.get_e2e_room_keys for full details. + + Returns: + A deferred list of dicts giving the session_data and message metadata for + these room keys. + """ + # we deliberately take the lock to get keys so that changing the version # works atomically with (yield self._upload_linearizer.queue(user_id)): @@ -52,20 +61,56 @@ class E2eRoomKeysHandler(object): defer.returnValue(results) @defer.inlineCallbacks - def delete_room_keys(self, user_id, version, room_id, session_id): + def delete_room_keys(self, user_id, version, room_id=None, session_id=None): + """Bulk delete the E2E room keys for a given backup, optionally filtered to a given + room or a given session. + See EndToEndRoomKeyStore.delete_e2e_room_keys for full details. + + Returns: + A deferred of the deletion transaction + """ + # lock for consistency with uploading with (yield self._upload_linearizer.queue(user_id)): yield self.store.delete_e2e_room_keys(user_id, version, room_id, session_id) @defer.inlineCallbacks def upload_room_keys(self, user_id, version, room_keys): + """Bulk upload a list of room keys into a given backup version, asserting + that the given version is the current backup version. room_keys are merged + into the current backup as described in RoomKeysServlet.on_PUT(). + + Args: + user_id(str): the user whose backup we're setting + version(str): the version ID of the backup we're updating + room_keys(dict): a nested dict describing the room_keys we're setting: + + { + "rooms": { + "!abc:matrix.org": { + "sessions": { + "c0ff33": { + "first_message_index": 1, + "forwarded_count": 1, + "is_verified": false, + "session_data": "SSBBTSBBIEZJU0gK" + } + } + } + } + } + + Raises: + SynapseError: with code 404 if there are no versions defined + RoomKeysVersionError: if the uploaded version is not the current version + """ + # TODO: Validate the JSON to make sure it has the right keys. # XXX: perhaps we should use a finer grained lock here? with (yield self._upload_linearizer.queue(user_id)): # Check that the version we're trying to upload is the current version - try: - version_info = yield self.get_version_info(user_id, version) + version_info = yield self.get_current_version_info(user_id) except StoreError as e: if e.code == 404: raise SynapseError(404, "Version '%s' not found" % (version,)) @@ -87,6 +132,17 @@ class E2eRoomKeysHandler(object): @defer.inlineCallbacks def _upload_room_key(self, user_id, version, room_id, session_id, room_key): + """Upload a given room_key for a given room and session into a given + version of the backup. Merges the key with any which might already exist. + + Args: + user_id(str): the user whose backup we're setting + version(str): the version ID of the backup we're updating + room_id(str): the ID of the room whose keys we're setting + session_id(str): the session whose room_key we're setting + room_key(dict): the room_key being set + """ + # get the room_key for this particular row current_room_key = None try: @@ -138,6 +194,23 @@ class E2eRoomKeysHandler(object): @defer.inlineCallbacks def create_version(self, user_id, version_info): + """Create a new backup version. This automatically becomes the new + backup version for the user's keys; previous backups will no longer be + writeable to. + + Args: + user_id(str): the user whose backup version we're creating + version_info(dict): metadata about the new version being created + + { + "algorithm": "m.megolm_backup.v1", + "auth_data": "dGhpcyBzaG91bGQgYWN0dWFsbHkgYmUgZW5jcnlwdGVkIGpzb24K" + } + + Returns: + A deferred of a string that gives the new version number. + """ + # TODO: Validate the JSON to make sure it has the right keys. # lock everyone out until we've switched version @@ -148,14 +221,36 @@ class E2eRoomKeysHandler(object): defer.returnValue(new_version) @defer.inlineCallbacks - def get_version_info(self, user_id, version): + def get_current_version_info(self, user_id): + """Get the user's current backup version. + + Args: + user_id(str): the user whose current backup version we're querying + Raises: + StoreError: code 404 if there is no current backup version + Returns: + A deferred of a info dict that gives the info about the new version. + + { + "algorithm": "m.megolm_backup.v1", + "auth_data": "dGhpcyBzaG91bGQgYWN0dWFsbHkgYmUgZW5jcnlwdGVkIGpzb24K" + } + """ + with (yield self._upload_linearizer.queue(user_id)): - results = yield self.store.get_e2e_room_keys_version_info( - user_id, version - ) + results = yield self.store.get_e2e_room_keys_version_info(user_id) defer.returnValue(results) @defer.inlineCallbacks def delete_version(self, user_id, version): + """Deletes a given version of the user's e2e_room_keys backup + + Args: + user_id(str): the user whose current backup version we're deleting + version(str): the version id of the backup being deleted + Raises: + StoreError: code 404 if this backup version doesn't exist + """ + with (yield self._upload_linearizer.queue(user_id)): yield self.store.delete_e2e_room_keys_version(user_id, version) diff --git a/synapse/rest/client/v2_alpha/room_keys.py b/synapse/rest/client/v2_alpha/room_keys.py index 04547c7d43..d3f857aba2 100644 --- a/synapse/rest/client/v2_alpha/room_keys.py +++ b/synapse/rest/client/v2_alpha/room_keys.py @@ -47,7 +47,7 @@ class RoomKeysServlet(RestServlet): room_id: the ID of the room the keys are for (optional) session_id: the ID for the E2E room keys for the room (optional) version: the version of the user's backup which this data is for. - the version must already have been created via the /change_secret API. + the version must already have been created via the /room_keys/version API. Each session has: * first_message_index: a numeric index indicating the oldest message @@ -59,6 +59,9 @@ class RoomKeysServlet(RestServlet): * session_data: base64-encrypted data describing the session. Returns 200 OK on success with body {} + Returns 403 Forbidden if the version in question is not the most recently + created version (i.e. if this is an old client trying to write to a stale backup) + Returns 404 Not Found if the version in question doesn't exist The API is designed to be otherwise agnostic to the room_key encryption algorithm being used. Sessions are merged with existing ones in the @@ -251,6 +254,9 @@ class RoomKeysVersionServlet(RestServlet): changes the encryption key for their backups, ensuring that backups encrypted with different keys don't collide. + It takes out an exclusive lock on this user's room_key backups, to ensure + clients only upload to the current backup. + The algorithm passed in the version info is a reverse-DNS namespaced identifier to describe the format of the encrypted backupped keys. @@ -292,6 +298,9 @@ class RoomKeysVersionServlet(RestServlet): Retrieve the version information about a given version of the user's room_keys backup. + It takes out an exclusive lock on this user's room_key backups, to ensure + clients only upload to the current backup. + GET /room_keys/version/12345 HTTP/1.1 { "algorithm": "m.megolm_backup.v1", -- cgit 1.5.1 From 93d174bcc4cf773bea2534d3683a1e91e5489c89 Mon Sep 17 00:00:00 2001 From: Matthew Hodgson Date: Sun, 31 Dec 2017 14:10:49 +0000 Subject: improve docstring --- synapse/rest/client/v2_alpha/room_keys.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) (limited to 'synapse/rest/client/v2_alpha') diff --git a/synapse/rest/client/v2_alpha/room_keys.py b/synapse/rest/client/v2_alpha/room_keys.py index d3f857aba2..ca69ced1e3 100644 --- a/synapse/rest/client/v2_alpha/room_keys.py +++ b/synapse/rest/client/v2_alpha/room_keys.py @@ -296,13 +296,17 @@ class RoomKeysVersionServlet(RestServlet): def on_GET(self, request, version): """ Retrieve the version information about a given version of the user's - room_keys backup. + room_keys backup. If the version part is missing, returns info about the + most current backup version (if any) It takes out an exclusive lock on this user's room_key backups, to ensure clients only upload to the current backup. + Returns 404 is the given version does not exist. + GET /room_keys/version/12345 HTTP/1.1 { + "version": "12345", "algorithm": "m.megolm_backup.v1", "auth_data": "dGhpcyBzaG91bGQgYWN0dWFsbHkgYmUgZW5jcnlwdGVkIGpzb24K" } -- cgit 1.5.1 From fe87890b18f57f0268bd65aeca881e7817bbe9e4 Mon Sep 17 00:00:00 2001 From: Matthew Hodgson Date: Sun, 31 Dec 2017 17:47:11 +0000 Subject: implement remaining tests and make them work --- synapse/handlers/e2e_room_keys.py | 35 +++- synapse/rest/client/v2_alpha/room_keys.py | 6 + synapse/storage/e2e_room_keys.py | 3 +- tests/handlers/test_e2e_room_keys.py | 276 +++++++++++++++++++++++++++--- 4 files changed, 287 insertions(+), 33 deletions(-) (limited to 'synapse/rest/client/v2_alpha') diff --git a/synapse/handlers/e2e_room_keys.py b/synapse/handlers/e2e_room_keys.py index f08d80da3e..09c2888db6 100644 --- a/synapse/handlers/e2e_room_keys.py +++ b/synapse/handlers/e2e_room_keys.py @@ -58,6 +58,10 @@ class E2eRoomKeysHandler(object): results = yield self.store.get_e2e_room_keys( user_id, version, room_id, session_id ) + + if results['rooms'] == {}: + raise SynapseError(404, "No room_keys found") + defer.returnValue(results) @defer.inlineCallbacks @@ -109,9 +113,10 @@ class E2eRoomKeysHandler(object): # XXX: perhaps we should use a finer grained lock here? with (yield self._upload_linearizer.queue(user_id)): + # Check that the version we're trying to upload is the current version try: - version_info = yield self.get_version_info(user_id) + version_info = yield self._get_version_info_unlocked(user_id) except StoreError as e: if e.code == 404: raise SynapseError(404, "Version '%s' not found" % (version,)) @@ -119,16 +124,23 @@ class E2eRoomKeysHandler(object): raise e if version_info['version'] != version: - raise RoomKeysVersionError(current_version=version_info.version) + # Check that the version we're trying to upload actually exists + try: + version_info = yield self._get_version_info_unlocked(user_id, version) + # if we get this far, the version must exist + raise RoomKeysVersionError(current_version=version_info['version']) + except StoreError as e: + if e.code == 404: + raise SynapseError(404, "Version '%s' not found" % (version,)) + else: + raise e # go through the room_keys. # XXX: this should/could be done concurrently, given we're in a lock. for room_id, room in room_keys['rooms'].iteritems(): for session_id, session in room['sessions'].iteritems(): - room_key = session[session_id] - yield self._upload_room_key( - user_id, version, room_id, session_id, room_key + user_id, version, room_id, session_id, session ) @defer.inlineCallbacks @@ -242,8 +254,17 @@ class E2eRoomKeysHandler(object): """ with (yield self._upload_linearizer.queue(user_id)): - results = yield self.store.get_e2e_room_keys_version_info(user_id) - defer.returnValue(results) + res = yield self._get_version_info_unlocked(user_id, version) + defer.returnValue(res) + + @defer.inlineCallbacks + def _get_version_info_unlocked(self, user_id, version=None): + """Get the info about a given version of the user's backup + without obtaining the upload_linearizer lock. For params see get_version_info + """ + + results = yield self.store.get_e2e_room_keys_version_info(user_id, version) + defer.returnValue(results) @defer.inlineCallbacks def delete_version(self, user_id, version): diff --git a/synapse/rest/client/v2_alpha/room_keys.py b/synapse/rest/client/v2_alpha/room_keys.py index ca69ced1e3..8f10e4e1cd 100644 --- a/synapse/rest/client/v2_alpha/room_keys.py +++ b/synapse/rest/client/v2_alpha/room_keys.py @@ -204,6 +204,12 @@ class RoomKeysServlet(RestServlet): room_keys = yield self.e2e_room_keys_handler.get_room_keys( user_id, version, room_id, session_id ) + + if session_id: + room_keys = room_keys['rooms'][room_id]['sessions'][session_id] + elif room_id: + room_keys = room_keys['rooms'][room_id] + defer.returnValue((200, room_keys)) @defer.inlineCallbacks diff --git a/synapse/storage/e2e_room_keys.py b/synapse/storage/e2e_room_keys.py index e4d56b7c37..8e8e4e457c 100644 --- a/synapse/storage/e2e_room_keys.py +++ b/synapse/storage/e2e_room_keys.py @@ -58,6 +58,7 @@ class EndToEndRoomKeyStore(SQLBaseStore): defer.returnValue(row) + @defer.inlineCallbacks def set_e2e_room_key(self, user_id, version, room_id, session_id, room_key): """Replaces or inserts the encrypted E2E room key for a given session in a given backup @@ -135,7 +136,7 @@ class EndToEndRoomKeyStore(SQLBaseStore): desc="get_e2e_room_keys", ) - sessions = {} + sessions = { 'rooms': {} } for row in rows: room_entry = sessions['rooms'].setdefault(row['room_id'], {"sessions": {}}) room_entry['sessions'][row['session_id']] = { diff --git a/tests/handlers/test_e2e_room_keys.py b/tests/handlers/test_e2e_room_keys.py index afe6ecf27b..3cbfd6f9d0 100644 --- a/tests/handlers/test_e2e_room_keys.py +++ b/tests/handlers/test_e2e_room_keys.py @@ -17,6 +17,7 @@ import mock from synapse.api import errors from twisted.internet import defer +import copy import synapse.api.errors import synapse.handlers.e2e_room_keys @@ -25,6 +26,22 @@ import synapse.storage from tests import unittest, utils +# sample room_key data for use in the tests +room_keys = { + "rooms": { + "!abc:matrix.org": { + "sessions": { + "c0ff33": { + "first_message_index": 1, + "forwarded_count": 1, + "is_verified": False, + "session_data": "SSBBTSBBIEZJU0gK" + } + } + } + } +} + class E2eRoomKeysHandlerTestCase(unittest.TestCase): def __init__(self, *args, **kwargs): super(E2eRoomKeysHandlerTestCase, self).__init__(*args, **kwargs) @@ -38,46 +55,44 @@ class E2eRoomKeysHandlerTestCase(unittest.TestCase): replication_layer=mock.Mock(), ) self.handler = synapse.handlers.e2e_room_keys.E2eRoomKeysHandler(self.hs) + self.local_user = "@boris:" + self.hs.hostname; @defer.inlineCallbacks def test_get_missing_current_version_info(self): """Check that we get a 404 if we ask for info about the current version if there is no version. """ - local_user = "@boris:" + self.hs.hostname res = None try: - res = yield self.handler.get_version_info(local_user) + yield self.handler.get_version_info(self.local_user) except errors.SynapseError as e: - self.assertEqual(e.code, 404) - self.assertEqual(res, None) + res = e.code + self.assertEqual(res, 404) @defer.inlineCallbacks def test_get_missing_version_info(self): """Check that we get a 404 if we ask for info about a specific version if it doesn't exist. """ - local_user = "@boris:" + self.hs.hostname res = None try: - res = yield self.handler.get_version_info(local_user, "mrflibble") + yield self.handler.get_version_info(self.local_user, "bogus_version") except errors.SynapseError as e: - self.assertEqual(e.code, 404) - self.assertEqual(res, None) + res = e.code + self.assertEqual(res, 404) @defer.inlineCallbacks def test_create_version(self): """Check that we can create and then retrieve versions. """ - local_user = "@boris:" + self.hs.hostname - res = yield self.handler.create_version(local_user, { + res = yield self.handler.create_version(self.local_user, { "algorithm": "m.megolm_backup.v1", "auth_data": "first_version_auth_data", }) self.assertEqual(res, "1") # check we can retrieve it as the current version - res = yield self.handler.get_version_info(local_user) + res = yield self.handler.get_version_info(self.local_user) self.assertDictEqual(res, { "version": "1", "algorithm": "m.megolm_backup.v1", @@ -85,7 +100,7 @@ class E2eRoomKeysHandlerTestCase(unittest.TestCase): }) # check we can retrieve it as a specific version - res = yield self.handler.get_version_info(local_user, "1") + res = yield self.handler.get_version_info(self.local_user, "1") self.assertDictEqual(res, { "version": "1", "algorithm": "m.megolm_backup.v1", @@ -93,14 +108,14 @@ class E2eRoomKeysHandlerTestCase(unittest.TestCase): }) # upload a new one... - res = yield self.handler.create_version(local_user, { + res = yield self.handler.create_version(self.local_user, { "algorithm": "m.megolm_backup.v1", "auth_data": "second_version_auth_data", }) self.assertEqual(res, "2") # check we can retrieve it as the current version - res = yield self.handler.get_version_info(local_user) + res = yield self.handler.get_version_info(self.local_user) self.assertDictEqual(res, { "version": "2", "algorithm": "m.megolm_backup.v1", @@ -111,32 +126,243 @@ class E2eRoomKeysHandlerTestCase(unittest.TestCase): def test_delete_version(self): """Check that we can create and then delete versions. """ - local_user = "@boris:" + self.hs.hostname - res = yield self.handler.create_version(local_user, { + res = yield self.handler.create_version(self.local_user, { "algorithm": "m.megolm_backup.v1", "auth_data": "first_version_auth_data", }) self.assertEqual(res, "1") # check we can delete it - yield self.handler.delete_version(local_user, "1") + yield self.handler.delete_version(self.local_user, "1") # check that it's gone res = None try: - res = yield self.handler.get_version_info(local_user, "1") + yield self.handler.get_version_info(self.local_user, "1") except errors.SynapseError as e: - self.assertEqual(e.code, 404) - self.assertEqual(res, None) + res = e.code + self.assertEqual(res, 404) @defer.inlineCallbacks - def test_get_room_keys(self): - yield None + def test_get_missing_room_keys(self): + """Check that we get a 404 on querying missing room_keys + """ + res = None + try: + yield self.handler.get_room_keys(self.local_user, "bogus_version") + except errors.SynapseError as e: + res = e.code + self.assertEqual(res, 404) + + # check we also get a 404 even if the version is valid + version = yield self.handler.create_version(self.local_user, { + "algorithm": "m.megolm_backup.v1", + "auth_data": "first_version_auth_data", + }) + self.assertEqual(version, "1") + + res = None + try: + yield self.handler.get_room_keys(self.local_user, version) + except errors.SynapseError as e: + res = e.code + self.assertEqual(res, 404) + + # TODO: test the locking semantics when uploading room_keys, + # although this is probably best done in sytest @defer.inlineCallbacks - def test_upload_room_keys(self): - yield None + def test_upload_room_keys_no_versions(self): + """Check that we get a 404 on uploading keys when no versions are defined + """ + res = None + try: + yield self.handler.upload_room_keys(self.local_user, "no_version", room_keys) + except errors.SynapseError as e: + res = e.code + self.assertEqual(res, 404) + + @defer.inlineCallbacks + def test_upload_room_keys_bogus_version(self): + """Check that we get a 404 on uploading keys when an nonexistent version is specified + """ + version = yield self.handler.create_version(self.local_user, { + "algorithm": "m.megolm_backup.v1", + "auth_data": "first_version_auth_data", + }) + self.assertEqual(version, "1") + + res = None + try: + yield self.handler.upload_room_keys(self.local_user, "bogus_version", room_keys) + except errors.SynapseError as e: + res = e.code + self.assertEqual(res, 404) + + @defer.inlineCallbacks + def test_upload_room_keys_wrong_version(self): + """Check that we get a 403 on uploading keys for an old version + """ + version = yield self.handler.create_version(self.local_user, { + "algorithm": "m.megolm_backup.v1", + "auth_data": "first_version_auth_data", + }) + self.assertEqual(version, "1") + + version = yield self.handler.create_version(self.local_user, { + "algorithm": "m.megolm_backup.v1", + "auth_data": "second_version_auth_data", + }) + self.assertEqual(version, "2") + + res = None + try: + yield self.handler.upload_room_keys(self.local_user, "1", room_keys) + except errors.SynapseError as e: + res = e.code + self.assertEqual(res, 403) + + @defer.inlineCallbacks + def test_upload_room_keys_insert(self): + """Check that we can insert and retrieve keys for a session + """ + version = yield self.handler.create_version(self.local_user, { + "algorithm": "m.megolm_backup.v1", + "auth_data": "first_version_auth_data", + }) + self.assertEqual(version, "1") + + yield self.handler.upload_room_keys(self.local_user, version, room_keys) + + res = yield self.handler.get_room_keys(self.local_user, version) + self.assertDictEqual(res, room_keys) + + # check getting room_keys for a given room + res = yield self.handler.get_room_keys( + self.local_user, + version, + room_id="!abc:matrix.org" + ) + self.assertDictEqual(res, room_keys) + + # check getting room_keys for a given session_id + res = yield self.handler.get_room_keys( + self.local_user, + version, + room_id="!abc:matrix.org", + session_id="c0ff33", + ) + self.assertDictEqual(res, room_keys) + + @defer.inlineCallbacks + def test_upload_room_keys_merge(self): + """Check that we can upload a new room_key for an existing session and + have it correctly merged""" + version = yield self.handler.create_version(self.local_user, { + "algorithm": "m.megolm_backup.v1", + "auth_data": "first_version_auth_data", + }) + self.assertEqual(version, "1") + + yield self.handler.upload_room_keys(self.local_user, version, room_keys) + + new_room_keys = copy.deepcopy(room_keys) + + # test that increasing the message_index doesn't replace the existing session + new_room_keys['rooms']['!abc:matrix.org']['sessions']['c0ff33']['first_message_index'] = 2 + new_room_keys['rooms']['!abc:matrix.org']['sessions']['c0ff33']['session_data'] = 'new' + yield self.handler.upload_room_keys(self.local_user, version, new_room_keys) + + res = yield self.handler.get_room_keys(self.local_user, version) + self.assertEqual( + res['rooms']['!abc:matrix.org']['sessions']['c0ff33']['session_data'], + "SSBBTSBBIEZJU0gK" + ) + + # test that marking the session as verified however /does/ replace it + new_room_keys['rooms']['!abc:matrix.org']['sessions']['c0ff33']['is_verified'] = True + yield self.handler.upload_room_keys(self.local_user, version, new_room_keys) + + res = yield self.handler.get_room_keys(self.local_user, version) + self.assertEqual( + res['rooms']['!abc:matrix.org']['sessions']['c0ff33']['session_data'], + "new" + ) + + # test that a session with a higher forwarded_count doesn't replace one + # with a lower forwarding count + new_room_keys['rooms']['!abc:matrix.org']['sessions']['c0ff33']['forwarded_count'] = 2 + new_room_keys['rooms']['!abc:matrix.org']['sessions']['c0ff33']['session_data'] = 'other' + yield self.handler.upload_room_keys(self.local_user, version, new_room_keys) + + res = yield self.handler.get_room_keys(self.local_user, version) + self.assertEqual( + res['rooms']['!abc:matrix.org']['sessions']['c0ff33']['session_data'], + "new" + ) + + # TODO: check edge cases as well as the common variations here @defer.inlineCallbacks def test_delete_room_keys(self): - yield None + """Check that we can insert and delete keys for a session + """ + version = yield self.handler.create_version(self.local_user, { + "algorithm": "m.megolm_backup.v1", + "auth_data": "first_version_auth_data", + }) + self.assertEqual(version, "1") + + # check for bulk-delete + yield self.handler.upload_room_keys(self.local_user, version, room_keys) + yield self.handler.delete_room_keys(self.local_user, version) + res = None + try: + yield self.handler.get_room_keys( + self.local_user, + version, + room_id="!abc:matrix.org", + session_id="c0ff33", + ) + except errors.SynapseError as e: + res = e.code + self.assertEqual(res, 404) + + # check for bulk-delete per room + yield self.handler.upload_room_keys(self.local_user, version, room_keys) + yield self.handler.delete_room_keys( + self.local_user, + version, + room_id="!abc:matrix.org", + ) + res = None + try: + yield self.handler.get_room_keys( + self.local_user, + version, + room_id="!abc:matrix.org", + session_id="c0ff33", + ) + except errors.SynapseError as e: + res = e.code + self.assertEqual(res, 404) + + # check for bulk-delete per session + yield self.handler.upload_room_keys(self.local_user, version, room_keys) + yield self.handler.delete_room_keys( + self.local_user, + version, + room_id="!abc:matrix.org", + session_id="c0ff33", + ) + res = None + try: + yield self.handler.get_room_keys( + self.local_user, + version, + room_id="!abc:matrix.org", + session_id="c0ff33", + ) + except errors.SynapseError as e: + res = e.code + self.assertEqual(res, 404) -- cgit 1.5.1 From 72788cf9c1f018a5346a8f9204c6bfe058289fd2 Mon Sep 17 00:00:00 2001 From: Matthew Hodgson Date: Sun, 7 Jan 2018 23:45:26 +0000 Subject: support DELETE /version with no args --- synapse/handlers/e2e_room_keys.py | 2 +- synapse/rest/client/v2_alpha/room_keys.py | 5 +++-- 2 files changed, 4 insertions(+), 3 deletions(-) (limited to 'synapse/rest/client/v2_alpha') diff --git a/synapse/handlers/e2e_room_keys.py b/synapse/handlers/e2e_room_keys.py index 09c2888db6..a43fc7fc7e 100644 --- a/synapse/handlers/e2e_room_keys.py +++ b/synapse/handlers/e2e_room_keys.py @@ -267,7 +267,7 @@ class E2eRoomKeysHandler(object): defer.returnValue(results) @defer.inlineCallbacks - def delete_version(self, user_id, version): + def delete_version(self, user_id, version=None): """Deletes a given version of the user's e2e_room_keys backup Args: diff --git a/synapse/rest/client/v2_alpha/room_keys.py b/synapse/rest/client/v2_alpha/room_keys.py index 8f10e4e1cd..63b1f62f90 100644 --- a/synapse/rest/client/v2_alpha/room_keys.py +++ b/synapse/rest/client/v2_alpha/room_keys.py @@ -308,7 +308,7 @@ class RoomKeysVersionServlet(RestServlet): It takes out an exclusive lock on this user's room_key backups, to ensure clients only upload to the current backup. - Returns 404 is the given version does not exist. + Returns 404 if the given version does not exist. GET /room_keys/version/12345 HTTP/1.1 { @@ -330,7 +330,8 @@ class RoomKeysVersionServlet(RestServlet): def on_DELETE(self, request, version): """ Delete the information about a given version of the user's - room_keys backup. Doesn't delete the actual room data. + room_keys backup. If the version part is missing, deletes the most + current backup version (if any). Doesn't delete the actual room data. DELETE /room_keys/version/12345 HTTP/1.1 HTTP/1.1 200 OK -- cgit 1.5.1 From 54ac18e8327e689b9f91b93ad8caac2f6b3dd29c Mon Sep 17 00:00:00 2001 From: Matthew Hodgson Date: Sun, 7 Jan 2018 23:48:22 +0000 Subject: use parse_string --- synapse/rest/client/v2_alpha/room_keys.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) (limited to 'synapse/rest/client/v2_alpha') diff --git a/synapse/rest/client/v2_alpha/room_keys.py b/synapse/rest/client/v2_alpha/room_keys.py index 63b1f62f90..afb8a36dd0 100644 --- a/synapse/rest/client/v2_alpha/room_keys.py +++ b/synapse/rest/client/v2_alpha/room_keys.py @@ -123,7 +123,7 @@ class RoomKeysServlet(RestServlet): requester = yield self.auth.get_user_by_req(request, allow_guest=False) user_id = requester.user.to_string() body = parse_json_object_from_request(request) - version = request.args.get("version")[0] + version = parse_string(request, "version") if session_id: body = { @@ -199,7 +199,7 @@ class RoomKeysServlet(RestServlet): """ requester = yield self.auth.get_user_by_req(request, allow_guest=False) user_id = requester.user.to_string() - version = request.args.get("version")[0] + version = parse_string(request, "version") room_keys = yield self.e2e_room_keys_handler.get_room_keys( user_id, version, room_id, session_id @@ -229,7 +229,7 @@ class RoomKeysServlet(RestServlet): requester = yield self.auth.get_user_by_req(request, allow_guest=False) user_id = requester.user.to_string() - version = request.args.get("version")[0] + version = parse_string(request, "version") yield self.e2e_room_keys_handler.delete_room_keys( user_id, version, room_id, session_id -- cgit 1.5.1 From 4f7064f6b5a1822067bfbf358317bc1dbb51b9c6 Mon Sep 17 00:00:00 2001 From: Matthew Hodgson Date: Sun, 7 Jan 2018 23:59:07 +0000 Subject: missing import --- synapse/rest/client/v2_alpha/room_keys.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'synapse/rest/client/v2_alpha') diff --git a/synapse/rest/client/v2_alpha/room_keys.py b/synapse/rest/client/v2_alpha/room_keys.py index afb8a36dd0..9f0172e6f5 100644 --- a/synapse/rest/client/v2_alpha/room_keys.py +++ b/synapse/rest/client/v2_alpha/room_keys.py @@ -19,7 +19,7 @@ from twisted.internet import defer from synapse.api.errors import SynapseError from synapse.http.servlet import ( - RestServlet, parse_json_object_from_request + RestServlet, parse_json_object_from_request, parse_string ) from ._base import client_v2_patterns -- cgit 1.5.1 From ea068d6f3cd5ed1bc9a39b2fd43e19d6d40f18da Mon Sep 17 00:00:00 2001 From: Neil Johnson Date: Fri, 31 Aug 2018 10:49:14 +0100 Subject: fix bug where preserved threepid user comes to sign up and server is mau blocked --- synapse/api/auth.py | 10 +++++++++- synapse/handlers/register.py | 3 ++- synapse/rest/client/v1_only/register.py | 6 +++++- synapse/rest/client/v2_alpha/register.py | 5 +++++ tests/api/test_auth.py | 17 +++++++++++++++++ 5 files changed, 38 insertions(+), 3 deletions(-) (limited to 'synapse/rest/client/v2_alpha') diff --git a/synapse/api/auth.py b/synapse/api/auth.py index a7e3f7a7ac..9c207b9537 100644 --- a/synapse/api/auth.py +++ b/synapse/api/auth.py @@ -775,7 +775,7 @@ class Auth(object): ) @defer.inlineCallbacks - def check_auth_blocking(self, user_id=None): + def check_auth_blocking(self, user_id=None, threepid=None): """Checks if the user should be rejected for some external reason, such as monthly active user limiting or global disable flag @@ -806,6 +806,14 @@ class Auth(object): is_trial = yield self.store.is_trial_user(user_id) if is_trial: return + elif threepid: + # If the user does not exist yet, but is signing up with a + # reserved threepid then pass auth check + for tp in self.hs.config.mau_limits_reserved_threepids: + if (threepid['medium'] == tp['medium'] + and threepid['address'] == tp['address']): + return + # Else if there is no room in the MAU bucket, bail current_mau = yield self.store.get_monthly_active_count() if current_mau >= self.hs.config.max_mau_value: diff --git a/synapse/handlers/register.py b/synapse/handlers/register.py index f03ee1476b..1e53f2c635 100644 --- a/synapse/handlers/register.py +++ b/synapse/handlers/register.py @@ -125,6 +125,7 @@ class RegistrationHandler(BaseHandler): guest_access_token=None, make_guest=False, admin=False, + threepid=None, ): """Registers a new client on the server. @@ -145,7 +146,7 @@ class RegistrationHandler(BaseHandler): RegistrationError if there was a problem registering. """ - yield self.auth.check_auth_blocking() + yield self.auth.check_auth_blocking(threepid=threepid) password_hash = None if password: password_hash = yield self.auth_handler().hash(password) diff --git a/synapse/rest/client/v1_only/register.py b/synapse/rest/client/v1_only/register.py index 5e99cffbcb..2c7bbcb171 100644 --- a/synapse/rest/client/v1_only/register.py +++ b/synapse/rest/client/v1_only/register.py @@ -281,11 +281,15 @@ class RegisterRestServlet(ClientV1RestServlet): register_json["user"].encode("utf-8") if "user" in register_json else None ) + threepid = None + if session[LoginType.EMAIL_IDENTITY]: + threepid = session["threepidCreds"] handler = self.handlers.registration_handler (user_id, token) = yield handler.register( localpart=desired_user_id, - password=password + password=password, + threepid=threepid, ) if session[LoginType.EMAIL_IDENTITY]: diff --git a/synapse/rest/client/v2_alpha/register.py b/synapse/rest/client/v2_alpha/register.py index 2f64155d13..45113e5386 100644 --- a/synapse/rest/client/v2_alpha/register.py +++ b/synapse/rest/client/v2_alpha/register.py @@ -395,11 +395,16 @@ class RegisterRestServlet(RestServlet): if desired_username is not None: desired_username = desired_username.lower() + threepid = None + if auth_result: + threepid = auth_result.get(LoginType.EMAIL_IDENTITY) + (registered_user_id, _) = yield self.registration_handler.register( localpart=desired_username, password=new_password, guest_access_token=guest_access_token, generate_token=False, + threepid=threepid, ) # remember that we've now registered that user account, and with diff --git a/tests/api/test_auth.py b/tests/api/test_auth.py index 54e396d19d..f65a27e5f1 100644 --- a/tests/api/test_auth.py +++ b/tests/api/test_auth.py @@ -467,6 +467,23 @@ class AuthTestCase(unittest.TestCase): ) yield self.auth.check_auth_blocking() + @defer.inlineCallbacks + def test_reserved_threepid(self): + self.hs.config.limit_usage_by_mau = True + self.hs.config.max_mau_value = 1 + threepid = {'medium': 'email', 'address': 'reserved@server.com'} + unknown_threepid = {'medium': 'email', 'address': 'unreserved@server.com'} + self.hs.config.mau_limits_reserved_threepids = [threepid] + + yield self.store.register(user_id='user1', token="123", password_hash=None) + with self.assertRaises(ResourceLimitError): + yield self.auth.check_auth_blocking() + + with self.assertRaises(ResourceLimitError): + yield self.auth.check_auth_blocking(threepid=unknown_threepid) + + yield self.auth.check_auth_blocking(threepid=threepid) + @defer.inlineCallbacks def test_hs_disabled(self): self.hs.config.hs_disabled = True -- cgit 1.5.1 From 09f3cf1a7ef0c533d052a5c87257503b710093c6 Mon Sep 17 00:00:00 2001 From: Neil Johnson Date: Fri, 31 Aug 2018 15:42:51 +0100 Subject: ensure post registration auth checks do not fail erroneously --- synapse/api/auth.py | 7 ++----- synapse/rest/client/v1_only/register.py | 4 ++++ synapse/rest/client/v2_alpha/register.py | 4 ++++ synapse/storage/monthly_active_users.py | 15 ++++++++++++++- 4 files changed, 24 insertions(+), 6 deletions(-) (limited to 'synapse/rest/client/v2_alpha') diff --git a/synapse/api/auth.py b/synapse/api/auth.py index 9c207b9537..6a97c06110 100644 --- a/synapse/api/auth.py +++ b/synapse/api/auth.py @@ -809,11 +809,8 @@ class Auth(object): elif threepid: # If the user does not exist yet, but is signing up with a # reserved threepid then pass auth check - for tp in self.hs.config.mau_limits_reserved_threepids: - if (threepid['medium'] == tp['medium'] - and threepid['address'] == tp['address']): - return - + if is_threepid_reserved(threepid): + return # Else if there is no room in the MAU bucket, bail current_mau = yield self.store.get_monthly_active_count() if current_mau >= self.hs.config.max_mau_value: diff --git a/synapse/rest/client/v1_only/register.py b/synapse/rest/client/v1_only/register.py index 2c7bbcb171..95873e03d5 100644 --- a/synapse/rest/client/v1_only/register.py +++ b/synapse/rest/client/v1_only/register.py @@ -291,6 +291,10 @@ class RegisterRestServlet(ClientV1RestServlet): password=password, threepid=threepid, ) + # Necessary due to auth checks prior to the threepid being + # written to the db + if self.store.is_threepid_reserved(threepid): + self.store.upsert_monthly_active_user(registered_user_id) if session[LoginType.EMAIL_IDENTITY]: logger.debug("Binding emails %s to %s" % ( diff --git a/synapse/rest/client/v2_alpha/register.py b/synapse/rest/client/v2_alpha/register.py index 45113e5386..f22b7577ea 100644 --- a/synapse/rest/client/v2_alpha/register.py +++ b/synapse/rest/client/v2_alpha/register.py @@ -406,6 +406,10 @@ class RegisterRestServlet(RestServlet): generate_token=False, threepid=threepid, ) + # Necessary due to auth checks prior to the threepid being + # written to the db + if self.store.is_threepid_reserved(threepid): + self.store.upsert_monthly_active_user(registered_user_id) # remember that we've now registered that user account, and with # what user ID (since the user may not have specified) diff --git a/synapse/storage/monthly_active_users.py b/synapse/storage/monthly_active_users.py index d178f5c5ba..173867c4b1 100644 --- a/synapse/storage/monthly_active_users.py +++ b/synapse/storage/monthly_active_users.py @@ -36,7 +36,6 @@ class MonthlyActiveUsersStore(SQLBaseStore): @defer.inlineCallbacks def initialise_reserved_users(self, threepids): - # TODO Why can't I do this in init? store = self.hs.get_datastore() reserved_user_list = [] @@ -220,3 +219,17 @@ class MonthlyActiveUsersStore(SQLBaseStore): yield self.upsert_monthly_active_user(user_id) elif now - last_seen_timestamp > LAST_SEEN_GRANULARITY: yield self.upsert_monthly_active_user(user_id) + + def is_threepid_reserved(self, threepid): + """Check the threepid against the reserved threepid config + Args: + threepid(dict) - The threepid to test for + Returns: + boolean Is the threepid undertest reserved_user + """ + for tp in self.hs.config.mau_limits_reserved_threepids: + if (threepid['medium'] == tp['medium'] + and threepid['address'] == tp['address']): + return True + else: + return False -- cgit 1.5.1 From 0b01281e77aee7e69925f36dbb6a798772a98a45 Mon Sep 17 00:00:00 2001 From: Neil Johnson Date: Fri, 31 Aug 2018 17:11:11 +0100 Subject: move threepid checker to config, add missing yields --- synapse/api/auth.py | 13 +++++++++++-- synapse/config/server.py | 17 +++++++++++++++++ synapse/rest/client/v1_only/register.py | 7 ++++--- synapse/rest/client/v2_alpha/register.py | 5 +++-- synapse/storage/monthly_active_users.py | 14 -------------- tests/utils.py | 6 ++++++ 6 files changed, 41 insertions(+), 21 deletions(-) (limited to 'synapse/rest/client/v2_alpha') diff --git a/synapse/api/auth.py b/synapse/api/auth.py index a36fb6b3bd..a89687f420 100644 --- a/synapse/api/auth.py +++ b/synapse/api/auth.py @@ -26,6 +26,7 @@ import synapse.types from synapse import event_auth from synapse.api.constants import EventTypes, JoinRules, Membership from synapse.api.errors import AuthError, Codes, ResourceLimitError +from synapse.config.server import is_threepid_reserved from synapse.types import UserID from synapse.util.caches import CACHE_SIZE_FACTOR, register_cache from synapse.util.caches.lrucache import LruCache @@ -782,6 +783,11 @@ class Auth(object): Args: user_id(str|None): If present, checks for presence against existing MAU cohort + threepid(dict|None): If present, checks for presence against configured + reserved threepid. Used in cases where the user is trying register + with a MAU blocked server, normally they would be rejected but their + threepid is on the reserved list. user_id and + threepid should never be set at the same time. """ # Never fail an auth check for the server notices users @@ -797,6 +803,10 @@ class Auth(object): limit_type=self.hs.config.hs_disabled_limit_type ) if self.hs.config.limit_usage_by_mau is True: + + if user_id and threepid: + logger.warn("Called with both user_id and threepid, this shoudn't happen") + # If the user is already part of the MAU cohort or a trial user if user_id: timestamp = yield self.store.user_last_seen_monthly_active(user_id) @@ -809,14 +819,13 @@ class Auth(object): elif threepid: # If the user does not exist yet, but is signing up with a # reserved threepid then pass auth check - if self.store.is_threepid_reserved(threepid): + if is_threepid_reserved(self.hs.config, threepid): return # Else if there is no room in the MAU bucket, bail current_mau = yield self.store.get_monthly_active_count() if current_mau >= self.hs.config.max_mau_value: raise ResourceLimitError( 403, "Monthly Active User Limit Exceeded", - admin_contact=self.hs.config.admin_contact, errcode=Codes.RESOURCE_LIMIT_EXCEEDED, limit_type="monthly_active_user" diff --git a/synapse/config/server.py b/synapse/config/server.py index 2faf472189..c1c7c0105e 100644 --- a/synapse/config/server.py +++ b/synapse/config/server.py @@ -404,6 +404,23 @@ class ServerConfig(Config): " service on the given port.") +def is_threepid_reserved(config, threepid): + """Check the threepid against the reserved threepid config + Args: + config(ServerConfig) - to access server config attributes + threepid(dict) - The threepid to test for + + Returns: + boolean Is the threepid undertest reserved_user + """ + + for tp in config.mau_limits_reserved_threepids: + if (threepid['medium'] == tp['medium'] + and threepid['address'] == tp['address']): + return True + return False + + def read_gc_thresholds(thresholds): """Reads the three integer thresholds for garbage collection. Ensures that the thresholds are integers if thresholds are supplied. diff --git a/synapse/rest/client/v1_only/register.py b/synapse/rest/client/v1_only/register.py index 95873e03d5..dadb376b02 100644 --- a/synapse/rest/client/v1_only/register.py +++ b/synapse/rest/client/v1_only/register.py @@ -23,6 +23,7 @@ from twisted.internet import defer import synapse.util.stringutils as stringutils from synapse.api.constants import LoginType from synapse.api.errors import Codes, SynapseError +from synapse.config.server import is_threepid_reserved from synapse.http.servlet import assert_params_in_dict, parse_json_object_from_request from synapse.rest.client.v1.base import ClientV1RestServlet from synapse.types import create_requester @@ -282,7 +283,7 @@ class RegisterRestServlet(ClientV1RestServlet): if "user" in register_json else None ) threepid = None - if session[LoginType.EMAIL_IDENTITY]: + if session.get(LoginType.EMAIL_IDENTITY): threepid = session["threepidCreds"] handler = self.handlers.registration_handler @@ -293,8 +294,8 @@ class RegisterRestServlet(ClientV1RestServlet): ) # Necessary due to auth checks prior to the threepid being # written to the db - if self.store.is_threepid_reserved(threepid): - self.store.upsert_monthly_active_user(registered_user_id) + if is_threepid_reserved(self.hs.config, threepid): + yield self.store.upsert_monthly_active_user(user_id) if session[LoginType.EMAIL_IDENTITY]: logger.debug("Binding emails %s to %s" % ( diff --git a/synapse/rest/client/v2_alpha/register.py b/synapse/rest/client/v2_alpha/register.py index f22b7577ea..2fb4d43ccb 100644 --- a/synapse/rest/client/v2_alpha/register.py +++ b/synapse/rest/client/v2_alpha/register.py @@ -26,6 +26,7 @@ import synapse import synapse.types from synapse.api.constants import LoginType from synapse.api.errors import Codes, SynapseError, UnrecognizedRequestError +from synapse.config.server import is_threepid_reserved from synapse.http.servlet import ( RestServlet, assert_params_in_dict, @@ -408,8 +409,8 @@ class RegisterRestServlet(RestServlet): ) # Necessary due to auth checks prior to the threepid being # written to the db - if self.store.is_threepid_reserved(threepid): - self.store.upsert_monthly_active_user(registered_user_id) + if is_threepid_reserved(self.hs.config, threepid): + yield self.store.upsert_monthly_active_user(registered_user_id) # remember that we've now registered that user account, and with # what user ID (since the user may not have specified) diff --git a/synapse/storage/monthly_active_users.py b/synapse/storage/monthly_active_users.py index 173867c4b1..c7899d7fd2 100644 --- a/synapse/storage/monthly_active_users.py +++ b/synapse/storage/monthly_active_users.py @@ -219,17 +219,3 @@ class MonthlyActiveUsersStore(SQLBaseStore): yield self.upsert_monthly_active_user(user_id) elif now - last_seen_timestamp > LAST_SEEN_GRANULARITY: yield self.upsert_monthly_active_user(user_id) - - def is_threepid_reserved(self, threepid): - """Check the threepid against the reserved threepid config - Args: - threepid(dict) - The threepid to test for - Returns: - boolean Is the threepid undertest reserved_user - """ - for tp in self.hs.config.mau_limits_reserved_threepids: - if (threepid['medium'] == tp['medium'] - and threepid['address'] == tp['address']): - return True - else: - return False diff --git a/tests/utils.py b/tests/utils.py index 179b592501..63e30dc6c0 100644 --- a/tests/utils.py +++ b/tests/utils.py @@ -26,6 +26,7 @@ from twisted.internet import defer, reactor from synapse.api.constants import EventTypes from synapse.api.errors import CodeMessageException, cs_error +from synapse.config.server import ServerConfig from synapse.federation.transport import server from synapse.http.server import HttpServer from synapse.server import HomeServer @@ -158,6 +159,11 @@ def setup_test_homeserver( # background, which upsets the test runner. config.update_user_directory = False + def is_threepid_reserved(threepid): + return ServerConfig.is_threepid_reserved(config, threepid) + + config.is_threepid_reserved.side_effect = is_threepid_reserved + config.use_frozen_dicts = True config.ldap_enabled = False -- cgit 1.5.1 From c42f7fd7b9f99923d71ad064fff7d75a5fc3cd93 Mon Sep 17 00:00:00 2001 From: Neil Johnson Date: Tue, 4 Sep 2018 12:03:17 +0100 Subject: improve human readable error messages --- synapse/rest/client/v2_alpha/account.py | 16 ++++++++++++---- synapse/rest/client/v2_alpha/register.py | 11 ++++++++--- 2 files changed, 20 insertions(+), 7 deletions(-) (limited to 'synapse/rest/client/v2_alpha') diff --git a/synapse/rest/client/v2_alpha/account.py b/synapse/rest/client/v2_alpha/account.py index 372648cafd..37b32dd37b 100644 --- a/synapse/rest/client/v2_alpha/account.py +++ b/synapse/rest/client/v2_alpha/account.py @@ -53,7 +53,9 @@ class EmailPasswordRequestTokenRestServlet(RestServlet): if not check_3pid_allowed(self.hs, "email", body['email']): raise SynapseError( - 403, "Third party identifier is not allowed", Codes.THREEPID_DENIED, + 403, + "Your email domain is not authorized on this server", + Codes.THREEPID_DENIED, ) existingUid = yield self.hs.get_datastore().get_user_id_by_threepid( @@ -89,7 +91,9 @@ class MsisdnPasswordRequestTokenRestServlet(RestServlet): if not check_3pid_allowed(self.hs, "msisdn", msisdn): raise SynapseError( - 403, "Third party identifier is not allowed", Codes.THREEPID_DENIED, + 403, + "Account phone numbers are not authorized on this server", + Codes.THREEPID_DENIED, ) existingUid = yield self.datastore.get_user_id_by_threepid( @@ -241,7 +245,9 @@ class EmailThreepidRequestTokenRestServlet(RestServlet): if not check_3pid_allowed(self.hs, "email", body['email']): raise SynapseError( - 403, "Third party identifier is not allowed", Codes.THREEPID_DENIED, + 403, + "Your email domain is not authorized on this server", + Codes.THREEPID_DENIED, ) existingUid = yield self.datastore.get_user_id_by_threepid( @@ -276,7 +282,9 @@ class MsisdnThreepidRequestTokenRestServlet(RestServlet): if not check_3pid_allowed(self.hs, "msisdn", msisdn): raise SynapseError( - 403, "Third party identifier is not allowed", Codes.THREEPID_DENIED, + 403, + "Account phone numbers are not authorized on this server", + Codes.THREEPID_DENIED, ) existingUid = yield self.datastore.get_user_id_by_threepid( diff --git a/synapse/rest/client/v2_alpha/register.py b/synapse/rest/client/v2_alpha/register.py index 2fb4d43ccb..200b29fa38 100644 --- a/synapse/rest/client/v2_alpha/register.py +++ b/synapse/rest/client/v2_alpha/register.py @@ -75,7 +75,9 @@ class EmailRegisterRequestTokenRestServlet(RestServlet): if not check_3pid_allowed(self.hs, "email", body['email']): raise SynapseError( - 403, "Third party identifier is not allowed", Codes.THREEPID_DENIED, + 403, + "Your email domain is not authorized to register on this server", + Codes.THREEPID_DENIED, ) existingUid = yield self.hs.get_datastore().get_user_id_by_threepid( @@ -115,7 +117,9 @@ class MsisdnRegisterRequestTokenRestServlet(RestServlet): if not check_3pid_allowed(self.hs, "msisdn", msisdn): raise SynapseError( - 403, "Third party identifier is not allowed", Codes.THREEPID_DENIED, + 403, + "Phone numbers are not authorized to register on this server", + Codes.THREEPID_DENIED, ) existingUid = yield self.hs.get_datastore().get_user_id_by_threepid( @@ -373,7 +377,8 @@ class RegisterRestServlet(RestServlet): if not check_3pid_allowed(self.hs, medium, address): raise SynapseError( - 403, "Third party identifier is not allowed", + 403, + "Third party identifier (email/phone number) is not allowed", Codes.THREEPID_DENIED, ) -- cgit 1.5.1 From bae37cd811fd71c68d28eb300eb0b74a5b6a1c27 Mon Sep 17 00:00:00 2001 From: Neil Johnson Date: Tue, 4 Sep 2018 12:07:00 +0100 Subject: improve human readable error message --- synapse/rest/client/v2_alpha/register.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) (limited to 'synapse/rest/client/v2_alpha') diff --git a/synapse/rest/client/v2_alpha/register.py b/synapse/rest/client/v2_alpha/register.py index 200b29fa38..192f52e462 100644 --- a/synapse/rest/client/v2_alpha/register.py +++ b/synapse/rest/client/v2_alpha/register.py @@ -378,7 +378,8 @@ class RegisterRestServlet(RestServlet): if not check_3pid_allowed(self.hs, medium, address): raise SynapseError( 403, - "Third party identifier (email/phone number) is not allowed", + "Third party identifiers (email/phone numbers)" + + " are not authorized on this server", Codes.THREEPID_DENIED, ) -- cgit 1.5.1 From 87c18d12ee30c84a108ac686454720d6ac6a2f84 Mon Sep 17 00:00:00 2001 From: Richard van der Hoff Date: Tue, 4 Sep 2018 15:18:25 +0100 Subject: Implement 'event_format' filter param in /sync This has been specced and part-implemented; let's implement it for /sync (but no other endpoints yet :/). --- synapse/api/filtering.py | 1 + synapse/rest/client/v2_alpha/sync.py | 51 +++++++++++++++++++++++++++--------- 2 files changed, 39 insertions(+), 13 deletions(-) (limited to 'synapse/rest/client/v2_alpha') diff --git a/synapse/api/filtering.py b/synapse/api/filtering.py index 186831e118..a31a9a17e0 100644 --- a/synapse/api/filtering.py +++ b/synapse/api/filtering.py @@ -251,6 +251,7 @@ class FilterCollection(object): "include_leave", False ) self.event_fields = filter_json.get("event_fields", []) + self.event_format = filter_json.get("event_format", "client") def __repr__(self): return "" % (json.dumps(self._filter_json),) diff --git a/synapse/rest/client/v2_alpha/sync.py b/synapse/rest/client/v2_alpha/sync.py index 1275baa1ba..263d8eb73e 100644 --- a/synapse/rest/client/v2_alpha/sync.py +++ b/synapse/rest/client/v2_alpha/sync.py @@ -25,6 +25,7 @@ from synapse.api.errors import SynapseError from synapse.api.filtering import DEFAULT_FILTER_COLLECTION, FilterCollection from synapse.events.utils import ( format_event_for_client_v2_without_room_id, + format_event_raw, serialize_event, ) from synapse.handlers.presence import format_user_presence_state @@ -175,17 +176,28 @@ class SyncRestServlet(RestServlet): @staticmethod def encode_response(time_now, sync_result, access_token_id, filter): + if filter.event_format == 'client': + event_formatter = format_event_for_client_v2_without_room_id + elif filter.event_format == 'federation': + event_formatter = format_event_raw + else: + raise Exception("Unknown event format %s" % (filter.event_format, )) + joined = SyncRestServlet.encode_joined( - sync_result.joined, time_now, access_token_id, filter.event_fields + sync_result.joined, time_now, access_token_id, + filter.event_fields, + event_formatter, ) invited = SyncRestServlet.encode_invited( sync_result.invited, time_now, access_token_id, + event_formatter, ) archived = SyncRestServlet.encode_archived( sync_result.archived, time_now, access_token_id, filter.event_fields, + event_formatter, ) return { @@ -228,7 +240,7 @@ class SyncRestServlet(RestServlet): } @staticmethod - def encode_joined(rooms, time_now, token_id, event_fields): + def encode_joined(rooms, time_now, token_id, event_fields, event_formatter): """ Encode the joined rooms in a sync result @@ -240,7 +252,9 @@ class SyncRestServlet(RestServlet): token_id(int): ID of the user's auth token - used for namespacing of transaction IDs event_fields(list): List of event fields to include. If empty, - all fields will be returned. + all fields will be returned. + event_formatter (func[dict]): function to convert from federation format + to client format Returns: dict[str, dict[str, object]]: the joined rooms list, in our response format @@ -248,13 +262,14 @@ class SyncRestServlet(RestServlet): joined = {} for room in rooms: joined[room.room_id] = SyncRestServlet.encode_room( - room, time_now, token_id, only_fields=event_fields + room, time_now, token_id, joined=True, only_fields=event_fields, + event_formatter=event_formatter, ) return joined @staticmethod - def encode_invited(rooms, time_now, token_id): + def encode_invited(rooms, time_now, token_id, event_formatter): """ Encode the invited rooms in a sync result @@ -264,7 +279,9 @@ class SyncRestServlet(RestServlet): time_now(int): current time - used as a baseline for age calculations token_id(int): ID of the user's auth token - used for namespacing - of transaction IDs + of transaction IDs + event_formatter (func[dict]): function to convert from federation format + to client format Returns: dict[str, dict[str, object]]: the invited rooms list, in our @@ -274,7 +291,7 @@ class SyncRestServlet(RestServlet): for room in rooms: invite = serialize_event( room.invite, time_now, token_id=token_id, - event_format=format_event_for_client_v2_without_room_id, + event_format=event_formatter, is_invite=True, ) unsigned = dict(invite.get("unsigned", {})) @@ -288,7 +305,7 @@ class SyncRestServlet(RestServlet): return invited @staticmethod - def encode_archived(rooms, time_now, token_id, event_fields): + def encode_archived(rooms, time_now, token_id, event_fields, event_formatter): """ Encode the archived rooms in a sync result @@ -300,7 +317,9 @@ class SyncRestServlet(RestServlet): token_id(int): ID of the user's auth token - used for namespacing of transaction IDs event_fields(list): List of event fields to include. If empty, - all fields will be returned. + all fields will be returned. + event_formatter (func[dict]): function to convert from federation format + to client format Returns: dict[str, dict[str, object]]: The invited rooms list, in our response format @@ -308,13 +327,18 @@ class SyncRestServlet(RestServlet): joined = {} for room in rooms: joined[room.room_id] = SyncRestServlet.encode_room( - room, time_now, token_id, joined=False, only_fields=event_fields + room, time_now, token_id, joined=False, + only_fields=event_fields, + event_formatter=event_formatter, ) return joined @staticmethod - def encode_room(room, time_now, token_id, joined=True, only_fields=None): + def encode_room( + room, time_now, token_id, joined, + only_fields, event_formatter, + ): """ Args: room (JoinedSyncResult|ArchivedSyncResult): sync result for a @@ -326,14 +350,15 @@ class SyncRestServlet(RestServlet): joined (bool): True if the user is joined to this room - will mean we handle ephemeral events only_fields(list): Optional. The list of event fields to include. + event_formatter (func[dict]): function to convert from federation format + to client format Returns: dict[str, object]: the room, encoded in our response format """ def serialize(event): - # TODO(mjark): Respect formatting requirements in the filter. return serialize_event( event, time_now, token_id=token_id, - event_format=format_event_for_client_v2_without_room_id, + event_format=event_formatter, only_event_fields=only_fields, ) -- cgit 1.5.1 From 3801b8aa035594972c400c8bd036894a388c4ab3 Mon Sep 17 00:00:00 2001 From: Hubert Chathi Date: Thu, 6 Sep 2018 11:23:16 -0400 Subject: try to make flake8 and isort happy --- synapse/api/errors.py | 1 + synapse/handlers/e2e_room_keys.py | 2 +- synapse/rest/client/v2_alpha/room_keys.py | 5 ++++- synapse/storage/e2e_room_keys.py | 6 ++++-- tests/handlers/test_e2e_room_keys.py | 9 +++++---- 5 files changed, 15 insertions(+), 8 deletions(-) (limited to 'synapse/rest/client/v2_alpha') diff --git a/synapse/api/errors.py b/synapse/api/errors.py index 3002c95dd1..140dbfe8b8 100644 --- a/synapse/api/errors.py +++ b/synapse/api/errors.py @@ -326,6 +326,7 @@ class RoomKeysVersionError(SynapseError): ) self.current_version = current_version + class IncompatibleRoomVersionError(SynapseError): """A server is trying to join a room whose version it does not support.""" diff --git a/synapse/handlers/e2e_room_keys.py b/synapse/handlers/e2e_room_keys.py index c09816b372..2c330382cf 100644 --- a/synapse/handlers/e2e_room_keys.py +++ b/synapse/handlers/e2e_room_keys.py @@ -17,7 +17,7 @@ import logging from twisted.internet import defer -from synapse.api.errors import StoreError, SynapseError, RoomKeysVersionError +from synapse.api.errors import RoomKeysVersionError, StoreError, SynapseError from synapse.util.async_helpers import Linearizer logger = logging.getLogger(__name__) diff --git a/synapse/rest/client/v2_alpha/room_keys.py b/synapse/rest/client/v2_alpha/room_keys.py index 9f0172e6f5..1ed18e986f 100644 --- a/synapse/rest/client/v2_alpha/room_keys.py +++ b/synapse/rest/client/v2_alpha/room_keys.py @@ -19,8 +19,11 @@ from twisted.internet import defer from synapse.api.errors import SynapseError from synapse.http.servlet import ( - RestServlet, parse_json_object_from_request, parse_string + RestServlet, + parse_json_object_from_request, + parse_string, ) + from ._base import client_v2_patterns logger = logging.getLogger(__name__) diff --git a/synapse/storage/e2e_room_keys.py b/synapse/storage/e2e_room_keys.py index b695570a7b..969f4aef9c 100644 --- a/synapse/storage/e2e_room_keys.py +++ b/synapse/storage/e2e_room_keys.py @@ -13,9 +13,11 @@ # See the License for the specific language governing permissions and # limitations under the License. +import simplejson as json + from twisted.internet import defer + from synapse.api.errors import StoreError -import simplejson as json from ._base import SQLBaseStore @@ -58,7 +60,7 @@ class EndToEndRoomKeyStore(SQLBaseStore): desc="get_e2e_room_key", ) - row["session_data"] = json.loads(row["session_data"]); + row["session_data"] = json.loads(row["session_data"]) defer.returnValue(row) diff --git a/tests/handlers/test_e2e_room_keys.py b/tests/handlers/test_e2e_room_keys.py index 7fa4264441..9e08eac0a5 100644 --- a/tests/handlers/test_e2e_room_keys.py +++ b/tests/handlers/test_e2e_room_keys.py @@ -14,17 +14,18 @@ # See the License for the specific language governing permissions and # limitations under the License. +import copy + import mock -from synapse.api import errors + from twisted.internet import defer -import copy import synapse.api.errors import synapse.handlers.e2e_room_keys - import synapse.storage -from tests import unittest, utils +from synapse.api import errors +from tests import unittest, utils # sample room_key data for use in the tests room_keys = { -- cgit 1.5.1 From 02aa41809bd48cc1dd02da3634f02f5eae71b41c Mon Sep 17 00:00:00 2001 From: Amber Brown Date: Wed, 12 Sep 2018 20:41:31 +1000 Subject: Port rest/ to Python 3 (#3823) --- changelog.d/3823.misc | 1 + synapse/rest/client/v1/admin.py | 9 ++++-- synapse/rest/client/v1/events.py | 12 ++++---- synapse/rest/client/v1/initial_sync.py | 2 +- synapse/rest/client/v1/login.py | 44 +++++++++++++-------------- synapse/rest/client/v1/push_rule.py | 24 +++++++-------- synapse/rest/client/v1/pusher.py | 4 +-- synapse/rest/client/v1/room.py | 14 +++++---- synapse/rest/client/v2_alpha/sync.py | 2 +- synapse/rest/client/v2_alpha/thirdparty.py | 4 +-- synapse/rest/key/v1/server_key_resource.py | 2 +- synapse/rest/key/v2/__init__.py | 4 +-- synapse/rest/key/v2/remote_key_resource.py | 6 ++-- synapse/rest/media/v0/content_repository.py | 4 +-- synapse/rest/media/v1/_base.py | 30 +++++++++--------- synapse/rest/media/v1/download_resource.py | 12 ++++---- synapse/rest/media/v1/media_repository.py | 29 ++++++++++-------- synapse/rest/media/v1/preview_url_resource.py | 10 +++--- 18 files changed, 113 insertions(+), 100 deletions(-) create mode 100644 changelog.d/3823.misc (limited to 'synapse/rest/client/v2_alpha') diff --git a/changelog.d/3823.misc b/changelog.d/3823.misc new file mode 100644 index 0000000000..0da491ddaa --- /dev/null +++ b/changelog.d/3823.misc @@ -0,0 +1 @@ +rest/ is now ported to Python 3. diff --git a/synapse/rest/client/v1/admin.py b/synapse/rest/client/v1/admin.py index ad536ab570..41534b8c2a 100644 --- a/synapse/rest/client/v1/admin.py +++ b/synapse/rest/client/v1/admin.py @@ -101,7 +101,7 @@ class UserRegisterServlet(ClientV1RestServlet): nonce = self.hs.get_secrets().token_hex(64) self.nonces[nonce] = int(self.reactor.seconds()) - return (200, {"nonce": nonce.encode('ascii')}) + return (200, {"nonce": nonce}) @defer.inlineCallbacks def on_POST(self, request): @@ -164,7 +164,7 @@ class UserRegisterServlet(ClientV1RestServlet): key=self.hs.config.registration_shared_secret.encode(), digestmod=hashlib.sha1, ) - want_mac.update(nonce) + want_mac.update(nonce.encode('utf8')) want_mac.update(b"\x00") want_mac.update(username) want_mac.update(b"\x00") @@ -173,7 +173,10 @@ class UserRegisterServlet(ClientV1RestServlet): want_mac.update(b"admin" if admin else b"notadmin") want_mac = want_mac.hexdigest() - if not hmac.compare_digest(want_mac, got_mac.encode('ascii')): + if not hmac.compare_digest( + want_mac.encode('ascii'), + got_mac.encode('ascii') + ): raise SynapseError(403, "HMAC incorrect") # Reuse the parts of RegisterRestServlet to reduce code duplication diff --git a/synapse/rest/client/v1/events.py b/synapse/rest/client/v1/events.py index 0f3a2e8b51..cd9b3bdbd1 100644 --- a/synapse/rest/client/v1/events.py +++ b/synapse/rest/client/v1/events.py @@ -45,20 +45,20 @@ class EventStreamRestServlet(ClientV1RestServlet): is_guest = requester.is_guest room_id = None if is_guest: - if "room_id" not in request.args: + if b"room_id" not in request.args: raise SynapseError(400, "Guest users must specify room_id param") - if "room_id" in request.args: - room_id = request.args["room_id"][0] + if b"room_id" in request.args: + room_id = request.args[b"room_id"][0].decode('ascii') pagin_config = PaginationConfig.from_request(request) timeout = EventStreamRestServlet.DEFAULT_LONGPOLL_TIME_MS - if "timeout" in request.args: + if b"timeout" in request.args: try: - timeout = int(request.args["timeout"][0]) + timeout = int(request.args[b"timeout"][0]) except ValueError: raise SynapseError(400, "timeout must be in milliseconds.") - as_client_event = "raw" not in request.args + as_client_event = b"raw" not in request.args chunk = yield self.event_stream_handler.get_stream( requester.user.to_string(), diff --git a/synapse/rest/client/v1/initial_sync.py b/synapse/rest/client/v1/initial_sync.py index fd5f85b53e..3ead75cb77 100644 --- a/synapse/rest/client/v1/initial_sync.py +++ b/synapse/rest/client/v1/initial_sync.py @@ -32,7 +32,7 @@ class InitialSyncRestServlet(ClientV1RestServlet): @defer.inlineCallbacks def on_GET(self, request): requester = yield self.auth.get_user_by_req(request) - as_client_event = "raw" not in request.args + as_client_event = b"raw" not in request.args pagination_config = PaginationConfig.from_request(request) include_archived = parse_boolean(request, "archived", default=False) content = yield self.initial_sync_handler.snapshot_all_rooms( diff --git a/synapse/rest/client/v1/login.py b/synapse/rest/client/v1/login.py index cb85fa1436..0010699d31 100644 --- a/synapse/rest/client/v1/login.py +++ b/synapse/rest/client/v1/login.py @@ -14,10 +14,9 @@ # limitations under the License. import logging -import urllib import xml.etree.ElementTree as ET -from six.moves.urllib import parse as urlparse +from six.moves import urllib from canonicaljson import json from saml2 import BINDING_HTTP_POST, config @@ -134,7 +133,7 @@ class LoginRestServlet(ClientV1RestServlet): LoginRestServlet.SAML2_TYPE): relay_state = "" if "relay_state" in login_submission: - relay_state = "&RelayState=" + urllib.quote( + relay_state = "&RelayState=" + urllib.parse.quote( login_submission["relay_state"]) result = { "uri": "%s%s" % (self.idp_redirect_url, relay_state) @@ -366,7 +365,7 @@ class SAML2RestServlet(ClientV1RestServlet): (user_id, token) = yield handler.register_saml2(username) # Forward to the RelayState callback along with ava if 'RelayState' in request.args: - request.redirect(urllib.unquote( + request.redirect(urllib.parse.unquote( request.args['RelayState'][0]) + '?status=authenticated&access_token=' + token + '&user_id=' + user_id + '&ava=' + @@ -377,7 +376,7 @@ class SAML2RestServlet(ClientV1RestServlet): "user_id": user_id, "token": token, "ava": saml2_auth.ava})) elif 'RelayState' in request.args: - request.redirect(urllib.unquote( + request.redirect(urllib.parse.unquote( request.args['RelayState'][0]) + '?status=not_authenticated') finish_request(request) @@ -390,21 +389,22 @@ class CasRedirectServlet(ClientV1RestServlet): def __init__(self, hs): super(CasRedirectServlet, self).__init__(hs) - self.cas_server_url = hs.config.cas_server_url - self.cas_service_url = hs.config.cas_service_url + self.cas_server_url = hs.config.cas_server_url.encode('ascii') + self.cas_service_url = hs.config.cas_service_url.encode('ascii') def on_GET(self, request): args = request.args - if "redirectUrl" not in args: + if b"redirectUrl" not in args: return (400, "Redirect URL not specified for CAS auth") - client_redirect_url_param = urllib.urlencode({ - "redirectUrl": args["redirectUrl"][0] - }) - hs_redirect_url = self.cas_service_url + "/_matrix/client/api/v1/login/cas/ticket" - service_param = urllib.urlencode({ - "service": "%s?%s" % (hs_redirect_url, client_redirect_url_param) - }) - request.redirect("%s/login?%s" % (self.cas_server_url, service_param)) + client_redirect_url_param = urllib.parse.urlencode({ + b"redirectUrl": args[b"redirectUrl"][0] + }).encode('ascii') + hs_redirect_url = (self.cas_service_url + + b"/_matrix/client/api/v1/login/cas/ticket") + service_param = urllib.parse.urlencode({ + b"service": b"%s?%s" % (hs_redirect_url, client_redirect_url_param) + }).encode('ascii') + request.redirect(b"%s/login?%s" % (self.cas_server_url, service_param)) finish_request(request) @@ -422,11 +422,11 @@ class CasTicketServlet(ClientV1RestServlet): @defer.inlineCallbacks def on_GET(self, request): - client_redirect_url = request.args["redirectUrl"][0] + client_redirect_url = request.args[b"redirectUrl"][0] http_client = self.hs.get_simple_http_client() uri = self.cas_server_url + "/proxyValidate" args = { - "ticket": request.args["ticket"], + "ticket": request.args[b"ticket"][0].decode('ascii'), "service": self.cas_service_url } try: @@ -471,11 +471,11 @@ class CasTicketServlet(ClientV1RestServlet): finish_request(request) def add_login_token_to_redirect_url(self, url, token): - url_parts = list(urlparse.urlparse(url)) - query = dict(urlparse.parse_qsl(url_parts[4])) + url_parts = list(urllib.parse.urlparse(url)) + query = dict(urllib.parse.parse_qsl(url_parts[4])) query.update({"loginToken": token}) - url_parts[4] = urllib.urlencode(query) - return urlparse.urlunparse(url_parts) + url_parts[4] = urllib.parse.urlencode(query).encode('ascii') + return urllib.parse.urlunparse(url_parts) def parse_cas_response(self, cas_response_body): user = None diff --git a/synapse/rest/client/v1/push_rule.py b/synapse/rest/client/v1/push_rule.py index 6e95d9bec2..9382b1f124 100644 --- a/synapse/rest/client/v1/push_rule.py +++ b/synapse/rest/client/v1/push_rule.py @@ -46,7 +46,7 @@ class PushRuleRestServlet(ClientV1RestServlet): try: priority_class = _priority_class_from_spec(spec) except InvalidRuleException as e: - raise SynapseError(400, e.message) + raise SynapseError(400, str(e)) requester = yield self.auth.get_user_by_req(request) @@ -73,7 +73,7 @@ class PushRuleRestServlet(ClientV1RestServlet): content, ) except InvalidRuleException as e: - raise SynapseError(400, e.message) + raise SynapseError(400, str(e)) before = parse_string(request, "before") if before: @@ -95,9 +95,9 @@ class PushRuleRestServlet(ClientV1RestServlet): ) self.notify_user(user_id) except InconsistentRuleException as e: - raise SynapseError(400, e.message) + raise SynapseError(400, str(e)) except RuleNotFoundException as e: - raise SynapseError(400, e.message) + raise SynapseError(400, str(e)) defer.returnValue((200, {})) @@ -142,10 +142,10 @@ class PushRuleRestServlet(ClientV1RestServlet): PushRuleRestServlet.SLIGHTLY_PEDANTIC_TRAILING_SLASH_ERROR ) - if path[0] == '': + if path[0] == b'': defer.returnValue((200, rules)) - elif path[0] == 'global': - path = path[1:] + elif path[0] == b'global': + path = [x.decode('ascii') for x in path[1:]] result = _filter_ruleset_with_path(rules['global'], path) defer.returnValue((200, result)) else: @@ -192,10 +192,10 @@ class PushRuleRestServlet(ClientV1RestServlet): def _rule_spec_from_path(path): if len(path) < 2: raise UnrecognizedRequestError() - if path[0] != 'pushrules': + if path[0] != b'pushrules': raise UnrecognizedRequestError() - scope = path[1] + scope = path[1].decode('ascii') path = path[2:] if scope != 'global': raise UnrecognizedRequestError() @@ -203,13 +203,13 @@ def _rule_spec_from_path(path): if len(path) == 0: raise UnrecognizedRequestError() - template = path[0] + template = path[0].decode('ascii') path = path[1:] if len(path) == 0 or len(path[0]) == 0: raise UnrecognizedRequestError() - rule_id = path[0] + rule_id = path[0].decode('ascii') spec = { 'scope': scope, @@ -220,7 +220,7 @@ def _rule_spec_from_path(path): path = path[1:] if len(path) > 0 and len(path[0]) > 0: - spec['attr'] = path[0] + spec['attr'] = path[0].decode('ascii') return spec diff --git a/synapse/rest/client/v1/pusher.py b/synapse/rest/client/v1/pusher.py index 182a68b1e2..b84f0260f2 100644 --- a/synapse/rest/client/v1/pusher.py +++ b/synapse/rest/client/v1/pusher.py @@ -59,7 +59,7 @@ class PushersRestServlet(ClientV1RestServlet): ] for p in pushers: - for k, v in p.items(): + for k, v in list(p.items()): if k not in allowed_keys: del p[k] @@ -126,7 +126,7 @@ class PushersSetRestServlet(ClientV1RestServlet): profile_tag=content.get('profile_tag', ""), ) except PusherConfigException as pce: - raise SynapseError(400, "Config Error: " + pce.message, + raise SynapseError(400, "Config Error: " + str(pce), errcode=Codes.MISSING_PARAM) self.notifier.on_new_replication_data() diff --git a/synapse/rest/client/v1/room.py b/synapse/rest/client/v1/room.py index 976d98387d..663934efd0 100644 --- a/synapse/rest/client/v1/room.py +++ b/synapse/rest/client/v1/room.py @@ -207,7 +207,7 @@ class RoomSendEventRestServlet(ClientV1RestServlet): "sender": requester.user.to_string(), } - if 'ts' in request.args and requester.app_service: + if b'ts' in request.args and requester.app_service: event_dict['origin_server_ts'] = parse_integer(request, "ts", 0) event = yield self.event_creation_hander.create_and_send_nonmember_event( @@ -255,7 +255,9 @@ class JoinRoomAliasServlet(ClientV1RestServlet): if RoomID.is_valid(room_identifier): room_id = room_identifier try: - remote_room_hosts = request.args["server_name"] + remote_room_hosts = [ + x.decode('ascii') for x in request.args[b"server_name"] + ] except Exception: remote_room_hosts = None elif RoomAlias.is_valid(room_identifier): @@ -461,10 +463,10 @@ class RoomMessageListRestServlet(ClientV1RestServlet): pagination_config = PaginationConfig.from_request( request, default_limit=10, ) - as_client_event = "raw" not in request.args - filter_bytes = parse_string(request, "filter") + as_client_event = b"raw" not in request.args + filter_bytes = parse_string(request, b"filter", encoding=None) if filter_bytes: - filter_json = urlparse.unquote(filter_bytes).decode("UTF-8") + filter_json = urlparse.unquote(filter_bytes.decode("UTF-8")) event_filter = Filter(json.loads(filter_json)) else: event_filter = None @@ -560,7 +562,7 @@ class RoomEventContextServlet(ClientV1RestServlet): # picking the API shape for symmetry with /messages filter_bytes = parse_string(request, "filter") if filter_bytes: - filter_json = urlparse.unquote(filter_bytes).decode("UTF-8") + filter_json = urlparse.unquote(filter_bytes) event_filter = Filter(json.loads(filter_json)) else: event_filter = None diff --git a/synapse/rest/client/v2_alpha/sync.py b/synapse/rest/client/v2_alpha/sync.py index 263d8eb73e..0251146722 100644 --- a/synapse/rest/client/v2_alpha/sync.py +++ b/synapse/rest/client/v2_alpha/sync.py @@ -89,7 +89,7 @@ class SyncRestServlet(RestServlet): @defer.inlineCallbacks def on_GET(self, request): - if "from" in request.args: + if b"from" in request.args: # /events used to use 'from', but /sync uses 'since'. # Lets be helpful and whine if we see a 'from'. raise SynapseError( diff --git a/synapse/rest/client/v2_alpha/thirdparty.py b/synapse/rest/client/v2_alpha/thirdparty.py index d9d379182e..b9b5d07677 100644 --- a/synapse/rest/client/v2_alpha/thirdparty.py +++ b/synapse/rest/client/v2_alpha/thirdparty.py @@ -79,7 +79,7 @@ class ThirdPartyUserServlet(RestServlet): yield self.auth.get_user_by_req(request, allow_guest=True) fields = request.args - fields.pop("access_token", None) + fields.pop(b"access_token", None) results = yield self.appservice_handler.query_3pe( ThirdPartyEntityKind.USER, protocol, fields @@ -102,7 +102,7 @@ class ThirdPartyLocationServlet(RestServlet): yield self.auth.get_user_by_req(request, allow_guest=True) fields = request.args - fields.pop("access_token", None) + fields.pop(b"access_token", None) results = yield self.appservice_handler.query_3pe( ThirdPartyEntityKind.LOCATION, protocol, fields diff --git a/synapse/rest/key/v1/server_key_resource.py b/synapse/rest/key/v1/server_key_resource.py index b9ee6e1c13..38eb2ee23f 100644 --- a/synapse/rest/key/v1/server_key_resource.py +++ b/synapse/rest/key/v1/server_key_resource.py @@ -88,5 +88,5 @@ class LocalKey(Resource): ) def getChild(self, name, request): - if name == '': + if name == b'': return self diff --git a/synapse/rest/key/v2/__init__.py b/synapse/rest/key/v2/__init__.py index 3491fd2118..cb5abcf826 100644 --- a/synapse/rest/key/v2/__init__.py +++ b/synapse/rest/key/v2/__init__.py @@ -22,5 +22,5 @@ from .remote_key_resource import RemoteKey class KeyApiV2Resource(Resource): def __init__(self, hs): Resource.__init__(self) - self.putChild("server", LocalKey(hs)) - self.putChild("query", RemoteKey(hs)) + self.putChild(b"server", LocalKey(hs)) + self.putChild(b"query", RemoteKey(hs)) diff --git a/synapse/rest/key/v2/remote_key_resource.py b/synapse/rest/key/v2/remote_key_resource.py index 7d67e4b064..eb8782aa6e 100644 --- a/synapse/rest/key/v2/remote_key_resource.py +++ b/synapse/rest/key/v2/remote_key_resource.py @@ -103,7 +103,7 @@ class RemoteKey(Resource): def async_render_GET(self, request): if len(request.postpath) == 1: server, = request.postpath - query = {server: {}} + query = {server.decode('ascii'): {}} elif len(request.postpath) == 2: server, key_id = request.postpath minimum_valid_until_ts = parse_integer( @@ -112,11 +112,12 @@ class RemoteKey(Resource): arguments = {} if minimum_valid_until_ts is not None: arguments["minimum_valid_until_ts"] = minimum_valid_until_ts - query = {server: {key_id: arguments}} + query = {server.decode('ascii'): {key_id.decode('ascii'): arguments}} else: raise SynapseError( 404, "Not found %r" % request.postpath, Codes.NOT_FOUND ) + yield self.query_keys(request, query, query_remote_on_cache_miss=True) def render_POST(self, request): @@ -135,6 +136,7 @@ class RemoteKey(Resource): @defer.inlineCallbacks def query_keys(self, request, query, query_remote_on_cache_miss=False): logger.info("Handling query for keys %r", query) + store_queries = [] for server_name, key_ids in query.items(): if ( diff --git a/synapse/rest/media/v0/content_repository.py b/synapse/rest/media/v0/content_repository.py index f255f2883f..5a426ff2f6 100644 --- a/synapse/rest/media/v0/content_repository.py +++ b/synapse/rest/media/v0/content_repository.py @@ -56,7 +56,7 @@ class ContentRepoResource(resource.Resource): # servers. # TODO: A little crude here, we could do this better. - filename = request.path.split('/')[-1] + filename = request.path.decode('ascii').split('/')[-1] # be paranoid filename = re.sub("[^0-9A-z.-_]", "", filename) @@ -78,7 +78,7 @@ class ContentRepoResource(resource.Resource): # select private. don't bother setting Expires as all our matrix # clients are smart enough to be happy with Cache-Control (right?) request.setHeader( - "Cache-Control", "public,max-age=86400,s-maxage=86400" + b"Cache-Control", b"public,max-age=86400,s-maxage=86400" ) d = FileSender().beginFileTransfer(f, request) diff --git a/synapse/rest/media/v1/_base.py b/synapse/rest/media/v1/_base.py index 65f4bd2910..76e479afa3 100644 --- a/synapse/rest/media/v1/_base.py +++ b/synapse/rest/media/v1/_base.py @@ -15,9 +15,8 @@ import logging import os -import urllib -from six.moves.urllib import parse as urlparse +from six.moves import urllib from twisted.internet import defer from twisted.protocols.basic import FileSender @@ -35,10 +34,15 @@ def parse_media_id(request): # This allows users to append e.g. /test.png to the URL. Useful for # clients that parse the URL to see content type. server_name, media_id = request.postpath[:2] + + if isinstance(server_name, bytes): + server_name = server_name.decode('utf-8') + media_id = media_id.decode('utf8') + file_name = None if len(request.postpath) > 2: try: - file_name = urlparse.unquote(request.postpath[-1]).decode("utf-8") + file_name = urllib.parse.unquote(request.postpath[-1].decode("utf-8")) except UnicodeDecodeError: pass return server_name, media_id, file_name @@ -93,22 +97,18 @@ def add_file_headers(request, media_type, file_size, upload_name): file_size (int): Size in bytes of the media, if known. upload_name (str): The name of the requested file, if any. """ + def _quote(x): + return urllib.parse.quote(x.encode("utf-8")) + request.setHeader(b"Content-Type", media_type.encode("UTF-8")) if upload_name: if is_ascii(upload_name): - request.setHeader( - b"Content-Disposition", - b"inline; filename=%s" % ( - urllib.quote(upload_name.encode("utf-8")), - ), - ) + disposition = ("inline; filename=%s" % (_quote(upload_name),)).encode("ascii") else: - request.setHeader( - b"Content-Disposition", - b"inline; filename*=utf-8''%s" % ( - urllib.quote(upload_name.encode("utf-8")), - ), - ) + disposition = ( + "inline; filename*=utf-8''%s" % (_quote(upload_name),)).encode("ascii") + + request.setHeader(b"Content-Disposition", disposition) # cache for at least a day. # XXX: we might want to turn this off for data we don't want to diff --git a/synapse/rest/media/v1/download_resource.py b/synapse/rest/media/v1/download_resource.py index fbfa85f74f..ca90964d1d 100644 --- a/synapse/rest/media/v1/download_resource.py +++ b/synapse/rest/media/v1/download_resource.py @@ -47,12 +47,12 @@ class DownloadResource(Resource): def _async_render_GET(self, request): set_cors_headers(request) request.setHeader( - "Content-Security-Policy", - "default-src 'none';" - " script-src 'none';" - " plugin-types application/pdf;" - " style-src 'unsafe-inline';" - " object-src 'self';" + b"Content-Security-Policy", + b"default-src 'none';" + b" script-src 'none';" + b" plugin-types application/pdf;" + b" style-src 'unsafe-inline';" + b" object-src 'self';" ) server_name, media_id, name = parse_media_id(request) if server_name == self.server_name: diff --git a/synapse/rest/media/v1/media_repository.py b/synapse/rest/media/v1/media_repository.py index 241c972070..a828ff4438 100644 --- a/synapse/rest/media/v1/media_repository.py +++ b/synapse/rest/media/v1/media_repository.py @@ -20,7 +20,7 @@ import logging import os import shutil -from six import iteritems +from six import PY3, iteritems from six.moves.urllib import parse as urlparse import twisted.internet.error @@ -397,13 +397,13 @@ class MediaRepository(object): yield finish() - media_type = headers["Content-Type"][0] + media_type = headers[b"Content-Type"][0].decode('ascii') time_now_ms = self.clock.time_msec() - content_disposition = headers.get("Content-Disposition", None) + content_disposition = headers.get(b"Content-Disposition", None) if content_disposition: - _, params = cgi.parse_header(content_disposition[0],) + _, params = cgi.parse_header(content_disposition[0].decode('ascii'),) upload_name = None # First check if there is a valid UTF-8 filename @@ -419,9 +419,13 @@ class MediaRepository(object): upload_name = upload_name_ascii if upload_name: - upload_name = urlparse.unquote(upload_name) + if PY3: + upload_name = urlparse.unquote(upload_name) + else: + upload_name = urlparse.unquote(upload_name.encode('ascii')) try: - upload_name = upload_name.decode("utf-8") + if isinstance(upload_name, bytes): + upload_name = upload_name.decode("utf-8") except UnicodeDecodeError: upload_name = None else: @@ -755,14 +759,15 @@ class MediaRepositoryResource(Resource): Resource.__init__(self) media_repo = hs.get_media_repository() - self.putChild("upload", UploadResource(hs, media_repo)) - self.putChild("download", DownloadResource(hs, media_repo)) - self.putChild("thumbnail", ThumbnailResource( + + self.putChild(b"upload", UploadResource(hs, media_repo)) + self.putChild(b"download", DownloadResource(hs, media_repo)) + self.putChild(b"thumbnail", ThumbnailResource( hs, media_repo, media_repo.media_storage, )) - self.putChild("identicon", IdenticonResource()) + self.putChild(b"identicon", IdenticonResource()) if hs.config.url_preview_enabled: - self.putChild("preview_url", PreviewUrlResource( + self.putChild(b"preview_url", PreviewUrlResource( hs, media_repo, media_repo.media_storage, )) - self.putChild("config", MediaConfigResource(hs)) + self.putChild(b"config", MediaConfigResource(hs)) diff --git a/synapse/rest/media/v1/preview_url_resource.py b/synapse/rest/media/v1/preview_url_resource.py index 778ef97337..cad2dec33a 100644 --- a/synapse/rest/media/v1/preview_url_resource.py +++ b/synapse/rest/media/v1/preview_url_resource.py @@ -261,7 +261,7 @@ class PreviewUrlResource(Resource): logger.debug("Calculated OG for %s as %s" % (url, og)) - jsonog = json.dumps(og) + jsonog = json.dumps(og).encode('utf8') # store OG in history-aware DB cache yield self.store.store_url_cache( @@ -301,20 +301,20 @@ class PreviewUrlResource(Resource): logger.warn("Error downloading %s: %r", url, e) raise SynapseError( 500, "Failed to download content: %s" % ( - traceback.format_exception_only(sys.exc_type, e), + traceback.format_exception_only(sys.exc_info()[0], e), ), Codes.UNKNOWN, ) yield finish() try: - if "Content-Type" in headers: - media_type = headers["Content-Type"][0] + if b"Content-Type" in headers: + media_type = headers[b"Content-Type"][0].decode('ascii') else: media_type = "application/octet-stream" time_now_ms = self.clock.time_msec() - content_disposition = headers.get("Content-Disposition", None) + content_disposition = headers.get(b"Content-Disposition", None) if content_disposition: _, params = cgi.parse_header(content_disposition[0],) download_name = None -- cgit 1.5.1 From bc74925c5b94f0c02603297d4ccb7e05008d5124 Mon Sep 17 00:00:00 2001 From: David Baker Date: Thu, 13 Sep 2018 17:02:59 +0100 Subject: WIP e2e key backups Continues from uhoreg's branch This just fixed the errcode on /room_keys/version if no backup and updates the schema delta to be on the latest so it gets run --- synapse/rest/client/v2_alpha/room_keys.py | 14 ++++++--- synapse/storage/schema/delta/46/e2e_room_keys.sql | 38 ----------------------- synapse/storage/schema/delta/51/e2e_room_keys.sql | 38 +++++++++++++++++++++++ 3 files changed, 48 insertions(+), 42 deletions(-) delete mode 100644 synapse/storage/schema/delta/46/e2e_room_keys.sql create mode 100644 synapse/storage/schema/delta/51/e2e_room_keys.sql (limited to 'synapse/rest/client/v2_alpha') diff --git a/synapse/rest/client/v2_alpha/room_keys.py b/synapse/rest/client/v2_alpha/room_keys.py index 1ed18e986f..ea114bc8b4 100644 --- a/synapse/rest/client/v2_alpha/room_keys.py +++ b/synapse/rest/client/v2_alpha/room_keys.py @@ -17,7 +17,7 @@ import logging from twisted.internet import defer -from synapse.api.errors import SynapseError +from synapse.api.errors import SynapseError, Codes from synapse.http.servlet import ( RestServlet, parse_json_object_from_request, @@ -324,9 +324,15 @@ class RoomKeysVersionServlet(RestServlet): requester = yield self.auth.get_user_by_req(request, allow_guest=False) user_id = requester.user.to_string() - info = yield self.e2e_room_keys_handler.get_version_info( - user_id, version - ) + try: + info = yield self.e2e_room_keys_handler.get_version_info( + user_id, version + ) + except SynapseError as e: + if e.code == 404: + e.errcode = Codes.NOT_FOUND + e.msg = "No backup found" + raise e defer.returnValue((200, info)) @defer.inlineCallbacks diff --git a/synapse/storage/schema/delta/46/e2e_room_keys.sql b/synapse/storage/schema/delta/46/e2e_room_keys.sql deleted file mode 100644 index 4531fd56ee..0000000000 --- a/synapse/storage/schema/delta/46/e2e_room_keys.sql +++ /dev/null @@ -1,38 +0,0 @@ -/* Copyright 2017 New Vector Ltd - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - --- users' optionally backed up encrypted e2e sessions -CREATE TABLE e2e_room_keys ( - user_id TEXT NOT NULL, - room_id TEXT NOT NULL, - session_id TEXT NOT NULL, - version TEXT NOT NULL, - first_message_index INT, - forwarded_count INT, - is_verified BOOLEAN, - session_data TEXT NOT NULL -); - -CREATE UNIQUE INDEX e2e_room_keys_idx ON e2e_room_keys(user_id, room_id, session_id); - --- the metadata for each generation of encrypted e2e session backups -CREATE TABLE e2e_room_keys_versions ( - user_id TEXT NOT NULL, - version TEXT NOT NULL, - algorithm TEXT NOT NULL, - auth_data TEXT NOT NULL -); - -CREATE UNIQUE INDEX e2e_room_keys_versions_idx ON e2e_room_keys_versions(user_id, version); diff --git a/synapse/storage/schema/delta/51/e2e_room_keys.sql b/synapse/storage/schema/delta/51/e2e_room_keys.sql new file mode 100644 index 0000000000..4531fd56ee --- /dev/null +++ b/synapse/storage/schema/delta/51/e2e_room_keys.sql @@ -0,0 +1,38 @@ +/* Copyright 2017 New Vector Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +-- users' optionally backed up encrypted e2e sessions +CREATE TABLE e2e_room_keys ( + user_id TEXT NOT NULL, + room_id TEXT NOT NULL, + session_id TEXT NOT NULL, + version TEXT NOT NULL, + first_message_index INT, + forwarded_count INT, + is_verified BOOLEAN, + session_data TEXT NOT NULL +); + +CREATE UNIQUE INDEX e2e_room_keys_idx ON e2e_room_keys(user_id, room_id, session_id); + +-- the metadata for each generation of encrypted e2e session backups +CREATE TABLE e2e_room_keys_versions ( + user_id TEXT NOT NULL, + version TEXT NOT NULL, + algorithm TEXT NOT NULL, + auth_data TEXT NOT NULL +); + +CREATE UNIQUE INDEX e2e_room_keys_versions_idx ON e2e_room_keys_versions(user_id, version); -- cgit 1.5.1 From d3464ce708cd857216d88d3e67867f1cf0c8bf60 Mon Sep 17 00:00:00 2001 From: David Baker Date: Tue, 9 Oct 2018 10:33:59 +0100 Subject: isort --- synapse/rest/client/v2_alpha/room_keys.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'synapse/rest/client/v2_alpha') diff --git a/synapse/rest/client/v2_alpha/room_keys.py b/synapse/rest/client/v2_alpha/room_keys.py index ea114bc8b4..539893a5d6 100644 --- a/synapse/rest/client/v2_alpha/room_keys.py +++ b/synapse/rest/client/v2_alpha/room_keys.py @@ -17,7 +17,7 @@ import logging from twisted.internet import defer -from synapse.api.errors import SynapseError, Codes +from synapse.api.errors import Codes, SynapseError from synapse.http.servlet import ( RestServlet, parse_json_object_from_request, -- cgit 1.5.1 From 83e72bb2f0c6ef282190a378941c856afbb33c16 Mon Sep 17 00:00:00 2001 From: David Baker Date: Fri, 12 Oct 2018 11:26:18 +0100 Subject: PR feedback pt. 1 --- synapse/api/errors.py | 8 ------ synapse/handlers/e2e_room_keys.py | 41 ++++++++++++++++--------------- synapse/rest/client/v2_alpha/room_keys.py | 2 +- 3 files changed, 22 insertions(+), 29 deletions(-) (limited to 'synapse/rest/client/v2_alpha') diff --git a/synapse/api/errors.py b/synapse/api/errors.py index 0a6e78711f..48b903374d 100644 --- a/synapse/api/errors.py +++ b/synapse/api/errors.py @@ -348,14 +348,6 @@ class IncompatibleRoomVersionError(SynapseError): ) -def cs_exception(exception): - if isinstance(exception, CodeMessageException): - return exception.error_dict() - else: - logger.error("Unknown exception type: %s", type(exception)) - return {} - - def cs_error(msg, code=Codes.UNKNOWN, **kwargs): """ Utility method for constructing an error response for client-server interactions. diff --git a/synapse/handlers/e2e_room_keys.py b/synapse/handlers/e2e_room_keys.py index bf2a83cc31..4e3141dac8 100644 --- a/synapse/handlers/e2e_room_keys.py +++ b/synapse/handlers/e2e_room_keys.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -# Copyright 2017 New Vector Ltd +# Copyright 2017, 2018 New Vector Ltd # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -49,6 +49,11 @@ class E2eRoomKeysHandler(object): room, or a given session. See EndToEndRoomKeyStore.get_e2e_room_keys for full details. + Args: + user_id(str): the user whose keys we're getting + version(str): the version ID of the backup we're getting keys from + room_id(string): room ID to get keys for, for None to get keys for all rooms + session_id(string): session ID to get keys for, for None to get keys for all sessions Returns: A deferred list of dicts giving the session_data and message metadata for these room keys. @@ -72,6 +77,11 @@ class E2eRoomKeysHandler(object): room or a given session. See EndToEndRoomKeyStore.delete_e2e_room_keys for full details. + Args: + user_id(str): the user whose backup we're deleting + version(str): the version ID of the backup we're deleting + room_id(string): room ID to delete keys for, for None to delete keys for all rooms + session_id(string): session ID to delete keys for, for None to delete keys for all sessions Returns: A deferred of the deletion transaction """ @@ -118,24 +128,24 @@ class E2eRoomKeysHandler(object): # Check that the version we're trying to upload is the current version try: - version_info = yield self._get_version_info_unlocked(user_id) + version_info = yield self.store.get_e2e_room_keys_version_info(user_id) except StoreError as e: if e.code == 404: raise SynapseError(404, "Version '%s' not found" % (version,)) else: - raise e + raise if version_info['version'] != version: # Check that the version we're trying to upload actually exists try: - version_info = yield self._get_version_info_unlocked(user_id, version) + version_info = yield self.store.get_e2e_room_keys_version_info(user_id, version) # if we get this far, the version must exist raise RoomKeysVersionError(current_version=version_info['version']) except StoreError as e: if e.code == 404: raise SynapseError(404, "Version '%s' not found" % (version,)) else: - raise e + raise # go through the room_keys. # XXX: this should/could be done concurrently, given we're in a lock. @@ -168,9 +178,9 @@ class E2eRoomKeysHandler(object): if e.code == 404: pass else: - raise e + raise - if E2eRoomKeysHandler._should_replace_room_key(current_room_key, room_key): + if self._should_replace_room_key(current_room_key, room_key): yield self.store.set_e2e_room_key( user_id, version, room_id, session_id, room_key ) @@ -195,14 +205,14 @@ class E2eRoomKeysHandler(object): # purely for legibility. if room_key['is_verified'] and not current_room_key['is_verified']: - pass + return True elif ( room_key['first_message_index'] < current_room_key['first_message_index'] ): - pass + return True elif room_key['forwarded_count'] < current_room_key['forwarded_count']: - pass + return True else: return False return True @@ -256,18 +266,9 @@ class E2eRoomKeysHandler(object): """ with (yield self._upload_linearizer.queue(user_id)): - res = yield self._get_version_info_unlocked(user_id, version) + res = yield self.store.get_e2e_room_keys_version_info(user_id, version) defer.returnValue(res) - @defer.inlineCallbacks - def _get_version_info_unlocked(self, user_id, version=None): - """Get the info about a given version of the user's backup - without obtaining the upload_linearizer lock. For params see get_version_info - """ - - results = yield self.store.get_e2e_room_keys_version_info(user_id, version) - defer.returnValue(results) - @defer.inlineCallbacks def delete_version(self, user_id, version=None): """Deletes a given version of the user's e2e_room_keys backup diff --git a/synapse/rest/client/v2_alpha/room_keys.py b/synapse/rest/client/v2_alpha/room_keys.py index 539893a5d6..4807170ea6 100644 --- a/synapse/rest/client/v2_alpha/room_keys.py +++ b/synapse/rest/client/v2_alpha/room_keys.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -# Copyright 2017 New Vector Ltd +# Copyright 2017, 2018 New Vector Ltd # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. -- cgit 1.5.1 From 86ef9760a786e7de128120c11d623e3303178505 Mon Sep 17 00:00:00 2001 From: David Baker Date: Fri, 12 Oct 2018 11:35:08 +0100 Subject: Split /room_keys/version into 2 servlets --- synapse/rest/client/v2_alpha/room_keys.py | 30 +++++++++++++++++++++--------- 1 file changed, 21 insertions(+), 9 deletions(-) (limited to 'synapse/rest/client/v2_alpha') diff --git a/synapse/rest/client/v2_alpha/room_keys.py b/synapse/rest/client/v2_alpha/room_keys.py index 4807170ea6..aee6419179 100644 --- a/synapse/rest/client/v2_alpha/room_keys.py +++ b/synapse/rest/client/v2_alpha/room_keys.py @@ -240,9 +240,9 @@ class RoomKeysServlet(RestServlet): defer.returnValue((200, {})) -class RoomKeysVersionServlet(RestServlet): +class RoomKeysNewVersionServlet(RestServlet): PATTERNS = client_v2_patterns( - "/room_keys/version(/(?P[^/]+))?$" + "/room_keys/version$" ) def __init__(self, hs): @@ -250,12 +250,12 @@ class RoomKeysVersionServlet(RestServlet): Args: hs (synapse.server.HomeServer): server """ - super(RoomKeysVersionServlet, self).__init__() + super(RoomKeysNewVersionServlet, self).__init__() self.auth = hs.get_auth() self.e2e_room_keys_handler = hs.get_e2e_room_keys_handler() @defer.inlineCallbacks - def on_POST(self, request, version): + def on_POST(self, request): """ Create a new backup version for this user's room_keys with the given info. The version is allocated by the server and returned to the user @@ -285,10 +285,6 @@ class RoomKeysVersionServlet(RestServlet): "version": 12345 } """ - - if version: - raise SynapseError(405, "Cannot POST to a specific version") - requester = yield self.auth.get_user_by_req(request, allow_guest=False) user_id = requester.user.to_string() info = parse_json_object_from_request(request) @@ -301,6 +297,20 @@ class RoomKeysVersionServlet(RestServlet): # we deliberately don't have a PUT /version, as these things really should # be immutable to avoid people footgunning +class RoomKeysVersionServlet(RestServlet): + PATTERNS = client_v2_patterns( + "/room_keys/version(/(?P[^/]+))?$" + ) + + def __init__(self, hs): + """ + Args: + hs (synapse.server.HomeServer): server + """ + super(RoomKeysVersionServlet, self).__init__() + self.auth = hs.get_auth() + self.e2e_room_keys_handler = hs.get_e2e_room_keys_handler() + @defer.inlineCallbacks def on_GET(self, request, version): """ @@ -320,7 +330,6 @@ class RoomKeysVersionServlet(RestServlet): "auth_data": "dGhpcyBzaG91bGQgYWN0dWFsbHkgYmUgZW5jcnlwdGVkIGpzb24K" } """ - requester = yield self.auth.get_user_by_req(request, allow_guest=False) user_id = requester.user.to_string() @@ -346,6 +355,8 @@ class RoomKeysVersionServlet(RestServlet): HTTP/1.1 200 OK {} """ + if version is None: + raise SynapseError(400, "No version specified to delete") requester = yield self.auth.get_user_by_req(request, allow_guest=False) user_id = requester.user.to_string() @@ -359,3 +370,4 @@ class RoomKeysVersionServlet(RestServlet): def register_servlets(hs, http_server): RoomKeysServlet(hs).register(http_server) RoomKeysVersionServlet(hs).register(http_server) + RoomKeysNewVersionServlet(hs).register(http_server) -- cgit 1.5.1 From bddfad253a2ca8ea0645d10e89055c4ce2ff1c38 Mon Sep 17 00:00:00 2001 From: David Baker Date: Fri, 12 Oct 2018 11:48:02 +0100 Subject: Don't mangle exceptions --- synapse/rest/client/v2_alpha/room_keys.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) (limited to 'synapse/rest/client/v2_alpha') diff --git a/synapse/rest/client/v2_alpha/room_keys.py b/synapse/rest/client/v2_alpha/room_keys.py index aee6419179..8d006d819d 100644 --- a/synapse/rest/client/v2_alpha/room_keys.py +++ b/synapse/rest/client/v2_alpha/room_keys.py @@ -339,9 +339,7 @@ class RoomKeysVersionServlet(RestServlet): ) except SynapseError as e: if e.code == 404: - e.errcode = Codes.NOT_FOUND - e.msg = "No backup found" - raise e + raise SynapseError(404, "No backup found", Codes.NOT_FOUND) defer.returnValue((200, info)) @defer.inlineCallbacks @@ -356,7 +354,7 @@ class RoomKeysVersionServlet(RestServlet): {} """ if version is None: - raise SynapseError(400, "No version specified to delete") + raise SynapseError(400, "No version specified to delete", Codes.NOT_FOUND) requester = yield self.auth.get_user_by_req(request, allow_guest=False) user_id = requester.user.to_string() -- cgit 1.5.1 From a45f2c3a002dd50ba4dc6dcbe45a51f2da4057b2 Mon Sep 17 00:00:00 2001 From: David Baker Date: Fri, 12 Oct 2018 14:33:55 +0100 Subject: missed one --- synapse/rest/client/v2_alpha/room_keys.py | 1 + 1 file changed, 1 insertion(+) (limited to 'synapse/rest/client/v2_alpha') diff --git a/synapse/rest/client/v2_alpha/room_keys.py b/synapse/rest/client/v2_alpha/room_keys.py index 8d006d819d..45b5817d8b 100644 --- a/synapse/rest/client/v2_alpha/room_keys.py +++ b/synapse/rest/client/v2_alpha/room_keys.py @@ -297,6 +297,7 @@ class RoomKeysNewVersionServlet(RestServlet): # we deliberately don't have a PUT /version, as these things really should # be immutable to avoid people footgunning + class RoomKeysVersionServlet(RestServlet): PATTERNS = client_v2_patterns( "/room_keys/version(/(?P[^/]+))?$" -- cgit 1.5.1 From ef771cc4c2d988e5188ba7e75df9adebb0ebafe1 Mon Sep 17 00:00:00 2001 From: Richard van der Hoff Date: Wed, 24 Oct 2018 10:35:01 +0100 Subject: Fix a number of flake8 errors Broadly three things here: * disable W504 which seems a bit whacko * remove a bunch of `as e` expressions from exception handlers that don't use them * use `r""` for strings which include backslashes Also, we don't use pep8 any more, so we can get rid of the duplicate config there. --- changelog.d/4082.misc | 1 + scripts-dev/tail-synapse.py | 2 +- scripts/synapse_port_db | 3 ++- setup.cfg | 17 ++++++++--------- synapse/config/repository.py | 2 +- synapse/crypto/keyclient.py | 2 +- synapse/federation/federation_server.py | 2 +- synapse/rest/client/v2_alpha/auth.py | 2 +- synapse/rest/media/v1/preview_url_resource.py | 2 +- synapse/storage/registration.py | 2 +- tests/config/test_generate.py | 2 +- tests/events/test_utils.py | 4 ++-- 12 files changed, 21 insertions(+), 20 deletions(-) create mode 100644 changelog.d/4082.misc (limited to 'synapse/rest/client/v2_alpha') diff --git a/changelog.d/4082.misc b/changelog.d/4082.misc new file mode 100644 index 0000000000..a81faf5e9b --- /dev/null +++ b/changelog.d/4082.misc @@ -0,0 +1 @@ +Clean up some bits of code which were flagged by the linter diff --git a/scripts-dev/tail-synapse.py b/scripts-dev/tail-synapse.py index 1a36b94038..7c9985d9f0 100644 --- a/scripts-dev/tail-synapse.py +++ b/scripts-dev/tail-synapse.py @@ -48,7 +48,7 @@ def main(): row.name: row.position for row in replicate(server, {"streams": "-1"})["streams"].rows } - except requests.exceptions.ConnectionError as e: + except requests.exceptions.ConnectionError: time.sleep(0.1) while True: diff --git a/scripts/synapse_port_db b/scripts/synapse_port_db index 2f6e69e552..3c7b606323 100755 --- a/scripts/synapse_port_db +++ b/scripts/synapse_port_db @@ -501,7 +501,8 @@ class Porter(object): try: yield self.postgres_store.runInteraction("alter_table", alter_table) - except Exception as e: + except Exception: + # On Error Resume Next pass yield self.postgres_store.runInteraction( diff --git a/setup.cfg b/setup.cfg index 52feaa9cc7..b6b4aa740d 100644 --- a/setup.cfg +++ b/setup.cfg @@ -14,17 +14,16 @@ ignore = pylint.cfg tox.ini -[pep8] -max-line-length = 90 -# W503 requires that binary operators be at the end, not start, of lines. Erik -# doesn't like it. E203 is contrary to PEP8. E731 is silly. -ignore = W503,E203,E731 - [flake8] -# note that flake8 inherits the "ignore" settings from "pep8" (because it uses -# pep8 to do those checks), but not the "max-line-length" setting max-line-length = 90 -ignore=W503,E203,E731 + +# see https://pycodestyle.readthedocs.io/en/latest/intro.html#error-codes +# for error codes. The ones we ignore are: +# W503: line break before binary operator +# W504: line break after binary operator +# E203: whitespace before ':' (which is contrary to pep8?) +# E731: do not assign a lambda expression, use a def +ignore=W503,W504,E203,E731 [isort] line_length = 89 diff --git a/synapse/config/repository.py b/synapse/config/repository.py index fc909c1fac..06c62ab62c 100644 --- a/synapse/config/repository.py +++ b/synapse/config/repository.py @@ -178,7 +178,7 @@ class ContentRepositoryConfig(Config): def default_config(self, **kwargs): media_store = self.default_path("media_store") uploads_path = self.default_path("uploads") - return """ + return r""" # Directory where uploaded images and attachments are stored. media_store_path: "%(media_store)s" diff --git a/synapse/crypto/keyclient.py b/synapse/crypto/keyclient.py index 57d4665e84..080c81f14b 100644 --- a/synapse/crypto/keyclient.py +++ b/synapse/crypto/keyclient.py @@ -55,7 +55,7 @@ def fetch_server_key(server_name, tls_client_options_factory, path=KEY_API_V1): raise IOError("Cannot get key for %r" % server_name) except (ConnectError, DomainError) as e: logger.warn("Error getting key for %r: %s", server_name, e) - except Exception as e: + except Exception: logger.exception("Error getting key for %r", server_name) raise IOError("Cannot get key for %r" % server_name) diff --git a/synapse/federation/federation_server.py b/synapse/federation/federation_server.py index 4efe95faa4..af0107a46e 100644 --- a/synapse/federation/federation_server.py +++ b/synapse/federation/federation_server.py @@ -800,7 +800,7 @@ class FederationHandlerRegistry(object): yield handler(origin, content) except SynapseError as e: logger.info("Failed to handle edu %r: %r", edu_type, e) - except Exception as e: + except Exception: logger.exception("Failed to handle edu %r", edu_type) def on_query(self, query_type, args): diff --git a/synapse/rest/client/v2_alpha/auth.py b/synapse/rest/client/v2_alpha/auth.py index bd8b5f4afa..693b303881 100644 --- a/synapse/rest/client/v2_alpha/auth.py +++ b/synapse/rest/client/v2_alpha/auth.py @@ -99,7 +99,7 @@ class AuthRestServlet(RestServlet): cannot be handled in the normal flow (with requests to the same endpoint). Current use is for web fallback auth. """ - PATTERNS = client_v2_patterns("/auth/(?P[\w\.]*)/fallback/web") + PATTERNS = client_v2_patterns(r"/auth/(?P[\w\.]*)/fallback/web") def __init__(self, hs): super(AuthRestServlet, self).__init__() diff --git a/synapse/rest/media/v1/preview_url_resource.py b/synapse/rest/media/v1/preview_url_resource.py index 8c892ff187..1a7bfd6b56 100644 --- a/synapse/rest/media/v1/preview_url_resource.py +++ b/synapse/rest/media/v1/preview_url_resource.py @@ -674,7 +674,7 @@ def summarize_paragraphs(text_nodes, min_size=200, max_size=500): # This splits the paragraph into words, but keeping the # (preceeding) whitespace intact so we can easily concat # words back together. - for match in re.finditer("\s*\S+", description): + for match in re.finditer(r"\s*\S+", description): word = match.group() # Keep adding words while the total length is less than diff --git a/synapse/storage/registration.py b/synapse/storage/registration.py index 26b429e307..2dd14aba1c 100644 --- a/synapse/storage/registration.py +++ b/synapse/storage/registration.py @@ -567,7 +567,7 @@ class RegistrationStore(RegistrationWorkerStore, def _find_next_generated_user_id(txn): txn.execute("SELECT name FROM users") - regex = re.compile("^@(\d+):") + regex = re.compile(r"^@(\d+):") found = set() diff --git a/tests/config/test_generate.py b/tests/config/test_generate.py index f88d28a19d..0c23068bcf 100644 --- a/tests/config/test_generate.py +++ b/tests/config/test_generate.py @@ -67,6 +67,6 @@ class ConfigGenerationTestCase(unittest.TestCase): with open(log_config_file) as f: config = f.read() # find the 'filename' line - matches = re.findall("^\s*filename:\s*(.*)$", config, re.M) + matches = re.findall(r"^\s*filename:\s*(.*)$", config, re.M) self.assertEqual(1, len(matches)) self.assertEqual(matches[0], expected) diff --git a/tests/events/test_utils.py b/tests/events/test_utils.py index ff217ca8b9..d0cc492deb 100644 --- a/tests/events/test_utils.py +++ b/tests/events/test_utils.py @@ -156,7 +156,7 @@ class SerializeEventTestCase(unittest.TestCase): room_id="!foo:bar", content={"key.with.dots": {}}, ), - ["content.key\.with\.dots"], + [r"content.key\.with\.dots"], ), {"content": {"key.with.dots": {}}}, ) @@ -172,7 +172,7 @@ class SerializeEventTestCase(unittest.TestCase): "nested.dot.key": {"leaf.key": 42, "not_me_either": 1}, }, ), - ["content.nested\.dot\.key.leaf\.key"], + [r"content.nested\.dot\.key.leaf\.key"], ), {"content": {"nested.dot.key": {"leaf.key": 42}}}, ) -- cgit 1.5.1