From 8df16a8aeef3e169ca8795abe5fd9baac650f3bb Mon Sep 17 00:00:00 2001 From: Brendan Abolivier Date: Tue, 30 Apr 2019 14:36:35 +0100 Subject: Move lookup endpoint to CS API (and s/is_server/id_server/) --- synapse/api/urls.py | 1 - synapse/app/homeserver.py | 5 --- synapse/config/server.py | 1 - synapse/rest/client/v2_alpha/account.py | 32 +++++++++++++++ synapse/rest/identity/__init__.py | 14 ------- synapse/rest/identity/v1/__init__.py | 24 ----------- synapse/rest/identity/v1/lookup.py | 70 --------------------------------- 7 files changed, 32 insertions(+), 115 deletions(-) delete mode 100644 synapse/rest/identity/__init__.py delete mode 100644 synapse/rest/identity/v1/__init__.py delete mode 100644 synapse/rest/identity/v1/lookup.py diff --git a/synapse/api/urls.py b/synapse/api/urls.py index e33ea0cf65..cb71d80875 100644 --- a/synapse/api/urls.py +++ b/synapse/api/urls.py @@ -33,7 +33,6 @@ CONTENT_REPO_PREFIX = "/_matrix/content" SERVER_KEY_V2_PREFIX = "/_matrix/key/v2" MEDIA_PREFIX = "/_matrix/media/r0" LEGACY_MEDIA_PREFIX = "/_matrix/media/v1" -IDENTITY_PREFIX = "/_matrix/identity/api/v1" class ConsentURIBuilder(object): diff --git a/synapse/app/homeserver.py b/synapse/app/homeserver.py index 06aa582129..79be977ea6 100755 --- a/synapse/app/homeserver.py +++ b/synapse/app/homeserver.py @@ -40,7 +40,6 @@ from synapse import events from synapse.api.urls import ( CONTENT_REPO_PREFIX, FEDERATION_PREFIX, - IDENTITY_PREFIX, LEGACY_MEDIA_PREFIX, MEDIA_PREFIX, SERVER_KEY_V2_PREFIX, @@ -63,7 +62,6 @@ from synapse.python_dependencies import check_requirements from synapse.replication.http import REPLICATION_PREFIX, ReplicationRestResource from synapse.replication.tcp.resource import ReplicationStreamProtocolFactory from synapse.rest import ClientRestResource -from synapse.rest.identity.v1 import IdentityApiV1Resource from synapse.rest.key.v2 import KeyApiV2Resource from synapse.rest.media.v0.content_repository import ContentRepoResource from synapse.rest.well_known import WellKnownResource @@ -229,9 +227,6 @@ class SynapseHomeServer(HomeServer): "'media' resource conflicts with enable_media_repo=False", ) - if name in ["identity"]: - resources[IDENTITY_PREFIX] = IdentityApiV1Resource(self) - if name in ["keys", "federation"]: resources[SERVER_KEY_V2_PREFIX] = KeyApiV2Resource(self) diff --git a/synapse/config/server.py b/synapse/config/server.py index ac44b17a09..c5e5679d52 100644 --- a/synapse/config/server.py +++ b/synapse/config/server.py @@ -573,7 +573,6 @@ KNOWN_RESOURCES = ( 'replication', 'static', 'webclient', - 'identity', ) diff --git a/synapse/rest/client/v2_alpha/account.py b/synapse/rest/client/v2_alpha/account.py index 7e5884df0c..752ea265bd 100644 --- a/synapse/rest/client/v2_alpha/account.py +++ b/synapse/rest/client/v2_alpha/account.py @@ -480,6 +480,38 @@ class ThreepidDeleteRestServlet(RestServlet): ) +class ThreepidLookupRestServlet(RestServlet): + PATTERNS = client_v2_patterns("/account/3pid/lookup$") + + def __init__(self, hs): + super(ThreepidLookupRestServlet, self).__init__() + self.config = hs.config + self.auth = hs.get_auth() + self.identity_handler = IdentityHandler(hs) + + @defer.inlineCallbacks + def on_GET(self, request): + """Proxy a /_matrix/identity/api/v1/lookup request to an identity + server + """ + yield self.auth.get_user_by_req(request) + + # Extract query parameters + query_params = request.args + assert_params_in_dict(query_params, [b"medium", b"address", b"id_server"]) + + # Retrieve address and medium from the request parameters + medium = parse_string(request, "medium") + address = parse_string(request, "address") + id_server = parse_string(request, "id_server") + + # Proxy the request to the identity server. lookup_3pid handles checking + # if the lookup is allowed so we don't need to do it here. + ret = yield self.identity_handler.lookup_3pid(id_server, medium, address) + + respond_with_json(200, ret) + + class WhoamiRestServlet(RestServlet): PATTERNS = client_v2_patterns("/account/whoami$") diff --git a/synapse/rest/identity/__init__.py b/synapse/rest/identity/__init__.py deleted file mode 100644 index 1453d04571..0000000000 --- a/synapse/rest/identity/__init__.py +++ /dev/null @@ -1,14 +0,0 @@ -# -*- coding: utf-8 -*- -# Copyright 2019 New Vector Ltd -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. diff --git a/synapse/rest/identity/v1/__init__.py b/synapse/rest/identity/v1/__init__.py deleted file mode 100644 index 09057ea16b..0000000000 --- a/synapse/rest/identity/v1/__init__.py +++ /dev/null @@ -1,24 +0,0 @@ -# -*- coding: utf-8 -*- -# Copyright 2019 New Vector Ltd -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from twisted.web.resource import Resource - -from .lookup import IdentityLookup - - -class IdentityApiV1Resource(Resource): - def __init__(self, hs): - Resource.__init__(self) - self.putChild(b"lookup", IdentityLookup(hs)) diff --git a/synapse/rest/identity/v1/lookup.py b/synapse/rest/identity/v1/lookup.py deleted file mode 100644 index c2d18fbb63..0000000000 --- a/synapse/rest/identity/v1/lookup.py +++ /dev/null @@ -1,70 +0,0 @@ -# -*- coding: utf-8 -*- -# Copyright 2019 New Vector Ltd -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - - -import logging - -from twisted.internet import defer -from twisted.web.resource import Resource -from twisted.web.server import NOT_DONE_YET - -from synapse.api.errors import SynapseError -from synapse.handlers.identity import IdentityHandler -from synapse.http.server import respond_with_json, wrap_json_request_handler -from synapse.http.servlet import assert_params_in_dict, parse_string - -logger = logging.getLogger(__name__) - - -class IdentityLookup(Resource): - isLeaf = True - - def __init__(self, hs): - self.config = hs.config - self.auth = hs.get_auth() - self.identity_handler = IdentityHandler(hs) - Resource.__init__(self) - - def render_GET(self, request): - self.async_render_GET(request) - return NOT_DONE_YET - - @wrap_json_request_handler - @defer.inlineCallbacks - def async_render_GET(self, request): - """Proxy a /_matrix/identity/api/v1/lookup request to an identity - server - """ - yield self.auth.get_user_by_req(request, allow_guest=True) - - if not self.config.enable_3pid_lookup: - raise SynapseError( - 403, - "Looking up third-party identifiers is denied from this server" - ) - - # Extract query parameters - query_params = request.args - assert_params_in_dict(query_params, [b"medium", b"address", b"is_server"]) - - # Retrieve address and medium from the request parameters - medium = parse_string(request, "medium") - address = parse_string(request, "address") - is_server = parse_string(request, "is_server") - - # Proxy the request to the identity server - ret = yield self.identity_handler.lookup_3pid(is_server, medium, address) - - respond_with_json(request, 200, ret, send_cors=True) -- cgit 1.5.1 From d296cdc9ddb799e8354dae4308a01b0984933186 Mon Sep 17 00:00:00 2001 From: Brendan Abolivier Date: Tue, 30 Apr 2019 15:15:02 +0100 Subject: Add bulk lookup --- synapse/handlers/identity.py | 46 ++++++++++++++++++++++++++++++++- synapse/rest/client/v2_alpha/account.py | 27 ++++++++++++++++++- 2 files changed, 71 insertions(+), 2 deletions(-) diff --git a/synapse/handlers/identity.py b/synapse/handlers/identity.py index d2039e2825..910f572a1d 100644 --- a/synapse/handlers/identity.py +++ b/synapse/handlers/identity.py @@ -347,7 +347,7 @@ class IdentityHandler(BaseHandler): Returns: Deferred[dict]: The result of the lookup. See - https://matrix.org/docs/spec/identity_service/r0.1.0.html#id15 + https://matrix.org/docs/spec/identity_service/r0.1.0.html#association-lookup for details """ if not self._enable_lookup: @@ -380,6 +380,50 @@ class IdentityHandler(BaseHandler): defer.returnValue(data) + @defer.inlineCallbacks + def bulk_lookup_3pid(self, id_server, threepids): + """Looks up a 3pid in the passed identity server. + + Args: + id_server (str): The server name (including port, if required) + of the identity server to use. + threepids ([[str, str]]): The third party identifiers to lookup, as + a list of 2-string sized lists ([medium, address]). + + Returns: + Deferred[dict]: The result of the lookup. See + https://matrix.org/docs/spec/identity_service/r0.1.0.html#association-lookup + for details + """ + if not self._enable_lookup: + raise AuthError( + 403, "Looking up third-party identifiers is denied from this server", + ) + + target = self.rewrite_identity_server_urls.get(id_server, id_server) + + try: + data = yield self.http_client.get_json( + "https://%s/_matrix/identity/api/v1/lookup" % (target,), + { + "threepids": threepids, + } + ) + + if "mxid" in data: + if "signatures" not in data: + raise AuthError(401, "No signatures on 3pid bindings") + yield self._verify_any_signature(data, id_server) + + except HttpResponseException as e: + logger.info("Proxied lookup failed: %r", e) + raise e.to_synapse_error() + except IOError as e: + logger.info("Failed to contact %r: %s", id_server, e) + raise ProxiedRequestError(503, "Failed to contact homeserver") + + defer.returnValue(data) + @defer.inlineCallbacks def _verify_any_signature(self, data, server_hostname): if server_hostname not in data["signatures"]: diff --git a/synapse/rest/client/v2_alpha/account.py b/synapse/rest/client/v2_alpha/account.py index 752ea265bd..de9e2cd5f1 100644 --- a/synapse/rest/client/v2_alpha/account.py +++ b/synapse/rest/client/v2_alpha/account.py @@ -1,7 +1,7 @@ # -*- coding: utf-8 -*- # Copyright 2015, 2016 OpenMarket Ltd # Copyright 2017 Vector Creations Ltd -# Copyright 2018 New Vector Ltd +# Copyright 2018, 2019 New Vector Ltd # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -512,6 +512,31 @@ class ThreepidLookupRestServlet(RestServlet): respond_with_json(200, ret) +class ThreepidBulkLookupRestServlet(RestServlet): + PATTERNS = client_v2_patterns("/account/3pid/bulk_lookup$") + + def __init__(self, hs): + super(ThreepidLookupRestServlet, self).__init__() + self.config = hs.config + self.auth = hs.get_auth() + self.identity_handler = IdentityHandler(hs) + + @defer.inlineCallbacks + def on_GET(self, request): + """Proxy a /_matrix/identity/api/v1/bulk_lookup request to an identity + server + """ + yield self.auth.get_user_by_req(request) + + body = parse_json_object_from_request(request) + + # Proxy the request to the identity server. lookup_3pid handles checking + # if the lookup is allowed so we don't need to do it here. + ret = yield self.identity_handler.bulk_lookup_3pid(id_server, body["threepids"]) + + respond_with_json(200, ret) + + class WhoamiRestServlet(RestServlet): PATTERNS = client_v2_patterns("/account/whoami$") -- cgit 1.5.1 From 2d979e639da9e6757b58cf10f8a47e58105c3720 Mon Sep 17 00:00:00 2001 From: Brendan Abolivier Date: Tue, 30 Apr 2019 16:51:50 +0100 Subject: Register servlets --- synapse/rest/client/v2_alpha/account.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/synapse/rest/client/v2_alpha/account.py b/synapse/rest/client/v2_alpha/account.py index de9e2cd5f1..cdf499af6d 100644 --- a/synapse/rest/client/v2_alpha/account.py +++ b/synapse/rest/client/v2_alpha/account.py @@ -560,4 +560,6 @@ def register_servlets(hs, http_server): MsisdnThreepidRequestTokenRestServlet(hs).register(http_server) ThreepidRestServlet(hs).register(http_server) ThreepidDeleteRestServlet(hs).register(http_server) + ThreepidLookupRestServlet(hs).register(http_server) + ThreepidBulkLookupRestServlet(hs).register(http_server) WhoamiRestServlet(hs).register(http_server) -- cgit 1.5.1 From 34bbbe81a605655c3d2a548a34a94a93b621ad48 Mon Sep 17 00:00:00 2001 From: Brendan Abolivier Date: Tue, 30 Apr 2019 16:59:46 +0100 Subject: Fix method --- synapse/rest/client/v2_alpha/account.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/synapse/rest/client/v2_alpha/account.py b/synapse/rest/client/v2_alpha/account.py index cdf499af6d..f3707068bb 100644 --- a/synapse/rest/client/v2_alpha/account.py +++ b/synapse/rest/client/v2_alpha/account.py @@ -522,7 +522,7 @@ class ThreepidBulkLookupRestServlet(RestServlet): self.identity_handler = IdentityHandler(hs) @defer.inlineCallbacks - def on_GET(self, request): + def on_POST(self, request): """Proxy a /_matrix/identity/api/v1/bulk_lookup request to an identity server """ -- cgit 1.5.1 From 70da5202ba2c88333b18c1968e446d3dd0268282 Mon Sep 17 00:00:00 2001 From: Brendan Abolivier Date: Tue, 30 Apr 2019 17:27:47 +0100 Subject: Fixes --- synapse/rest/client/v2_alpha/account.py | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/synapse/rest/client/v2_alpha/account.py b/synapse/rest/client/v2_alpha/account.py index f3707068bb..f7110f3b40 100644 --- a/synapse/rest/client/v2_alpha/account.py +++ b/synapse/rest/client/v2_alpha/account.py @@ -26,6 +26,7 @@ from synapse.http.servlet import ( RestServlet, assert_params_in_dict, parse_json_object_from_request, + parse_string, ) from synapse.types import UserID from synapse.util.msisdn import phone_number_to_msisdn @@ -487,7 +488,7 @@ class ThreepidLookupRestServlet(RestServlet): super(ThreepidLookupRestServlet, self).__init__() self.config = hs.config self.auth = hs.get_auth() - self.identity_handler = IdentityHandler(hs) + self.identity_handler = hs.get_handlers().identity_handler @defer.inlineCallbacks def on_GET(self, request): @@ -516,10 +517,10 @@ class ThreepidBulkLookupRestServlet(RestServlet): PATTERNS = client_v2_patterns("/account/3pid/bulk_lookup$") def __init__(self, hs): - super(ThreepidLookupRestServlet, self).__init__() + super(ThreepidBulkLookupRestServlet, self).__init__() self.config = hs.config self.auth = hs.get_auth() - self.identity_handler = IdentityHandler(hs) + self.identity_handler = hs.get_handlers().identity_handler @defer.inlineCallbacks def on_POST(self, request): @@ -532,7 +533,9 @@ class ThreepidBulkLookupRestServlet(RestServlet): # Proxy the request to the identity server. lookup_3pid handles checking # if the lookup is allowed so we don't need to do it here. - ret = yield self.identity_handler.bulk_lookup_3pid(id_server, body["threepids"]) + ret = yield self.identity_handler.bulk_lookup_3pid( + body["id_server"], body["threepids"], + ) respond_with_json(200, ret) -- cgit 1.5.1 From 909ceecc28fc3815971f1dbcebb160f0ad13ab13 Mon Sep 17 00:00:00 2001 From: Brendan Abolivier Date: Tue, 30 Apr 2019 17:27:58 +0100 Subject: Tests --- tests/rest/client/test_identity.py | 41 +++++++++++++++++++++++++++++++++----- 1 file changed, 36 insertions(+), 5 deletions(-) diff --git a/tests/rest/client/test_identity.py b/tests/rest/client/test_identity.py index 7e8b95887f..14756a31b7 100644 --- a/tests/rest/client/test_identity.py +++ b/tests/rest/client/test_identity.py @@ -16,6 +16,7 @@ import json from synapse.rest.client.v1 import admin, login, room +from synapse.rest.client.v2_alpha import account from tests import unittest @@ -23,6 +24,7 @@ from tests import unittest class IdentityTestCase(unittest.HomeserverTestCase): servlets = [ + account.register_servlets, admin.register_servlets, room.register_servlets, login.register_servlets, @@ -36,12 +38,13 @@ class IdentityTestCase(unittest.HomeserverTestCase): return self.hs - def test_3pid_lookup_disabled(self): - self.register_user("kermit", "monkey") - tok = self.login("kermit", "monkey") + def prepare(self, reactor, clock, hs): + self.user_id = self.register_user("kermit", "monkey") + self.tok = self.login("kermit", "monkey") + def test_3pid_invite_disabled(self): request, channel = self.make_request( - b"POST", "/createRoom", b"{}", access_token=tok, + b"POST", "/createRoom", b"{}", access_token=self.tok, ) self.render(request) self.assertEquals(channel.result["code"], b"200", channel.result) @@ -57,7 +60,35 @@ class IdentityTestCase(unittest.HomeserverTestCase): "/rooms/%s/invite" % (room_id) ).encode('ascii') request, channel = self.make_request( - b"POST", request_url, request_data, access_token=tok, + b"POST", request_url, request_data, access_token=self.tok, ) self.render(request) self.assertEquals(channel.result["code"], b"403", channel.result) + + def test_3pid_lookup_disabled(self): + url = "/_matrix/client/unstable/account/3pid/lookup?id_server=testis&medium=email&address=foo@bar.baz" + request, channel = self.make_request("GET", url, access_token=self.tok) + self.render(request) + self.assertEqual(channel.result["code"], b"403", channel.result) + + def test_3pid_bulk_lookup_disabled(self): + url = "/_matrix/client/unstable/account/3pid/bulk_lookup" + data = { + "id_server": "testis", + "threepids": [ + [ + "email", + "foo@bar.baz" + ], + [ + "email", + "john.doe@matrix.org" + ] + ] + } + request_data = json.dumps(data) + request, channel = self.make_request( + "POST", url, request_data, access_token=self.tok, + ) + self.render(request) + self.assertEqual(channel.result["code"], b"403", channel.result) -- cgit 1.5.1 From 3a9c405a0fc36eed25d53a0beaeb485b341bc7f1 Mon Sep 17 00:00:00 2001 From: Brendan Abolivier Date: Tue, 30 Apr 2019 18:22:42 +0100 Subject: Fix url and method --- synapse/handlers/identity.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/synapse/handlers/identity.py b/synapse/handlers/identity.py index 910f572a1d..23cc161798 100644 --- a/synapse/handlers/identity.py +++ b/synapse/handlers/identity.py @@ -403,8 +403,8 @@ class IdentityHandler(BaseHandler): target = self.rewrite_identity_server_urls.get(id_server, id_server) try: - data = yield self.http_client.get_json( - "https://%s/_matrix/identity/api/v1/lookup" % (target,), + data = yield self.http_client.post_json( + "https://%s/_matrix/identity/api/v1/bulk_lookup" % (target,), { "threepids": threepids, } -- cgit 1.5.1 From 96bd70f6d03cff8c88e05ccb228671fce9cbfe15 Mon Sep 17 00:00:00 2001 From: Brendan Abolivier Date: Tue, 30 Apr 2019 18:22:52 +0100 Subject: Bunch of tests --- tests/rest/client/test_identity.py | 126 ++++++++++++++++++++++++++++++++++++- 1 file changed, 124 insertions(+), 2 deletions(-) diff --git a/tests/rest/client/test_identity.py b/tests/rest/client/test_identity.py index 14756a31b7..c1b1b11202 100644 --- a/tests/rest/client/test_identity.py +++ b/tests/rest/client/test_identity.py @@ -15,13 +15,17 @@ import json +from mock import Mock + +from twisted.internet import defer + from synapse.rest.client.v1 import admin, login, room from synapse.rest.client.v2_alpha import account from tests import unittest -class IdentityTestCase(unittest.HomeserverTestCase): +class IdentityDisabledTestCase(unittest.HomeserverTestCase): servlets = [ account.register_servlets, @@ -66,7 +70,8 @@ class IdentityTestCase(unittest.HomeserverTestCase): self.assertEquals(channel.result["code"], b"403", channel.result) def test_3pid_lookup_disabled(self): - url = "/_matrix/client/unstable/account/3pid/lookup?id_server=testis&medium=email&address=foo@bar.baz" + url = ("/_matrix/client/unstable/account/3pid/lookup" + "?id_server=testis&medium=email&address=foo@bar.baz") request, channel = self.make_request("GET", url, access_token=self.tok) self.render(request) self.assertEqual(channel.result["code"], b"403", channel.result) @@ -92,3 +97,120 @@ class IdentityTestCase(unittest.HomeserverTestCase): ) self.render(request) self.assertEqual(channel.result["code"], b"403", channel.result) + + +class IdentityEnabledTestCase(unittest.HomeserverTestCase): + + servlets = [ + account.register_servlets, + admin.register_servlets, + room.register_servlets, + login.register_servlets, + ] + + def make_homeserver(self, reactor, clock): + + config = self.default_config() + config.enable_3pid_lookup = True + + mock_http_client = Mock(spec=[ + "get_json", + "post_json", + ]) + mock_http_client.get_json.return_value = defer.succeed((200, "{}")) + mock_http_client.post_json.return_value = defer.succeed((200, "{}")) + + self.hs = self.setup_test_homeserver( + config=config, + simple_http_client=mock_http_client, + ) + + return self.hs + + def prepare(self, reactor, clock, hs): + self.user_id = self.register_user("kermit", "monkey") + self.tok = self.login("kermit", "monkey") + + def test_3pid_invite_enabled(self): + request, channel = self.make_request( + b"POST", "/createRoom", b"{}", access_token=self.tok, + ) + self.render(request) + self.assertEquals(channel.result["code"], b"200", channel.result) + room_id = channel.json_body["room_id"] + + params = { + "id_server": "testis", + "medium": "email", + "address": "test@example.com", + } + request_data = json.dumps(params) + request_url = ( + "/rooms/%s/invite" % (room_id) + ).encode('ascii') + request, channel = self.make_request( + b"POST", request_url, request_data, access_token=self.tok, + ) + self.render(request) + + get_json = self.hs.get_simple_http_client().get_json + get_json.assert_called_once_with( + "https://testis/_matrix/identity/api/v1/lookup", + { + "address": "test@example.com", + "medium": "email", + }, + ) + + def test_3pid_lookup_enabled(self): + url = ("/_matrix/client/unstable/account/3pid/lookup" + "?id_server=testis&medium=email&address=foo@bar.baz") + request, channel = self.make_request("GET", url, access_token=self.tok) + self.render(request) + + get_json = self.hs.get_simple_http_client().get_json + get_json.assert_called_once_with( + "https://testis/_matrix/identity/api/v1/lookup", + { + "address": "foo@bar.baz", + "medium": "email", + }, + ) + + def test_3pid_bulk_lookup_enabled(self): + url = "/_matrix/client/unstable/account/3pid/bulk_lookup" + data = { + "id_server": "testis", + "threepids": [ + [ + "email", + "foo@bar.baz" + ], + [ + "email", + "john.doe@matrix.org" + ] + ] + } + request_data = json.dumps(data) + request, channel = self.make_request( + "POST", url, request_data, access_token=self.tok, + ) + self.render(request) + + post_json = self.hs.get_simple_http_client().post_json + post_json.assert_called_once_with( + "https://testis/_matrix/identity/api/v1/bulk_lookup", + { + "threepids": [ + [ + "email", + "foo@bar.baz" + ], + [ + "email", + "john.doe@matrix.org" + ] + ], + }, + ) -- cgit 1.5.1 From fe6ac9c5d709053abdd7a72297aaa64e6aa8e8fb Mon Sep 17 00:00:00 2001 From: Brendan Abolivier Date: Tue, 30 Apr 2019 18:28:01 +0100 Subject: Fixed return value --- synapse/rest/client/v2_alpha/account.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/synapse/rest/client/v2_alpha/account.py b/synapse/rest/client/v2_alpha/account.py index f7110f3b40..bf5d28b6e3 100644 --- a/synapse/rest/client/v2_alpha/account.py +++ b/synapse/rest/client/v2_alpha/account.py @@ -510,7 +510,7 @@ class ThreepidLookupRestServlet(RestServlet): # if the lookup is allowed so we don't need to do it here. ret = yield self.identity_handler.lookup_3pid(id_server, medium, address) - respond_with_json(200, ret) + defer.returnValue((200, ret)) class ThreepidBulkLookupRestServlet(RestServlet): @@ -537,7 +537,7 @@ class ThreepidBulkLookupRestServlet(RestServlet): body["id_server"], body["threepids"], ) - respond_with_json(200, ret) + defer.returnValue((200, ret)) class WhoamiRestServlet(RestServlet): -- cgit 1.5.1 From 0cc91efd6af3c8009e2c243e7a77e8fc18f1e808 Mon Sep 17 00:00:00 2001 From: Richard van der Hoff <1389908+richvdh@users.noreply.github.com> Date: Thu, 25 Apr 2019 15:11:27 +0100 Subject: Merge pull request #5098 from matrix-org/rav/fix_pep_517 Workarounds for pep-517 errors --- README.rst | 4 ++-- changelog.d/5098.misc | 1 + tox.ini | 5 +++++ 3 files changed, 8 insertions(+), 2 deletions(-) create mode 100644 changelog.d/5098.misc diff --git a/README.rst b/README.rst index 24afb93d7d..5409f0c563 100644 --- a/README.rst +++ b/README.rst @@ -173,7 +173,7 @@ Synapse offers two database engines: * `PostgreSQL `_ By default Synapse uses SQLite in and doing so trades performance for convenience. -SQLite is only recommended in Synapse for testing purposes or for servers with +SQLite is only recommended in Synapse for testing purposes or for servers with light workloads. Almost all installations should opt to use PostreSQL. Advantages include: @@ -272,7 +272,7 @@ to install using pip and a virtualenv:: virtualenv -p python3 env source env/bin/activate - python -m pip install -e .[all] + python -m pip install --no-pep-517 -e .[all] This will run a process of downloading and installing all the needed dependencies into a virtual env. diff --git a/changelog.d/5098.misc b/changelog.d/5098.misc new file mode 100644 index 0000000000..9cd83bf226 --- /dev/null +++ b/changelog.d/5098.misc @@ -0,0 +1 @@ +Add workarounds for pep-517 install errors. diff --git a/tox.ini b/tox.ini index ef543890f9..d0e519ce46 100644 --- a/tox.ini +++ b/tox.ini @@ -24,6 +24,11 @@ deps = pip>=10 setenv = + # we have a pyproject.toml, but don't want pip to use it for building. + # (otherwise we get an error about 'editable mode is not supported for + # pyproject.toml-style projects'). + PIP_USE_PEP517 = false + PYTHONDONTWRITEBYTECODE = no_byte_code COVERAGE_PROCESS_START = {toxinidir}/.coveragerc -- cgit 1.5.1 From 26c004129fcb26f5e1d91f65194fa276a4b0bdc5 Mon Sep 17 00:00:00 2001 From: Andrew Morgan <1342360+anoadragon453@users.noreply.github.com> Date: Wed, 1 May 2019 10:01:57 +0100 Subject: Update synapse/rest/client/v2_alpha/account.py Co-Authored-By: babolivier --- synapse/rest/client/v2_alpha/account.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/synapse/rest/client/v2_alpha/account.py b/synapse/rest/client/v2_alpha/account.py index bf5d28b6e3..fc3d0e789e 100644 --- a/synapse/rest/client/v2_alpha/account.py +++ b/synapse/rest/client/v2_alpha/account.py @@ -501,7 +501,7 @@ class ThreepidLookupRestServlet(RestServlet): query_params = request.args assert_params_in_dict(query_params, [b"medium", b"address", b"id_server"]) - # Retrieve address and medium from the request parameters + # Retrieve needed information from query parameters medium = parse_string(request, "medium") address = parse_string(request, "address") id_server = parse_string(request, "id_server") -- cgit 1.5.1 From 1973eb11d2d2392096f8be7fd17d947b8ffd0ec2 Mon Sep 17 00:00:00 2001 From: Andrew Morgan <1342360+anoadragon453@users.noreply.github.com> Date: Wed, 1 May 2019 10:02:18 +0100 Subject: Update synapse/rest/client/v2_alpha/account.py Co-Authored-By: babolivier --- synapse/rest/client/v2_alpha/account.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/synapse/rest/client/v2_alpha/account.py b/synapse/rest/client/v2_alpha/account.py index fc3d0e789e..154700d7d1 100644 --- a/synapse/rest/client/v2_alpha/account.py +++ b/synapse/rest/client/v2_alpha/account.py @@ -497,7 +497,7 @@ class ThreepidLookupRestServlet(RestServlet): """ yield self.auth.get_user_by_req(request) - # Extract query parameters + # Verify query parameters query_params = request.args assert_params_in_dict(query_params, [b"medium", b"address", b"id_server"]) -- cgit 1.5.1 From 371296443f7f6309462effb524d03c7cd02611c2 Mon Sep 17 00:00:00 2001 From: Andrew Morgan <1342360+anoadragon453@users.noreply.github.com> Date: Wed, 1 May 2019 10:03:46 +0100 Subject: Update synapse/handlers/identity.py Co-Authored-By: babolivier --- synapse/handlers/identity.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/synapse/handlers/identity.py b/synapse/handlers/identity.py index 23cc161798..4c9023145d 100644 --- a/synapse/handlers/identity.py +++ b/synapse/handlers/identity.py @@ -420,7 +420,7 @@ class IdentityHandler(BaseHandler): raise e.to_synapse_error() except IOError as e: logger.info("Failed to contact %r: %s", id_server, e) - raise ProxiedRequestError(503, "Failed to contact homeserver") + raise ProxiedRequestError(503, "Failed to contact identity server") defer.returnValue(data) -- cgit 1.5.1 From 3d031c211d6233e604fb50264174a877a9269172 Mon Sep 17 00:00:00 2001 From: Brendan Abolivier Date: Wed, 1 May 2019 10:10:38 +0100 Subject: Fix error message --- synapse/handlers/identity.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/synapse/handlers/identity.py b/synapse/handlers/identity.py index 4c9023145d..5b610990b6 100644 --- a/synapse/handlers/identity.py +++ b/synapse/handlers/identity.py @@ -376,7 +376,7 @@ class IdentityHandler(BaseHandler): raise e.to_synapse_error() except IOError as e: logger.info("Failed to contact %r: %s", id_server, e) - raise ProxiedRequestError(503, "Failed to contact homeserver") + raise ProxiedRequestError(503, "Failed to contact identity server") defer.returnValue(data) -- cgit 1.5.1 From b4f3d70b212b60576c12e99f62b53df169467067 Mon Sep 17 00:00:00 2001 From: Brendan Abolivier Date: Tue, 7 May 2019 11:47:37 +0100 Subject: Incorporate review --- synapse/handlers/identity.py | 2 +- synapse/rest/client/v2_alpha/account.py | 2 ++ tests/rest/client/test_identity.py | 6 +++--- 3 files changed, 6 insertions(+), 4 deletions(-) diff --git a/synapse/handlers/identity.py b/synapse/handlers/identity.py index 5b610990b6..dc77b6786f 100644 --- a/synapse/handlers/identity.py +++ b/synapse/handlers/identity.py @@ -403,7 +403,7 @@ class IdentityHandler(BaseHandler): target = self.rewrite_identity_server_urls.get(id_server, id_server) try: - data = yield self.http_client.post_json( + data = yield self.http_client.post_json_get_json( "https://%s/_matrix/identity/api/v1/bulk_lookup" % (target,), { "threepids": threepids, diff --git a/synapse/rest/client/v2_alpha/account.py b/synapse/rest/client/v2_alpha/account.py index 154700d7d1..f1037ce115 100644 --- a/synapse/rest/client/v2_alpha/account.py +++ b/synapse/rest/client/v2_alpha/account.py @@ -531,6 +531,8 @@ class ThreepidBulkLookupRestServlet(RestServlet): body = parse_json_object_from_request(request) + assert_params_in_dict(body, ["threepids", "id_server"]) + # Proxy the request to the identity server. lookup_3pid handles checking # if the lookup is allowed so we don't need to do it here. ret = yield self.identity_handler.bulk_lookup_3pid( diff --git a/tests/rest/client/test_identity.py b/tests/rest/client/test_identity.py index c1b1b11202..ed149f3600 100644 --- a/tests/rest/client/test_identity.py +++ b/tests/rest/client/test_identity.py @@ -115,10 +115,10 @@ class IdentityEnabledTestCase(unittest.HomeserverTestCase): mock_http_client = Mock(spec=[ "get_json", - "post_json", + "post_json_get_json", ]) mock_http_client.get_json.return_value = defer.succeed((200, "{}")) - mock_http_client.post_json.return_value = defer.succeed((200, "{}")) + mock_http_client.post_json_get_json.return_value = defer.succeed((200, "{}")) self.hs = self.setup_test_homeserver( config=config, @@ -198,7 +198,7 @@ class IdentityEnabledTestCase(unittest.HomeserverTestCase): ) self.render(request) - post_json = self.hs.get_simple_http_client().post_json + post_json = self.hs.get_simple_http_client().post_json_get_json post_json.assert_called_once_with( "https://testis/_matrix/identity/api/v1/bulk_lookup", { -- cgit 1.5.1 From 66f7588f87fe63b1a1c5069f58fcebd5c2beafc2 Mon Sep 17 00:00:00 2001 From: Brendan Abolivier Date: Tue, 7 May 2019 12:06:24 +0100 Subject: Fix expected key in bulk lookup response --- synapse/handlers/identity.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/synapse/handlers/identity.py b/synapse/handlers/identity.py index dc77b6786f..758390195b 100644 --- a/synapse/handlers/identity.py +++ b/synapse/handlers/identity.py @@ -410,7 +410,7 @@ class IdentityHandler(BaseHandler): } ) - if "mxid" in data: + if "threepids" in data: if "signatures" not in data: raise AuthError(401, "No signatures on 3pid bindings") yield self._verify_any_signature(data, id_server) -- cgit 1.5.1 From f059a910851ae15a88217c6c38b38807f2a491ac Mon Sep 17 00:00:00 2001 From: Brendan Abolivier Date: Tue, 7 May 2019 14:16:26 +0100 Subject: /bulk_lookup doesn't return a signature --- synapse/handlers/identity.py | 5 ----- 1 file changed, 5 deletions(-) diff --git a/synapse/handlers/identity.py b/synapse/handlers/identity.py index 758390195b..6066018275 100644 --- a/synapse/handlers/identity.py +++ b/synapse/handlers/identity.py @@ -410,11 +410,6 @@ class IdentityHandler(BaseHandler): } ) - if "threepids" in data: - if "signatures" not in data: - raise AuthError(401, "No signatures on 3pid bindings") - yield self._verify_any_signature(data, id_server) - except HttpResponseException as e: logger.info("Proxied lookup failed: %r", e) raise e.to_synapse_error() -- cgit 1.5.1 From 10e3ed83e9a63208eee07ec933b6a26fd76b51d2 Mon Sep 17 00:00:00 2001 From: Brendan Abolivier Date: Thu, 9 May 2019 12:53:24 +0100 Subject: Check if Synapse should check given ISs --- synapse/handlers/identity.py | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/synapse/handlers/identity.py b/synapse/handlers/identity.py index 6066018275..b4c6e94777 100644 --- a/synapse/handlers/identity.py +++ b/synapse/handlers/identity.py @@ -350,6 +350,12 @@ class IdentityHandler(BaseHandler): https://matrix.org/docs/spec/identity_service/r0.1.0.html#association-lookup for details """ + if not self._should_trust_id_server(id_server): + raise SynapseError( + 400, "Untrusted ID server '%s'" % id_server, + Codes.SERVER_NOT_TRUSTED + ) + if not self._enable_lookup: raise AuthError( 403, "Looking up third-party identifiers is denied from this server", @@ -395,6 +401,12 @@ class IdentityHandler(BaseHandler): https://matrix.org/docs/spec/identity_service/r0.1.0.html#association-lookup for details """ + if not self._should_trust_id_server(id_server): + raise SynapseError( + 400, "Untrusted ID server '%s'" % id_server, + Codes.SERVER_NOT_TRUSTED + ) + if not self._enable_lookup: raise AuthError( 403, "Looking up third-party identifiers is denied from this server", -- cgit 1.5.1 From 532ba44add046f0d25b3f36438ae45ea2ebae869 Mon Sep 17 00:00:00 2001 From: Brendan Abolivier Date: Thu, 9 May 2019 13:01:40 +0100 Subject: Fix test --- tests/rest/client/test_identity.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/tests/rest/client/test_identity.py b/tests/rest/client/test_identity.py index ed149f3600..7edcfa8f67 100644 --- a/tests/rest/client/test_identity.py +++ b/tests/rest/client/test_identity.py @@ -38,6 +38,10 @@ class IdentityDisabledTestCase(unittest.HomeserverTestCase): config = self.default_config() config.enable_3pid_lookup = False + config.trusted_third_party_id_servers = [ + "testis" + ] + self.hs = self.setup_test_homeserver(config=config) return self.hs @@ -112,6 +116,9 @@ class IdentityEnabledTestCase(unittest.HomeserverTestCase): config = self.default_config() config.enable_3pid_lookup = True + config.trusted_third_party_id_servers = [ + "testis" + ] mock_http_client = Mock(spec=[ "get_json", -- cgit 1.5.1 From f304f1a574a7114ad4ccc335563d1ecb3eb49501 Mon Sep 17 00:00:00 2001 From: Brendan Abolivier Date: Thu, 9 May 2019 13:07:43 +0100 Subject: Incorporate review --- synapse/handlers/identity.py | 2 +- synapse/rest/client/v2_alpha/account.py | 7 +++---- tests/rest/client/test_identity.py | 2 ++ 3 files changed, 6 insertions(+), 5 deletions(-) diff --git a/synapse/handlers/identity.py b/synapse/handlers/identity.py index b4c6e94777..f39803629e 100644 --- a/synapse/handlers/identity.py +++ b/synapse/handlers/identity.py @@ -388,7 +388,7 @@ class IdentityHandler(BaseHandler): @defer.inlineCallbacks def bulk_lookup_3pid(self, id_server, threepids): - """Looks up a 3pid in the passed identity server. + """Looks up given 3pids in the passed identity server. Args: id_server (str): The server name (including port, if required) diff --git a/synapse/rest/client/v2_alpha/account.py b/synapse/rest/client/v2_alpha/account.py index f1037ce115..08079a9bc6 100644 --- a/synapse/rest/client/v2_alpha/account.py +++ b/synapse/rest/client/v2_alpha/account.py @@ -15,6 +15,7 @@ # See the License for the specific language governing permissions and # limitations under the License. import logging +import re from six.moves import http_client @@ -482,11 +483,10 @@ class ThreepidDeleteRestServlet(RestServlet): class ThreepidLookupRestServlet(RestServlet): - PATTERNS = client_v2_patterns("/account/3pid/lookup$") + PATTERNS = [re.compile("^/_matrix/client/unstable/account/3pid/lookup$")] def __init__(self, hs): super(ThreepidLookupRestServlet, self).__init__() - self.config = hs.config self.auth = hs.get_auth() self.identity_handler = hs.get_handlers().identity_handler @@ -514,11 +514,10 @@ class ThreepidLookupRestServlet(RestServlet): class ThreepidBulkLookupRestServlet(RestServlet): - PATTERNS = client_v2_patterns("/account/3pid/bulk_lookup$") + PATTERNS = [re.compile("^/_matrix/client/unstable/account/3pid/bulk_lookup$")] def __init__(self, hs): super(ThreepidBulkLookupRestServlet, self).__init__() - self.config = hs.config self.auth = hs.get_auth() self.identity_handler = hs.get_handlers().identity_handler diff --git a/tests/rest/client/test_identity.py b/tests/rest/client/test_identity.py index 7edcfa8f67..b942f1ffe6 100644 --- a/tests/rest/client/test_identity.py +++ b/tests/rest/client/test_identity.py @@ -26,6 +26,7 @@ from tests import unittest class IdentityDisabledTestCase(unittest.HomeserverTestCase): + """Tests that 3PID lookup attempts fail when the HS's config disallows them.""" servlets = [ account.register_servlets, @@ -104,6 +105,7 @@ class IdentityDisabledTestCase(unittest.HomeserverTestCase): class IdentityEnabledTestCase(unittest.HomeserverTestCase): + """Tests that 3PID lookup attempts succeed when the HS's config allows them.""" servlets = [ account.register_servlets, -- cgit 1.5.1 From aa5c42f5bc5502f06ef41e03e0ea6863d2622bab Mon Sep 17 00:00:00 2001 From: Brendan Abolivier Date: Thu, 9 May 2019 13:59:57 +0100 Subject: Fix failing test --- tests/rest/client/test_identity.py | 4 ++-- tests/rulecheck/test_domainrulecheck.py | 3 +++ 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/tests/rest/client/test_identity.py b/tests/rest/client/test_identity.py index b942f1ffe6..ec260a2873 100644 --- a/tests/rest/client/test_identity.py +++ b/tests/rest/client/test_identity.py @@ -40,7 +40,7 @@ class IdentityDisabledTestCase(unittest.HomeserverTestCase): config = self.default_config() config.enable_3pid_lookup = False config.trusted_third_party_id_servers = [ - "testis" + "testis", ] self.hs = self.setup_test_homeserver(config=config) @@ -119,7 +119,7 @@ class IdentityEnabledTestCase(unittest.HomeserverTestCase): config = self.default_config() config.enable_3pid_lookup = True config.trusted_third_party_id_servers = [ - "testis" + "testis", ] mock_http_client = Mock(spec=[ diff --git a/tests/rulecheck/test_domainrulecheck.py b/tests/rulecheck/test_domainrulecheck.py index e3167aa06b..66b9cca4b3 100644 --- a/tests/rulecheck/test_domainrulecheck.py +++ b/tests/rulecheck/test_domainrulecheck.py @@ -164,6 +164,9 @@ class DomainRuleCheckerRoomTestCase(unittest.HomeserverTestCase): def make_homeserver(self, reactor, clock): config = self.default_config() + config.trusted_third_party_id_servers = [ + "localhost", + ] config.spam_checker = ( DomainRuleChecker, -- cgit 1.5.1