From 7b810e136ef2040fe55a778d4a4a790cdbd0f84c Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 3 Feb 2015 15:00:42 +0000 Subject: Add new FederationBase --- synapse/federation/federation_base.py | 126 ++++++++++++++++++++++++++++++++++ 1 file changed, 126 insertions(+) create mode 100644 synapse/federation/federation_base.py (limited to 'synapse/federation/federation_base.py') diff --git a/synapse/federation/federation_base.py b/synapse/federation/federation_base.py new file mode 100644 index 0000000000..d26a2396ac --- /dev/null +++ b/synapse/federation/federation_base.py @@ -0,0 +1,126 @@ +# -*- coding: utf-8 -*- +# Copyright 2015 OpenMarket Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +from twisted.internet import defer + +from synapse.events.utils import prune_event, old_prune_event + +from syutil.jsonutil import encode_canonical_json + +from synapse.crypto.event_signing import check_event_content_hash + +from synapse.api.errors import SynapseError + +import logging + + +logger = logging.getLogger(__name__) + + +class FederationBase(object): + @defer.inlineCallbacks + def _check_sigs_and_hash_and_fetch(self, origin, pdus, outlier=False): + """Takes a list of PDUs and checks the signatures and hashs of each + one. If a PDU fails its signature check then we check if we have it in + the database and if not then request if from the originating server of + that PDU. + + If a PDU fails its content hash check then it is redacted. + + The given list of PDUs are not modified, instead the function returns + a new list. + + Args: + pdu (list) + outlier (bool) + + Returns: + Deferred : A list of PDUs that have valid signatures and hashes. + """ + signed_pdus = [] + for pdu in pdus: + try: + new_pdu = yield self._check_sigs_and_hash(pdu) + signed_pdus.append(new_pdu) + except SynapseError: + # FIXME: We should handle signature failures more gracefully. + + # Check local db. + new_pdu = yield self.store.get_event( + pdu.event_id, + allow_rejected=True + ) + if new_pdu: + signed_pdus.append(new_pdu) + continue + + # Check pdu.origin + if pdu.origin != origin: + new_pdu = yield self.get_pdu( + destinations=[pdu.origin], + event_id=pdu.event_id, + outlier=outlier, + ) + + if new_pdu: + signed_pdus.append(new_pdu) + continue + + logger.warn("Failed to find copy of %s with valid signature") + + defer.returnValue(signed_pdus) + + @defer.inlineCallbacks + def _check_sigs_and_hash(self, pdu): + """Throws a SynapseError if the PDU does not have the correct + signatures. + + Returns: + FrozenEvent: Either the given event or it redacted if it failed the + content hash check. + """ + # Check signatures are correct. + redacted_event = prune_event(pdu) + redacted_pdu_json = redacted_event.get_pdu_json() + + old_redacted = old_prune_event(pdu) + old_redacted_pdu_json = old_redacted.get_pdu_json() + + try: + try: + yield self.keyring.verify_json_for_server( + pdu.origin, old_redacted_pdu_json + ) + except SynapseError: + yield self.keyring.verify_json_for_server( + pdu.origin, redacted_pdu_json + ) + except SynapseError: + logger.warn( + "Signature check failed for %s redacted to %s", + encode_canonical_json(pdu.get_pdu_json()), + encode_canonical_json(redacted_pdu_json), + ) + raise + + if not check_event_content_hash(pdu): + logger.warn( + "Event content has been tampered, redacting %s, %s", + pdu.event_id, encode_canonical_json(pdu.get_dict()) + ) + defer.returnValue(redacted_event) + + defer.returnValue(pdu) \ No newline at end of file -- cgit 1.4.1 From 9bace3a36751deed141225ccabd5bebecebc25f3 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 3 Feb 2015 15:32:17 +0000 Subject: Actually, the old prune_event function was non-deterministic, so no point keeping it around :( --- synapse/events/utils.py | 79 ----------------------------------- synapse/federation/federation_base.py | 16 ++----- 2 files changed, 4 insertions(+), 91 deletions(-) (limited to 'synapse/federation/federation_base.py') diff --git a/synapse/events/utils.py b/synapse/events/utils.py index 65a9f70982..1aa952150e 100644 --- a/synapse/events/utils.py +++ b/synapse/events/utils.py @@ -94,85 +94,6 @@ def prune_event(event): ) -def old_prune_event(event): - """This is an old and buggy version of the prune event function. The - difference between this and the new version is that when including dicts - in the content they were included as frozen_dicts rather than dicts. This - caused the JSON encoder to encode as a list of the keys rather than the - dict. - """ - event_type = event.type - - allowed_keys = [ - "event_id", - "sender", - "room_id", - "hashes", - "signatures", - "content", - "type", - "state_key", - "depth", - "prev_events", - "prev_state", - "auth_events", - "origin", - "origin_server_ts", - "membership", - ] - - event_dict = event.get_dict() - - new_content = {} - - def add_fields(*fields): - for field in fields: - if field in event.content: - # This is the line that is buggy: event.content may return - # a frozen_dict which the json encoders encode as lists rather - # than dicts. - new_content[field] = event.content[field] - - if event_type == EventTypes.Member: - add_fields("membership") - elif event_type == EventTypes.Create: - add_fields("creator") - elif event_type == EventTypes.JoinRules: - add_fields("join_rule") - elif event_type == EventTypes.PowerLevels: - add_fields( - "users", - "users_default", - "events", - "events_default", - "events_default", - "state_default", - "ban", - "kick", - "redact", - ) - elif event_type == EventTypes.Aliases: - add_fields("aliases") - - allowed_fields = { - k: v - for k, v in event_dict.items() - if k in allowed_keys - } - - allowed_fields["content"] = new_content - - allowed_fields["unsigned"] = {} - - if "age_ts" in event.unsigned: - allowed_fields["unsigned"]["age_ts"] = event.unsigned["age_ts"] - - return type(event)( - allowed_fields, - internal_metadata_dict=event.internal_metadata.get_dict() - ) - - def format_event_raw(d): return d diff --git a/synapse/federation/federation_base.py b/synapse/federation/federation_base.py index d26a2396ac..27c5918e0f 100644 --- a/synapse/federation/federation_base.py +++ b/synapse/federation/federation_base.py @@ -16,7 +16,7 @@ from twisted.internet import defer -from synapse.events.utils import prune_event, old_prune_event +from synapse.events.utils import prune_event from syutil.jsonutil import encode_canonical_json @@ -96,18 +96,10 @@ class FederationBase(object): redacted_event = prune_event(pdu) redacted_pdu_json = redacted_event.get_pdu_json() - old_redacted = old_prune_event(pdu) - old_redacted_pdu_json = old_redacted.get_pdu_json() - try: - try: - yield self.keyring.verify_json_for_server( - pdu.origin, old_redacted_pdu_json - ) - except SynapseError: - yield self.keyring.verify_json_for_server( - pdu.origin, redacted_pdu_json - ) + yield self.keyring.verify_json_for_server( + pdu.origin, redacted_pdu_json + ) except SynapseError: logger.warn( "Signature check failed for %s redacted to %s", -- cgit 1.4.1 From 3c39f42a0526186f85e69988504fff6adbf41d91 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 3 Feb 2015 16:14:19 +0000 Subject: New line --- synapse/federation/federation_base.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'synapse/federation/federation_base.py') diff --git a/synapse/federation/federation_base.py b/synapse/federation/federation_base.py index 27c5918e0f..a990aec4fd 100644 --- a/synapse/federation/federation_base.py +++ b/synapse/federation/federation_base.py @@ -115,4 +115,4 @@ class FederationBase(object): ) defer.returnValue(redacted_event) - defer.returnValue(pdu) \ No newline at end of file + defer.returnValue(pdu) -- cgit 1.4.1 From 963256638d5b3c3edee14bfbd7c00944b45d04c0 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Thu, 12 Feb 2015 18:17:11 +0000 Subject: Correctly handle all the places that can throw exceptions --- scripts/federation_client.py | 7 +++++-- synapse/federation/federation_base.py | 24 ++++++++++++++---------- synapse/federation/federation_server.py | 9 ++++++--- synapse/handlers/federation.py | 12 ++++++++++++ 4 files changed, 37 insertions(+), 15 deletions(-) (limited to 'synapse/federation/federation_base.py') diff --git a/scripts/federation_client.py b/scripts/federation_client.py index 3139c61761..ea62dceb36 100644 --- a/scripts/federation_client.py +++ b/scripts/federation_client.py @@ -97,8 +97,11 @@ def lookup(destination, path): if ":" in destination: return "https://%s%s" % (destination, path) else: - srv = srvlookup.lookup("matrix", "tcp", destination)[0] - return "https://%s:%d%s" % (srv.host, srv.port, path) + try: + srv = srvlookup.lookup("matrix", "tcp", destination)[0] + return "https://%s:%d%s" % (srv.host, srv.port, path) + except: + return "https://%s:%d%s" % (destination, 8448, path) def get_json(origin_name, origin_key, destination, path): request_json = { diff --git a/synapse/federation/federation_base.py b/synapse/federation/federation_base.py index a990aec4fd..30f3f5c8a4 100644 --- a/synapse/federation/federation_base.py +++ b/synapse/federation/federation_base.py @@ -61,7 +61,8 @@ class FederationBase(object): # Check local db. new_pdu = yield self.store.get_event( pdu.event_id, - allow_rejected=True + allow_rejected=True, + allow_none=True, ) if new_pdu: signed_pdus.append(new_pdu) @@ -69,15 +70,18 @@ class FederationBase(object): # Check pdu.origin if pdu.origin != origin: - new_pdu = yield self.get_pdu( - destinations=[pdu.origin], - event_id=pdu.event_id, - outlier=outlier, - ) - - if new_pdu: - signed_pdus.append(new_pdu) - continue + try: + new_pdu = yield self.get_pdu( + destinations=[pdu.origin], + event_id=pdu.event_id, + outlier=outlier, + ) + + if new_pdu: + signed_pdus.append(new_pdu) + continue + except: + pass logger.warn("Failed to find copy of %s with valid signature") diff --git a/synapse/federation/federation_server.py b/synapse/federation/federation_server.py index b23f72c7fa..9f5c98694c 100644 --- a/synapse/federation/federation_server.py +++ b/synapse/federation/federation_server.py @@ -411,9 +411,12 @@ class FederationServer(FederationBase): "_handle_new_pdu getting state for %s", pdu.room_id ) - state, auth_chain = yield self.get_state_for_room( - origin, pdu.room_id, pdu.event_id, - ) + try: + state, auth_chain = yield self.get_state_for_room( + origin, pdu.room_id, pdu.event_id, + ) + except: + logger.warn("Failed to get state for event: %s", pdu.event_id) ret = yield self.handler.on_receive_pdu( origin, diff --git a/synapse/handlers/federation.py b/synapse/handlers/federation.py index 77c81fe2da..5a7593cc3b 100644 --- a/synapse/handlers/federation.py +++ b/synapse/handlers/federation.py @@ -788,6 +788,18 @@ class FederationHandler(BaseHandler): defer.returnValue(ret) + @defer.inlineCallbacks + def trigger_query_auth(self, destination, event_id): + local_auth_chain = yield self.store.get_auth_chain([event_id]) + + result = yield self.replication_layer.query_auth( + destination, + event.room_id, + event.event_id, + local_auth_chain, + ) + + @defer.inlineCallbacks @log_function def do_auth(self, origin, event, context, auth_events): -- cgit 1.4.1 From 58d848adc0ee9ab8e44a780a19be1e46e0d5cec3 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Thu, 12 Feb 2015 18:35:36 +0000 Subject: Parrellize fetching of events --- synapse/federation/federation_base.py | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) (limited to 'synapse/federation/federation_base.py') diff --git a/synapse/federation/federation_base.py b/synapse/federation/federation_base.py index 30f3f5c8a4..966f4d2c5c 100644 --- a/synapse/federation/federation_base.py +++ b/synapse/federation/federation_base.py @@ -50,8 +50,11 @@ class FederationBase(object): Returns: Deferred : A list of PDUs that have valid signatures and hashes. """ + signed_pdus = [] - for pdu in pdus: + + @defer.inlineCallbacks + def do(pdu): try: new_pdu = yield self._check_sigs_and_hash(pdu) signed_pdus.append(new_pdu) @@ -66,7 +69,7 @@ class FederationBase(object): ) if new_pdu: signed_pdus.append(new_pdu) - continue + return # Check pdu.origin if pdu.origin != origin: @@ -79,12 +82,17 @@ class FederationBase(object): if new_pdu: signed_pdus.append(new_pdu) - continue + return except: pass logger.warn("Failed to find copy of %s with valid signature") + yield defer.gatherResults( + [do(pdu) for pdu in pdus], + consumeErrors=True + ) + defer.returnValue(signed_pdus) @defer.inlineCallbacks -- cgit 1.4.1 From 789251afa7d97a08318f92ffe96f5f7ce16d6157 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Thu, 12 Feb 2015 19:29:43 +0000 Subject: Fix logging --- synapse/federation/federation_base.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) (limited to 'synapse/federation/federation_base.py') diff --git a/synapse/federation/federation_base.py b/synapse/federation/federation_base.py index 966f4d2c5c..21a763214b 100644 --- a/synapse/federation/federation_base.py +++ b/synapse/federation/federation_base.py @@ -86,7 +86,10 @@ class FederationBase(object): except: pass - logger.warn("Failed to find copy of %s with valid signature") + logger.warn( + "Failed to find copy of %s with valid signature", + pdu.event_id, + ) yield defer.gatherResults( [do(pdu) for pdu in pdus], -- cgit 1.4.1