From 52ca8676700368098ca0689c424f972ac54ac780 Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Wed, 24 Sep 2014 17:25:41 +0100 Subject: Sign federation transactions --- tests/handlers/test_federation.py | 7 ++++++- tests/handlers/test_presence.py | 21 +++++++++++++++++++-- tests/handlers/test_room.py | 5 ++++- tests/handlers/test_typing.py | 6 +++++- 4 files changed, 34 insertions(+), 5 deletions(-) (limited to 'tests/handlers') diff --git a/tests/handlers/test_federation.py b/tests/handlers/test_federation.py index eb6b7c22ef..5c69997273 100644 --- a/tests/handlers/test_federation.py +++ b/tests/handlers/test_federation.py @@ -26,12 +26,16 @@ from synapse.federation.units import Pdu from mock import NonCallableMock, ANY -from ..utils import get_mock_call_args +from ..utils import get_mock_call_args, MockKey class FederationTestCase(unittest.TestCase): def setUp(self): + + self.mock_config = NonCallableMock() + self.mock_config.signing_key = [MockKey()] + self.hostname = "test" hs = HomeServer( self.hostname, @@ -48,6 +52,7 @@ class FederationTestCase(unittest.TestCase): "room_member_handler", "federation_handler", ]), + config=self.mock_config, ) self.datastore = hs.get_datastore() diff --git a/tests/handlers/test_presence.py b/tests/handlers/test_presence.py index 765929d204..fe4b892ffb 100644 --- a/tests/handlers/test_presence.py +++ b/tests/handlers/test_presence.py @@ -17,11 +17,12 @@ from tests import unittest from twisted.internet import defer, reactor -from mock import Mock, call, ANY +from mock import Mock, call, ANY, NonCallableMock import json from tests.utils import ( - MockHttpResource, MockClock, DeferredMockCallable, SQLiteMemoryDbPool + MockHttpResource, MockClock, DeferredMockCallable, SQLiteMemoryDbPool, + MockKey ) from synapse.server import HomeServer @@ -67,12 +68,16 @@ class PresenceStateTestCase(unittest.TestCase): db_pool = SQLiteMemoryDbPool() yield db_pool.prepare() + self.mock_config = NonCallableMock() + self.mock_config.signing_key = [MockKey()] + hs = HomeServer("test", clock=MockClock(), db_pool=db_pool, handlers=None, resource_for_federation=Mock(), http_client=None, + config=self.mock_config, ) hs.handlers = JustPresenceHandlers(hs) @@ -214,6 +219,9 @@ class PresenceInvitesTestCase(unittest.TestCase): db_pool = SQLiteMemoryDbPool() yield db_pool.prepare() + self.mock_config = NonCallableMock() + self.mock_config.signing_key = [MockKey()] + hs = HomeServer("test", clock=MockClock(), db_pool=db_pool, @@ -221,6 +229,7 @@ class PresenceInvitesTestCase(unittest.TestCase): resource_for_client=Mock(), resource_for_federation=self.mock_federation_resource, http_client=self.mock_http_client, + config=self.mock_config, ) hs.handlers = JustPresenceHandlers(hs) @@ -503,6 +512,9 @@ class PresencePushTestCase(unittest.TestCase): self.mock_federation_resource = MockHttpResource() + self.mock_config = NonCallableMock() + self.mock_config.signing_key = [MockKey()] + hs = HomeServer("test", clock=self.clock, db_pool=None, @@ -520,6 +532,7 @@ class PresencePushTestCase(unittest.TestCase): resource_for_client=Mock(), resource_for_federation=self.mock_federation_resource, http_client=self.mock_http_client, + config=self.mock_config, ) hs.handlers = JustPresenceHandlers(hs) @@ -995,6 +1008,9 @@ class PresencePollingTestCase(unittest.TestCase): self.mock_federation_resource = MockHttpResource() + self.mock_config = NonCallableMock() + self.mock_config.signing_key = [MockKey()] + hs = HomeServer("test", clock=MockClock(), db_pool=None, @@ -1009,6 +1025,7 @@ class PresencePollingTestCase(unittest.TestCase): resource_for_client=Mock(), resource_for_federation=self.mock_federation_resource, http_client=self.mock_http_client, + config = self.mock_config, ) hs.handlers = JustPresenceHandlers(hs) diff --git a/tests/handlers/test_room.py b/tests/handlers/test_room.py index a1a2e80492..c88d1c8840 100644 --- a/tests/handlers/test_room.py +++ b/tests/handlers/test_room.py @@ -24,6 +24,7 @@ from synapse.api.constants import Membership from synapse.handlers.room import RoomMemberHandler, RoomCreationHandler from synapse.handlers.profile import ProfileHandler from synapse.server import HomeServer +from ..utils import MockKey from mock import Mock, NonCallableMock @@ -31,6 +32,8 @@ from mock import Mock, NonCallableMock class RoomMemberHandlerTestCase(unittest.TestCase): def setUp(self): + self.mock_config = NonCallableMock() + self.mock_config.signing_key = [MockKey()] self.hostname = "red" hs = HomeServer( self.hostname, @@ -38,7 +41,6 @@ class RoomMemberHandlerTestCase(unittest.TestCase): ratelimiter=NonCallableMock(spec_set=[ "send_message", ]), - config=NonCallableMock(), datastore=NonCallableMock(spec_set=[ "persist_event", "get_joined_hosts_for_room", @@ -57,6 +59,7 @@ class RoomMemberHandlerTestCase(unittest.TestCase): ]), auth=NonCallableMock(spec_set=["check"]), state_handler=NonCallableMock(spec_set=["handle_new_event"]), + config=self.mock_config, ) self.federation = NonCallableMock(spec_set=[ diff --git a/tests/handlers/test_typing.py b/tests/handlers/test_typing.py index a66f208abf..0ab829b9ad 100644 --- a/tests/handlers/test_typing.py +++ b/tests/handlers/test_typing.py @@ -20,7 +20,7 @@ from twisted.internet import defer from mock import Mock, call, ANY import json -from ..utils import MockHttpResource, MockClock, DeferredMockCallable +from ..utils import MockHttpResource, MockClock, DeferredMockCallable, MockKey from synapse.server import HomeServer from synapse.handlers.typing import TypingNotificationHandler @@ -61,6 +61,9 @@ class TypingNotificationsTestCase(unittest.TestCase): self.mock_federation_resource = MockHttpResource() + self.mock_config = Mock() + self.mock_config.signing_key = [MockKey()] + hs = HomeServer("test", clock=self.clock, db_pool=None, @@ -75,6 +78,7 @@ class TypingNotificationsTestCase(unittest.TestCase): resource_for_client=Mock(), resource_for_federation=self.mock_federation_resource, http_client=self.mock_http_client, + config=self.mock_config, ) hs.handlers = JustTypingNotificationHandlers(hs) -- cgit 1.4.1 From b95a178584cac07018f47e571f48993878da7284 Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Tue, 30 Sep 2014 15:15:10 +0100 Subject: SYN-75 Verify signatures on server to server transactions --- synapse/crypto/keyclient.py | 75 +++++++++------------- synapse/crypto/keyring.py | 125 ++++++++++++++++++++++++++++++++++++ synapse/crypto/keyserver.py | 111 -------------------------------- synapse/crypto/resource/__init__.py | 15 ----- synapse/federation/replication.py | 24 ++++--- synapse/federation/transport.py | 22 ++++--- synapse/federation/units.py | 3 - synapse/server.py | 5 ++ synapse/storage/__init__.py | 1 + synapse/storage/keys.py | 75 +++++++++++++--------- synapse/storage/schema/keys.sql | 13 ++-- tests/federation/test_federation.py | 1 + tests/handlers/test_presence.py | 9 ++- tests/handlers/test_typing.py | 1 + 14 files changed, 245 insertions(+), 235 deletions(-) create mode 100644 synapse/crypto/keyring.py delete mode 100644 synapse/crypto/keyserver.py delete mode 100644 synapse/crypto/resource/__init__.py (limited to 'tests/handlers') diff --git a/synapse/crypto/keyclient.py b/synapse/crypto/keyclient.py index c11df5c529..c26f16a038 100644 --- a/synapse/crypto/keyclient.py +++ b/synapse/crypto/keyclient.py @@ -15,9 +15,10 @@ from twisted.web.http import HTTPClient +from twisted.internet.protocol import Factory from twisted.internet import defer, reactor -from twisted.internet.protocol import ClientFactory -from twisted.names.srvconnect import SRVConnector +from twisted.internet.endpoints import connectProtocol +from synapse.http.endpoint import matrix_endpoint import json import logging @@ -30,15 +31,19 @@ def fetch_server_key(server_name, ssl_context_factory): """Fetch the keys for a remote server.""" factory = SynapseKeyClientFactory() + endpoint = matrix_endpoint( + reactor, server_name, ssl_context_factory, timeout=30 + ) - SRVConnector( - reactor, "matrix", server_name, factory, - protocol="tcp", connectFuncName="connectSSL", defaultPort=443, - connectFuncKwArgs=dict(contextFactory=ssl_context_factory)).connect() - - server_key, server_certificate = yield factory.remote_key - - defer.returnValue((server_key, server_certificate)) + for i in range(5): + try: + protocol = yield endpoint.connect(factory) + server_response, server_certificate = yield protocol.remote_key + defer.returnValue((server_response, server_certificate)) + return + except Exception as e: + logger.exception(e) + raise IOError("Cannot get key for " % server_name) class SynapseKeyClientError(Exception): @@ -51,69 +56,47 @@ class SynapseKeyClientProtocol(HTTPClient): the server and extracts the X.509 certificate for the remote peer from the SSL connection.""" + timeout = 30 + + def __init__(self): + self.remote_key = defer.Deferred() + def connectionMade(self): logger.debug("Connected to %s", self.transport.getHost()) - self.sendCommand(b"GET", b"/key") + self.sendCommand(b"GET", b"/_matrix/key/v1/") self.endHeaders() self.timer = reactor.callLater( - self.factory.timeout_seconds, + self.timeout, self.on_timeout ) def handleStatus(self, version, status, message): if status != b"200": - logger.info("Non-200 response from %s: %s %s", - self.transport.getHost(), status, message) + #logger.info("Non-200 response from %s: %s %s", + # self.transport.getHost(), status, message) self.transport.abortConnection() def handleResponse(self, response_body_bytes): try: json_response = json.loads(response_body_bytes) except ValueError: - logger.info("Invalid JSON response from %s", - self.transport.getHost()) + #logger.info("Invalid JSON response from %s", + # self.transport.getHost()) self.transport.abortConnection() return certificate = self.transport.getPeerCertificate() - self.factory.on_remote_key((json_response, certificate)) + self.remote_key.callback((json_response, certificate)) self.transport.abortConnection() self.timer.cancel() def on_timeout(self): logger.debug("Timeout waiting for response from %s", self.transport.getHost()) + self.on_remote_key.errback(IOError("Timeout waiting for response")) self.transport.abortConnection() -class SynapseKeyClientFactory(ClientFactory): +class SynapseKeyClientFactory(Factory): protocol = SynapseKeyClientProtocol - max_retries = 5 - timeout_seconds = 30 - - def __init__(self): - self.succeeded = False - self.retries = 0 - self.remote_key = defer.Deferred() - def on_remote_key(self, key): - self.succeeded = True - self.remote_key.callback(key) - - def retry_connection(self, connector): - self.retries += 1 - if self.retries < self.max_retries: - connector.connector = None - connector.connect() - else: - self.remote_key.errback( - SynapseKeyClientError("Max retries exceeded")) - - def clientConnectionFailed(self, connector, reason): - logger.info("Connection failed %s", reason) - self.retry_connection(connector) - - def clientConnectionLost(self, connector, reason): - logger.info("Connection lost %s", reason) - if not self.succeeded: - self.retry_connection(connector) diff --git a/synapse/crypto/keyring.py b/synapse/crypto/keyring.py new file mode 100644 index 0000000000..ce19c69bd5 --- /dev/null +++ b/synapse/crypto/keyring.py @@ -0,0 +1,125 @@ +# -*- coding: utf-8 -*- +# Copyright 2014 OpenMarket Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from synapse.crypto.keyclient import fetch_server_key +from twisted.internet import defer +from syutil.crypto.jsonsign import verify_signed_json, signature_ids +from syutil.crypto.signing_key import ( + is_signing_algorithm_supported, decode_verify_key_bytes +) +from syutil.base64util import decode_base64, encode_base64 + +from OpenSSL import crypto + +import logging + + +logger = logging.getLogger(__name__) + + +class Keyring(object): + def __init__(self, hs): + self.store = hs.get_datastore() + self.clock = hs.get_clock() + self.hs = hs + + @defer.inlineCallbacks + def verify_json_for_server(self, server_name, json_object): + key_ids = signature_ids(json_object, server_name) + verify_key = yield self.get_server_verify_key(server_name, key_ids) + verify_signed_json(json_object, server_name, verify_key) + + @defer.inlineCallbacks + def get_server_verify_key(self, server_name, key_ids): + """Finds a verification key for the server with one of the key ids. + Args: + server_name (str): The name of the server to fetch a key for. + keys_ids (list of str): The key_ids to check for. + """ + + # Check the datastore to see if we have one cached. + cached = yield self.store.get_server_verify_keys(server_name, key_ids) + + if cached: + defer.returnValue(cached[0]) + return + + # Try to fetch the key from the remote server. + # TODO(markjh): Ratelimit requests to a given server. + + (response, tls_certificate) = yield fetch_server_key( + server_name, self.hs.tls_context_factory + ) + + # Check the response. + + x509_certificate_bytes = crypto.dump_certificate( + crypto.FILETYPE_ASN1, tls_certificate + ) + + if ("signatures" not in response + or server_name not in response["signatures"]): + raise ValueError("Key response not signed by remote server") + + if "tls_certificate" not in response: + raise ValueError("Key response missing TLS certificate") + + tls_certificate_b64 = response["tls_certificate"] + + if encode_base64(x509_certificate_bytes) != tls_certificate_b64: + raise ValueError("TLS certificate doesn't match") + + verify_keys = {} + for key_id, key_base64 in response["verify_keys"].items(): + if is_signing_algorithm_supported(key_id): + key_bytes = decode_base64(key_base64) + verify_key = decode_verify_key_bytes(key_id, key_bytes) + verify_keys[key_id] = verify_key + + for key_id in response["signatures"][server_name]: + if key_id not in response["verify_keys"]: + raise ValueError( + "Key response must include verification keys for all" + " signatures" + ) + if key_id in verify_keys: + verify_signed_json( + response, + server_name, + verify_keys[key_id] + ) + + # Cache the result in the datastore. + + time_now_ms = self.clock.time_msec() + + self.store.store_server_certificate( + server_name, + server_name, + time_now_ms, + tls_certificate, + ) + + for key_id, key in verify_keys.items(): + self.store.store_server_verify_key( + server_name, server_name, time_now_ms, key + ) + + for key_id in key_ids: + if key_id in verify_keys: + defer.returnValue(verify_keys[key_id]) + return + + raise ValueError("No verification key found for given key ids") diff --git a/synapse/crypto/keyserver.py b/synapse/crypto/keyserver.py deleted file mode 100644 index a23484dbae..0000000000 --- a/synapse/crypto/keyserver.py +++ /dev/null @@ -1,111 +0,0 @@ -# -*- coding: utf-8 -*- -# Copyright 2014 OpenMarket Ltd -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - - -from twisted.internet import reactor, ssl -from twisted.web import server -from twisted.web.resource import Resource -from twisted.python.log import PythonLoggingObserver - -from synapse.crypto.resource.key import LocalKey -from synapse.crypto.config import load_config - -from syutil.base64util import decode_base64 - -from OpenSSL import crypto, SSL - -import logging -import nacl.signing -import sys - - -class KeyServerSSLContextFactory(ssl.ContextFactory): - """Factory for PyOpenSSL SSL contexts that are used to handle incoming - connections and to make connections to remote servers.""" - - def __init__(self, key_server): - self._context = SSL.Context(SSL.SSLv23_METHOD) - self.configure_context(self._context, key_server) - - @staticmethod - def configure_context(context, key_server): - context.set_options(SSL.OP_NO_SSLv2 | SSL.OP_NO_SSLv3) - context.use_certificate(key_server.tls_certificate) - context.use_privatekey(key_server.tls_private_key) - context.load_tmp_dh(key_server.tls_dh_params_path) - context.set_cipher_list("!ADH:HIGH+kEDH:!AECDH:HIGH+kEECDH") - - def getContext(self): - return self._context - - -class KeyServer(object): - """An HTTPS server serving LocalKey and RemoteKey resources.""" - - def __init__(self, server_name, tls_certificate_path, tls_private_key_path, - tls_dh_params_path, signing_key_path, bind_host, bind_port): - self.server_name = server_name - self.tls_certificate = self.read_tls_certificate(tls_certificate_path) - self.tls_private_key = self.read_tls_private_key(tls_private_key_path) - self.tls_dh_params_path = tls_dh_params_path - self.signing_key = self.read_signing_key(signing_key_path) - self.bind_host = bind_host - self.bind_port = int(bind_port) - self.ssl_context_factory = KeyServerSSLContextFactory(self) - - @staticmethod - def read_tls_certificate(cert_path): - with open(cert_path) as cert_file: - cert_pem = cert_file.read() - return crypto.load_certificate(crypto.FILETYPE_PEM, cert_pem) - - @staticmethod - def read_tls_private_key(private_key_path): - with open(private_key_path) as private_key_file: - private_key_pem = private_key_file.read() - return crypto.load_privatekey(crypto.FILETYPE_PEM, private_key_pem) - - @staticmethod - def read_signing_key(signing_key_path): - with open(signing_key_path) as signing_key_file: - signing_key_b64 = signing_key_file.read() - signing_key_bytes = decode_base64(signing_key_b64) - return nacl.signing.SigningKey(signing_key_bytes) - - def run(self): - root = Resource() - root.putChild("key", LocalKey(self)) - site = server.Site(root) - reactor.listenSSL( - self.bind_port, - site, - self.ssl_context_factory, - interface=self.bind_host - ) - - logging.basicConfig(level=logging.DEBUG) - observer = PythonLoggingObserver() - observer.start() - - reactor.run() - - -def main(): - key_server = KeyServer(**load_config(__doc__, sys.argv[1:])) - key_server.run() - - -if __name__ == "__main__": - main() diff --git a/synapse/crypto/resource/__init__.py b/synapse/crypto/resource/__init__.py deleted file mode 100644 index 9bff9ec169..0000000000 --- a/synapse/crypto/resource/__init__.py +++ /dev/null @@ -1,15 +0,0 @@ -# -*- coding: utf-8 -*- -# Copyright 2014 OpenMarket Ltd -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - diff --git a/synapse/federation/replication.py b/synapse/federation/replication.py index 84977e7e57..a8dd038b0b 100644 --- a/synapse/federation/replication.py +++ b/synapse/federation/replication.py @@ -66,6 +66,8 @@ class ReplicationLayer(object): hs, self.transaction_actions, transport_layer ) + self.keyring = hs.get_keyring() + self.handler = None self.edu_handlers = {} self.query_handlers = {} @@ -291,6 +293,10 @@ class ReplicationLayer(object): @defer.inlineCallbacks @log_function def on_incoming_transaction(self, transaction_data): + yield self.keyring.verify_json_for_server( + transaction_data["origin"], transaction_data + ) + transaction = Transaction(**transaction_data) for p in transaction.pdus: @@ -590,7 +596,7 @@ class _TransactionQueue(object): transaction = Transaction.create_new( ts=self._clock.time_msec(), - transaction_id=self._next_txn_id, + transaction_id=str(self._next_txn_id), origin=self.server_name, destination=destination, pdus=pdus, @@ -611,20 +617,18 @@ class _TransactionQueue(object): # FIXME (erikj): This is a bit of a hack to make the Pdu age # keys work - def cb(transaction): + def json_data_cb(): + data = transaction.get_dict() now = int(self._clock.time_msec()) - if "pdus" in transaction: - for p in transaction["pdus"]: + if "pdus" in data: + for p in data["pdus"]: if "age_ts" in p: p["age"] = now - int(p["age_ts"]) - - transaction = sign_json(transaction, server_name, signing_key) - - return transaction + data = sign_json(data, server_name, signing_key) + return data code, response = yield self.transport_layer.send_transaction( - transaction, - on_send_callback=cb, + transaction, json_data_cb ) logger.debug("TX [%s] Sent transaction", destination) diff --git a/synapse/federation/transport.py b/synapse/federation/transport.py index afc777ec9e..5d595b7433 100644 --- a/synapse/federation/transport.py +++ b/synapse/federation/transport.py @@ -144,7 +144,7 @@ class TransportLayer(object): @defer.inlineCallbacks @log_function - def send_transaction(self, transaction, on_send_callback=None): + def send_transaction(self, transaction, json_data_callback=None): """ Sends the given Transaction to it's destination Args: @@ -163,24 +163,26 @@ class TransportLayer(object): if transaction.destination == self.server_name: raise RuntimeError("Transport layer cannot send to itself!") - data = transaction.get_dict() + if json_data_callback is None: + def json_data_callback(): + return transaction.get_dict() # FIXME (erikj): This is a bit of a hack to make the Pdu age # keys work def cb(destination, method, path_bytes, producer): - if not on_send_callback: - return + json_data = json_data_callback() + del json_data["destination"] + del json_data["transaction_id"] + producer.reset(json_data) - transaction = json.loads(producer.body) - - new_transaction = on_send_callback(transaction) - - producer.reset(new_transaction) + json_data = transaction.get_dict() + del json_data["destination"] + del json_data["transaction_id"] code, response = yield self.client.put_json( transaction.destination, path=PREFIX + "/send/%s/" % transaction.transaction_id, - data=data, + data=json_data, on_send_callback=cb, ) diff --git a/synapse/federation/units.py b/synapse/federation/units.py index 622fe66a8f..1ca123d1bf 100644 --- a/synapse/federation/units.py +++ b/synapse/federation/units.py @@ -186,9 +186,6 @@ class Transaction(JsonEncodedObject): "previous_ids", "pdus", "edus", - ] - - internal_keys = [ "transaction_id", "destination", ] diff --git a/synapse/server.py b/synapse/server.py index 529500d595..ed5b810d3e 100644 --- a/synapse/server.py +++ b/synapse/server.py @@ -34,6 +34,7 @@ from synapse.util.distributor import Distributor from synapse.util.lockutils import LockManager from synapse.streams.events import EventSources from synapse.api.ratelimiting import Ratelimiter +from synapse.crypto.keyring import Keyring class BaseHomeServer(object): @@ -78,6 +79,7 @@ class BaseHomeServer(object): 'resource_for_server_key', 'event_sources', 'ratelimiter', + 'keyring', ] def __init__(self, hostname, **kwargs): @@ -201,6 +203,9 @@ class HomeServer(BaseHomeServer): def build_ratelimiter(self): return Ratelimiter() + def build_keyring(self): + return Keyring(self) + def register_servlets(self): """ Register all servlets associated with this HomeServer. """ diff --git a/synapse/storage/__init__.py b/synapse/storage/__init__.py index 66658f6721..ef98b6a444 100644 --- a/synapse/storage/__init__.py +++ b/synapse/storage/__init__.py @@ -56,6 +56,7 @@ SCHEMAS = [ "presence", "im", "room_aliases", + "keys", ] diff --git a/synapse/storage/keys.py b/synapse/storage/keys.py index 5a38c3e8f2..253dc17be2 100644 --- a/synapse/storage/keys.py +++ b/synapse/storage/keys.py @@ -18,7 +18,8 @@ from _base import SQLBaseStore from twisted.internet import defer import OpenSSL -import nacl.signing +from syutil.crypto.signing_key import decode_verify_key_bytes +import hashlib class KeyStore(SQLBaseStore): """Persistence for signature verification keys and tls X.509 certificates @@ -42,62 +43,74 @@ class KeyStore(SQLBaseStore): ) defer.returnValue(tls_certificate) - def store_server_certificate(self, server_name, key_server, ts_now_ms, + def store_server_certificate(self, server_name, from_server, time_now_ms, tls_certificate): """Stores the TLS X.509 certificate for the given server Args: - server_name (bytes): The name of the server. - key_server (bytes): Where the certificate was looked up - ts_now_ms (int): The time now in milliseconds + server_name (str): The name of the server. + from_server (str): Where the certificate was looked up + time_now_ms (int): The time now in milliseconds tls_certificate (OpenSSL.crypto.X509): The X.509 certificate. """ tls_certificate_bytes = OpenSSL.crypto.dump_certificate( OpenSSL.crypto.FILETYPE_ASN1, tls_certificate ) + fingerprint = hashlib.sha256(tls_certificate_bytes).hexdigest() return self._simple_insert( table="server_tls_certificates", - keyvalues={ + values={ "server_name": server_name, - "key_server": key_server, - "ts_added_ms": ts_now_ms, - "tls_certificate": tls_certificate_bytes, + "fingerprint": fingerprint, + "from_server": from_server, + "ts_added_ms": time_now_ms, + "tls_certificate": buffer(tls_certificate_bytes), }, ) @defer.inlineCallbacks - def get_server_verification_key(self, server_name): - """Retrieve the NACL verification key for a given server + def get_server_verify_keys(self, server_name, key_ids): + """Retrieve the NACL verification key for a given server for the given + key_ids Args: - server_name (bytes): The name of the server. + server_name (str): The name of the server. + key_ids (list of str): List of key_ids to try and look up. Returns: - (nacl.signing.VerifyKey): The verification key. + (list of VerifyKey): The verification keys. """ - verification_key_bytes, = yield self._simple_select_one( - table="server_signature_keys", - key_values={"server_name": server_name}, - retcols=("tls_certificate",), + sql = ( + "SELECT key_id, verify_key FROM server_signature_keys" + " WHERE server_name = ?" + " AND key_id in (" + ",".join("?" for key_id in key_ids) + ")" ) - verification_key = nacl.signing.VerifyKey(verification_key_bytes) - defer.returnValue(verification_key) - def store_server_verification_key(self, server_name, key_version, - key_server, ts_now_ms, verification_key): + rows = yield self._execute_and_decode(sql, server_name, *key_ids) + + keys = [] + for row in rows: + key_id = row["key_id"] + key_bytes = row["verify_key"] + key = decode_verify_key_bytes(key_id, str(key_bytes)) + keys.append(key) + defer.returnValue(keys) + + def store_server_verify_key(self, server_name, from_server, time_now_ms, + verify_key): """Stores a NACL verification key for the given server. Args: - server_name (bytes): The name of the server. - key_version (bytes): The version of the key for the server. - key_server (bytes): Where the verification key was looked up + server_name (str): The name of the server. + key_id (str): The version of the key for the server. + from_server (str): Where the verification key was looked up ts_now_ms (int): The time now in milliseconds - verification_key (nacl.signing.VerifyKey): The NACL verify key. + verification_key (VerifyKey): The NACL verify key. """ - verification_key_bytes = verification_key.encode() + verify_key_bytes = verify_key.encode() return self._simple_insert( table="server_signature_keys", - key_values={ + values={ "server_name": server_name, - "key_version": key_version, - "key_server": key_server, - "ts_added_ms": ts_now_ms, - "verification_key": verification_key_bytes, + "key_id": "%s:%s" % (verify_key.alg, verify_key.version), + "from_server": from_server, + "ts_added_ms": time_now_ms, + "verify_key": buffer(verify_key.encode()), }, ) diff --git a/synapse/storage/schema/keys.sql b/synapse/storage/schema/keys.sql index 706a1a03ff..9bf2068d84 100644 --- a/synapse/storage/schema/keys.sql +++ b/synapse/storage/schema/keys.sql @@ -14,17 +14,18 @@ */ CREATE TABLE IF NOT EXISTS server_tls_certificates( server_name TEXT, -- Server name. - key_server TEXT, -- Which key server the certificate was fetched from. + fingerprint TEXT, -- Certificate fingerprint. + from_server TEXT, -- Which key server the certificate was fetched from. ts_added_ms INTEGER, -- When the certifcate was added. tls_certificate BLOB, -- DER encoded x509 certificate. - CONSTRAINT uniqueness UNIQUE (server_name) + CONSTRAINT uniqueness UNIQUE (server_name, fingerprint) ); CREATE TABLE IF NOT EXISTS server_signature_keys( server_name TEXT, -- Server name. - key_version TEXT, -- Key version. - key_server TEXT, -- Which key server the key was fetched form. + key_id TEXT, -- Key version. + from_server TEXT, -- Which key server the key was fetched form. ts_added_ms INTEGER, -- When the key was added. - verification_key BLOB, -- NACL verification key. - CONSTRAINT uniqueness UNIQUE (server_name, key_version) + verify_key BLOB, -- NACL verification key. + CONSTRAINT uniqueness UNIQUE (server_name, key_id) ); diff --git a/tests/federation/test_federation.py b/tests/federation/test_federation.py index eafc7879e0..c893c4863d 100644 --- a/tests/federation/test_federation.py +++ b/tests/federation/test_federation.py @@ -74,6 +74,7 @@ class FederationTestCase(unittest.TestCase): datastore=self.mock_persistence, clock=self.clock, config=self.mock_config, + keyring=Mock(), ) self.federation = initialize_http_replication(hs) self.distributor = hs.get_distributor() diff --git a/tests/handlers/test_presence.py b/tests/handlers/test_presence.py index fe4b892ffb..c9406b70c4 100644 --- a/tests/handlers/test_presence.py +++ b/tests/handlers/test_presence.py @@ -17,7 +17,7 @@ from tests import unittest from twisted.internet import defer, reactor -from mock import Mock, call, ANY, NonCallableMock +from mock import Mock, call, ANY, NonCallableMock, patch import json from tests.utils import ( @@ -59,7 +59,6 @@ class JustPresenceHandlers(object): def __init__(self, hs): self.presence_handler = PresenceHandler(hs) - class PresenceStateTestCase(unittest.TestCase): """ Tests presence management. """ @@ -78,6 +77,7 @@ class PresenceStateTestCase(unittest.TestCase): resource_for_federation=Mock(), http_client=None, config=self.mock_config, + keyring=Mock(), ) hs.handlers = JustPresenceHandlers(hs) @@ -230,6 +230,7 @@ class PresenceInvitesTestCase(unittest.TestCase): resource_for_federation=self.mock_federation_resource, http_client=self.mock_http_client, config=self.mock_config, + keyring=Mock(), ) hs.handlers = JustPresenceHandlers(hs) @@ -533,6 +534,7 @@ class PresencePushTestCase(unittest.TestCase): resource_for_federation=self.mock_federation_resource, http_client=self.mock_http_client, config=self.mock_config, + keyring=Mock(), ) hs.handlers = JustPresenceHandlers(hs) @@ -1025,7 +1027,8 @@ class PresencePollingTestCase(unittest.TestCase): resource_for_client=Mock(), resource_for_federation=self.mock_federation_resource, http_client=self.mock_http_client, - config = self.mock_config, + config=self.mock_config, + keyring=Mock(), ) hs.handlers = JustPresenceHandlers(hs) diff --git a/tests/handlers/test_typing.py b/tests/handlers/test_typing.py index 0ab829b9ad..ff40343b8c 100644 --- a/tests/handlers/test_typing.py +++ b/tests/handlers/test_typing.py @@ -79,6 +79,7 @@ class TypingNotificationsTestCase(unittest.TestCase): resource_for_federation=self.mock_federation_resource, http_client=self.mock_http_client, config=self.mock_config, + keyring=Mock(), ) hs.handlers = JustTypingNotificationHandlers(hs) -- cgit 1.4.1 From 4f11518934c3e032a763f115a73261414d67f87b Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Thu, 2 Oct 2014 13:57:48 +0100 Subject: Split PlainHttpClient into separate clients for talking to Identity servers and talking to Capatcha servers --- synapse/app/homeserver.py | 4 +- synapse/handlers/directory.py | 4 +- synapse/handlers/login.py | 6 +- synapse/handlers/register.py | 11 +- synapse/http/client.py | 290 ++++++++++++++++++++------------------- tests/handlers/test_directory.py | 4 +- 6 files changed, 163 insertions(+), 156 deletions(-) (limited to 'tests/handlers') diff --git a/synapse/app/homeserver.py b/synapse/app/homeserver.py index 2f1b954902..61d574a00f 100755 --- a/synapse/app/homeserver.py +++ b/synapse/app/homeserver.py @@ -25,7 +25,7 @@ from twisted.web.static import File from twisted.web.server import Site from synapse.http.server import JsonResource, RootRedirect from synapse.http.content_repository import ContentRepoResource -from synapse.http.client import TwistedHttpClient +from synapse.http.client import MatrixHttpClient from synapse.api.urls import ( CLIENT_PREFIX, FEDERATION_PREFIX, WEB_CLIENT_PREFIX, CONTENT_REPO_PREFIX ) @@ -47,7 +47,7 @@ logger = logging.getLogger(__name__) class SynapseHomeServer(HomeServer): def build_http_client(self): - return TwistedHttpClient(self) + return MatrixHttpClient(self) def build_resource_for_client(self): return JsonResource() diff --git a/synapse/handlers/directory.py b/synapse/handlers/directory.py index 84c3a1d56f..cec7737e09 100644 --- a/synapse/handlers/directory.py +++ b/synapse/handlers/directory.py @@ -18,7 +18,7 @@ from twisted.internet import defer from ._base import BaseHandler from synapse.api.errors import SynapseError -from synapse.http.client import HttpClient +from synapse.http.client import MatrixHttpClient from synapse.api.events.room import RoomAliasesEvent import logging @@ -98,7 +98,7 @@ class DirectoryHandler(BaseHandler): query_type="directory", args={ "room_alias": room_alias.to_string(), - HttpClient.RETRY_DNS_LOOKUP_FAILURES: False + MatrixHttpClient.RETRY_DNS_LOOKUP_FAILURES: False } ) diff --git a/synapse/handlers/login.py b/synapse/handlers/login.py index 80ffdd2726..3f152e18f0 100644 --- a/synapse/handlers/login.py +++ b/synapse/handlers/login.py @@ -17,7 +17,7 @@ from twisted.internet import defer from ._base import BaseHandler from synapse.api.errors import LoginError, Codes -from synapse.http.client import PlainHttpClient +from synapse.http.client import IdentityServerHttpClient from synapse.util.emailutils import EmailException import synapse.util.emailutils as emailutils @@ -97,10 +97,10 @@ class LoginHandler(BaseHandler): @defer.inlineCallbacks def _query_email(self, email): - httpCli = PlainHttpClient(self.hs) + httpCli = IdentityServerHttpClient(self.hs) data = yield httpCli.get_json( 'matrix.org:8090', # TODO FIXME This should be configurable. "/_matrix/identity/api/v1/lookup?medium=email&address=" + "%s" % urllib.quote(email) ) - defer.returnValue(data) \ No newline at end of file + defer.returnValue(data) diff --git a/synapse/handlers/register.py b/synapse/handlers/register.py index a019d770d4..266495056e 100644 --- a/synapse/handlers/register.py +++ b/synapse/handlers/register.py @@ -22,7 +22,8 @@ from synapse.api.errors import ( ) from ._base import BaseHandler import synapse.util.stringutils as stringutils -from synapse.http.client import PlainHttpClient +from synapse.http.client import IdentityServerHttpClient +from synapse.http.client import CaptchaServerHttpClient import base64 import bcrypt @@ -154,7 +155,9 @@ class RegistrationHandler(BaseHandler): @defer.inlineCallbacks def _threepid_from_creds(self, creds): - httpCli = PlainHttpClient(self.hs) + # TODO: get this from the homeserver rather than creating a new one for + # each request + httpCli = IdentityServerHttpClient(self.hs) # XXX: make this configurable! trustedIdServers = ['matrix.org:8090'] if not creds['idServer'] in trustedIdServers: @@ -203,7 +206,9 @@ class RegistrationHandler(BaseHandler): @defer.inlineCallbacks def _submit_captcha(self, ip_addr, private_key, challenge, response): - client = PlainHttpClient(self.hs) + # TODO: get this from the homeserver rather than creating a new one for + # each request + client = CaptchaServerHttpClient(self.hs) data = yield client.post_urlencoded_get_raw( "www.google.com:80", "/recaptcha/api/verify", diff --git a/synapse/http/client.py b/synapse/http/client.py index 822afeec1d..e02cce5642 100644 --- a/synapse/http/client.py +++ b/synapse/http/client.py @@ -36,49 +36,6 @@ import urllib logger = logging.getLogger(__name__) -class HttpClient(object): - """ Interface for talking json over http - """ - RETRY_DNS_LOOKUP_FAILURES = "__retry_dns" - - def put_json(self, destination, path, data): - """ Sends the specifed json data using PUT - - Args: - destination (str): The remote server to send the HTTP request - to. - path (str): The HTTP path. - data (dict): A dict containing the data that will be used as - the request body. This will be encoded as JSON. - - Returns: - Deferred: Succeeds when we get a 2xx HTTP response. The result - will be the decoded JSON body. On a 4xx or 5xx error response a - CodeMessageException is raised. - """ - pass - - def get_json(self, destination, path, args=None): - """ Get's some json from the given host homeserver and path - - Args: - destination (str): The remote server to send the HTTP request - to. - path (str): The HTTP path. - args (dict): A dictionary used to create query strings, defaults to - None. - **Note**: The value of each key is assumed to be an iterable - and *not* a string. - - Returns: - Deferred: Succeeds when we get *any* HTTP response. - - The result of the deferred is a tuple of `(code, response)`, - where `response` is a dict representing the decoded JSON body. - """ - pass - - class MatrixHttpAgent(_AgentBase): def __init__(self, reactor, pool=None): @@ -102,23 +59,114 @@ class MatrixHttpAgent(_AgentBase): parsed_URI.originForm) -class TwistedHttpClient(HttpClient): - """ Wrapper around the twisted HTTP client api. +class BaseHttpClient(object): + """Base class for HTTP clients using twisted. + """ + + def __init__(self, hs): + self.agent = MatrixHttpAgent(reactor) + self.hs = hs + + @defer.inlineCallbacks + def _create_request(self, destination, method, path_bytes, param_bytes=b"", + query_bytes=b"", producer=None, headers_dict={}, + retry_on_dns_fail=True, on_send_callback=None): + """ Creates and sends a request to the given url + """ + headers_dict[b"User-Agent"] = [b"Synapse"] + headers_dict[b"Host"] = [destination] + + logger.debug("Sending request to %s: %s %s;%s?%s", + destination, method, path_bytes, param_bytes, query_bytes) + + logger.debug( + "Types: %s", + [ + type(destination), type(method), type(path_bytes), + type(param_bytes), + type(query_bytes) + ] + ) + + retries_left = 5 + + endpoint = self._getEndpoint(reactor, destination); + + while True: + if on_send_callback: + on_send_callback(destination, method, path_bytes, producer) + + try: + response = yield self.agent.request( + destination, + endpoint, + method, + path_bytes, + param_bytes, + query_bytes, + Headers(headers_dict), + producer + ) + + logger.debug("Got response to %s", method) + break + except Exception as e: + if not retry_on_dns_fail and isinstance(e, DNSLookupError): + logger.warn("DNS Lookup failed to %s with %s", destination, + e) + raise SynapseError(400, "Domain specified not found.") + + logger.exception("Got error in _create_request") + _print_ex(e) + + if retries_left: + yield sleep(2 ** (5 - retries_left)) + retries_left -= 1 + else: + raise + + if 200 <= response.code < 300: + # We need to update the transactions table to say it was sent? + pass + else: + # :'( + # Update transactions table? + logger.error( + "Got response %d %s", response.code, response.phrase + ) + raise CodeMessageException( + response.code, response.phrase + ) + + defer.returnValue(response) + + +class MatrixHttpClient(BaseHttpClient): + """ Wrapper around the twisted HTTP client api. Implements Attributes: agent (twisted.web.client.Agent): The twisted Agent used to send the requests. """ - def __init__(self, hs): - self.agent = MatrixHttpAgent(reactor) - self.hs = hs + RETRY_DNS_LOOKUP_FAILURES = "__retry_dns" @defer.inlineCallbacks def put_json(self, destination, path, data, on_send_callback=None): - if destination in _destination_mappings: - destination = _destination_mappings[destination] + """ Sends the specifed json data using PUT + Args: + destination (str): The remote server to send the HTTP request + to. + path (str): The HTTP path. + data (dict): A dict containing the data that will be used as + the request body. This will be encoded as JSON. + + Returns: + Deferred: Succeeds when we get a 2xx HTTP response. The result + will be the decoded JSON body. On a 4xx or 5xx error response a + CodeMessageException is raised. + """ response = yield self._create_request( destination.encode("ascii"), "PUT", @@ -136,9 +184,23 @@ class TwistedHttpClient(HttpClient): @defer.inlineCallbacks def get_json(self, destination, path, args={}): - if destination in _destination_mappings: - destination = _destination_mappings[destination] + """ Get's some json from the given host homeserver and path + + Args: + destination (str): The remote server to send the HTTP request + to. + path (str): The HTTP path. + args (dict): A dictionary used to create query strings, defaults to + None. + **Note**: The value of each key is assumed to be an iterable + and *not* a string. + + Returns: + Deferred: Succeeds when we get *any* HTTP response. + The result of the deferred is a tuple of `(code, response)`, + where `response` is a dict representing the decoded JSON body. + """ logger.debug("get_json args: %s", args) retry_on_dns_fail = True @@ -163,6 +225,22 @@ class TwistedHttpClient(HttpClient): defer.returnValue(json.loads(body)) + + def _getEndpoint(self, reactor, destination): + return matrix_endpoint( + reactor, destination, timeout=10, + ssl_context_factory=self.hs.tls_context_factory + ) + + +class IdentityServerHttpClient(BaseHttpClient): + """Separate HTTP client for talking to the Identity servers since they + don't use SRV records and talk x-www-form-urlencoded rather than JSON. + """ + def _getEndpoint(self, reactor, destination): + #TODO: This should be talking TLS + return matrix_endpoint(reactor, destination, timeout=10) + @defer.inlineCallbacks def post_urlencoded_get_json(self, destination, path, args={}): if destination in _destination_mappings: @@ -176,16 +254,25 @@ class TwistedHttpClient(HttpClient): "POST", path.encode("ascii"), producer=FileBodyProducer(StringIO(urllib.urlencode(args))), - headers_dict={"Content-Type": ["application/x-www-form-urlencoded"]} + headers_dict={ + "Content-Type": ["application/x-www-form-urlencoded"] + } ) body = yield readBody(response) defer.returnValue(json.loads(body)) - - # XXX FIXME : I'm so sorry. + + +class CaptchaServerHttpClient(MatrixHttpClient): + """Separate HTTP client for talking to google's captcha servers""" + + def _getEndpoint(self, reactor, destination): + return matrix_endpoint(reactor, destination, timeout=10) + @defer.inlineCallbacks - def post_urlencoded_get_raw(self, destination, path, accept_partial=False, args={}): + def post_urlencoded_get_raw(self, destination, path, accept_partial=False, + args={}): if destination in _destination_mappings: destination = _destination_mappings[destination] @@ -196,7 +283,9 @@ class TwistedHttpClient(HttpClient): "POST", path.encode("ascii"), producer=FileBodyProducer(StringIO(urllib.urlencode(args))), - headers_dict={"Content-Type": ["application/x-www-form-urlencoded"]} + headers_dict={ + "Content-Type": ["application/x-www-form-urlencoded"] + } ) try: @@ -207,93 +296,6 @@ class TwistedHttpClient(HttpClient): defer.returnValue(e.response) else: raise e - - - @defer.inlineCallbacks - def _create_request(self, destination, method, path_bytes, param_bytes=b"", - query_bytes=b"", producer=None, headers_dict={}, - retry_on_dns_fail=True, on_send_callback=None): - """ Creates and sends a request to the given url - """ - headers_dict[b"User-Agent"] = [b"Synapse"] - headers_dict[b"Host"] = [destination] - - logger.debug("Sending request to %s: %s %s;%s?%s", - destination, method, path_bytes, param_bytes, query_bytes) - - logger.debug( - "Types: %s", - [ - type(destination), type(method), type(path_bytes), - type(param_bytes), - type(query_bytes) - ] - ) - - retries_left = 5 - - # TODO: setup and pass in an ssl_context to enable TLS - endpoint = self._getEndpoint(reactor, destination); - - while True: - if on_send_callback: - on_send_callback(destination, method, path_bytes, producer) - - try: - response = yield self.agent.request( - destination, - endpoint, - method, - path_bytes, - param_bytes, - query_bytes, - Headers(headers_dict), - producer - ) - - logger.debug("Got response to %s", method) - break - except Exception as e: - if not retry_on_dns_fail and isinstance(e, DNSLookupError): - logger.warn("DNS Lookup failed to %s with %s", destination, - e) - raise SynapseError(400, "Domain specified not found.") - - logger.exception("Got error in _create_request") - _print_ex(e) - - if retries_left: - yield sleep(2 ** (5 - retries_left)) - retries_left -= 1 - else: - raise - - if 200 <= response.code < 300: - # We need to update the transactions table to say it was sent? - pass - else: - # :'( - # Update transactions table? - logger.error( - "Got response %d %s", response.code, response.phrase - ) - raise CodeMessageException( - response.code, response.phrase - ) - - defer.returnValue(response) - - def _getEndpoint(self, reactor, destination): - return matrix_endpoint( - reactor, destination, timeout=10, - ssl_context_factory=self.hs.tls_context_factory - ) - - -class PlainHttpClient(TwistedHttpClient): - def _getEndpoint(self, reactor, destination): - return matrix_endpoint(reactor, destination, timeout=10) - def _print_ex(e): if hasattr(e, "reasons") and e.reasons: diff --git a/tests/handlers/test_directory.py b/tests/handlers/test_directory.py index dd5d85dde6..0c31502dd4 100644 --- a/tests/handlers/test_directory.py +++ b/tests/handlers/test_directory.py @@ -20,7 +20,7 @@ from twisted.internet import defer from mock import Mock from synapse.server import HomeServer -from synapse.http.client import HttpClient +from synapse.http.client import MatrixHttpClient from synapse.handlers.directory import DirectoryHandler from synapse.storage.directory import RoomAliasMapping @@ -95,7 +95,7 @@ class DirectoryTestCase(unittest.TestCase): query_type="directory", args={ "room_alias": "#another:remote", - HttpClient.RETRY_DNS_LOOKUP_FAILURES: False + MatrixHttpClient.RETRY_DNS_LOOKUP_FAILURES: False } ) -- cgit 1.4.1 From 574377636ee4eafba50580fc4d7a1d0793774332 Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Thu, 2 Oct 2014 14:09:15 +0100 Subject: Add a keyword argument to get_json to avoid retrying on DNS failures. Rather than passing MatrixHttpClient.RETRY_DNS_LOOKUP_FAILURES as a fake query string parameter --- synapse/federation/replication.py | 7 +++++-- synapse/federation/transport.py | 5 +++-- synapse/handlers/directory.py | 5 ++--- synapse/http/client.py | 9 +-------- tests/federation/test_federation.py | 5 +++-- tests/handlers/test_directory.py | 5 ++--- 6 files changed, 16 insertions(+), 20 deletions(-) (limited to 'tests/handlers') diff --git a/synapse/federation/replication.py b/synapse/federation/replication.py index 96b82f00cb..5f96f79998 100644 --- a/synapse/federation/replication.py +++ b/synapse/federation/replication.py @@ -159,7 +159,8 @@ class ReplicationLayer(object): return defer.succeed(None) @log_function - def make_query(self, destination, query_type, args): + def make_query(self, destination, query_type, args, + retry_on_dns_fail=True): """Sends a federation Query to a remote homeserver of the given type and arguments. @@ -174,7 +175,9 @@ class ReplicationLayer(object): a Deferred which will eventually yield a JSON object from the response """ - return self.transport_layer.make_query(destination, query_type, args) + return self.transport_layer.make_query( + destination, query_type, args, retry_on_dns_fail=retry_on_dns_fail + ) @defer.inlineCallbacks @log_function diff --git a/synapse/federation/transport.py b/synapse/federation/transport.py index afc777ec9e..93296af204 100644 --- a/synapse/federation/transport.py +++ b/synapse/federation/transport.py @@ -193,13 +193,14 @@ class TransportLayer(object): @defer.inlineCallbacks @log_function - def make_query(self, destination, query_type, args): + def make_query(self, destination, query_type, args, retry_on_dns_fail): path = PREFIX + "/query/%s" % query_type response = yield self.client.get_json( destination=destination, path=path, - args=args + args=args, + retry_on_dns_fail=retry_on_dns_fail, ) defer.returnValue(response) diff --git a/synapse/handlers/directory.py b/synapse/handlers/directory.py index cec7737e09..a56830d520 100644 --- a/synapse/handlers/directory.py +++ b/synapse/handlers/directory.py @@ -18,7 +18,6 @@ from twisted.internet import defer from ._base import BaseHandler from synapse.api.errors import SynapseError -from synapse.http.client import MatrixHttpClient from synapse.api.events.room import RoomAliasesEvent import logging @@ -98,8 +97,8 @@ class DirectoryHandler(BaseHandler): query_type="directory", args={ "room_alias": room_alias.to_string(), - MatrixHttpClient.RETRY_DNS_LOOKUP_FAILURES: False - } + }, + retry_on_dns_fail=False, ) if result and "room_id" in result and "servers" in result: diff --git a/synapse/http/client.py b/synapse/http/client.py index e02cce5642..57b49355f2 100644 --- a/synapse/http/client.py +++ b/synapse/http/client.py @@ -183,7 +183,7 @@ class MatrixHttpClient(BaseHttpClient): defer.returnValue((response.code, body)) @defer.inlineCallbacks - def get_json(self, destination, path, args={}): + def get_json(self, destination, path, args={}, retry_on_dns_fail=True): """ Get's some json from the given host homeserver and path Args: @@ -203,13 +203,6 @@ class MatrixHttpClient(BaseHttpClient): """ logger.debug("get_json args: %s", args) - retry_on_dns_fail = True - if HttpClient.RETRY_DNS_LOOKUP_FAILURES in args: - # FIXME: This isn't ideal, but the interface exposed in get_json - # isn't comprehensive enough to give caller's any control over - # their connection mechanics. - retry_on_dns_fail = args.pop(HttpClient.RETRY_DNS_LOOKUP_FAILURES) - query_bytes = urllib.urlencode(args, True) logger.debug("Query bytes: %s Retry DNS: %s", args, retry_on_dns_fail) diff --git a/tests/federation/test_federation.py b/tests/federation/test_federation.py index bb17e9aafe..d95b9013a3 100644 --- a/tests/federation/test_federation.py +++ b/tests/federation/test_federation.py @@ -253,7 +253,7 @@ class FederationTestCase(unittest.TestCase): response = yield self.federation.make_query( destination="remote", query_type="a-question", - args={"one": "1", "two": "2"} + args={"one": "1", "two": "2"}, ) self.assertEquals({"your": "response"}, response) @@ -261,7 +261,8 @@ class FederationTestCase(unittest.TestCase): self.mock_http_client.get_json.assert_called_with( destination="remote", path="/_matrix/federation/v1/query/a-question", - args={"one": "1", "two": "2"} + args={"one": "1", "two": "2"}, + retry_on_dns_fail=True, ) @defer.inlineCallbacks diff --git a/tests/handlers/test_directory.py b/tests/handlers/test_directory.py index 0c31502dd4..e10a49a8ac 100644 --- a/tests/handlers/test_directory.py +++ b/tests/handlers/test_directory.py @@ -20,7 +20,6 @@ from twisted.internet import defer from mock import Mock from synapse.server import HomeServer -from synapse.http.client import MatrixHttpClient from synapse.handlers.directory import DirectoryHandler from synapse.storage.directory import RoomAliasMapping @@ -95,8 +94,8 @@ class DirectoryTestCase(unittest.TestCase): query_type="directory", args={ "room_alias": "#another:remote", - MatrixHttpClient.RETRY_DNS_LOOKUP_FAILURES: False - } + }, + retry_on_dns_fail=False, ) @defer.inlineCallbacks -- cgit 1.4.1 From 10ef8e6e4bb9d50fd2c636cfbb66d3dd6d6f94e9 Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Mon, 13 Oct 2014 11:49:40 +0100 Subject: SYN-75 sign at the request level rather than the transaction level --- synapse/federation/replication.py | 13 ---------- synapse/federation/transport.py | 18 +++---------- synapse/federation/units.py | 5 ++++ synapse/http/client.py | 52 ++++++++++++++++++++++++++++++++----- tests/federation/test_federation.py | 4 +-- tests/handlers/test_presence.py | 26 +++++++++---------- tests/handlers/test_typing.py | 4 +-- 7 files changed, 70 insertions(+), 52 deletions(-) (limited to 'tests/handlers') diff --git a/synapse/federation/replication.py b/synapse/federation/replication.py index b4235585a3..2346d55045 100644 --- a/synapse/federation/replication.py +++ b/synapse/federation/replication.py @@ -25,8 +25,6 @@ from .persistence import PduActions, TransactionActions from synapse.util.logutils import log_function -from syutil.crypto.jsonsign import sign_json - import logging @@ -66,8 +64,6 @@ class ReplicationLayer(object): hs, self.transaction_actions, transport_layer ) - self.keyring = hs.get_keyring() - self.handler = None self.edu_handlers = {} self.query_handlers = {} @@ -296,10 +292,6 @@ class ReplicationLayer(object): @defer.inlineCallbacks @log_function def on_incoming_transaction(self, transaction_data): - yield self.keyring.verify_json_for_server( - transaction_data["origin"], transaction_data - ) - transaction = Transaction(**transaction_data) for p in transaction.pdus: @@ -500,7 +492,6 @@ class _TransactionQueue(object): """ def __init__(self, hs, transaction_actions, transport_layer): - self.signing_key = hs.config.signing_key[0] self.server_name = hs.hostname self.transaction_actions = transaction_actions self.transport_layer = transport_layer @@ -615,9 +606,6 @@ class _TransactionQueue(object): # Actually send the transaction - server_name = self.server_name - signing_key = self.signing_key - # FIXME (erikj): This is a bit of a hack to make the Pdu age # keys work def json_data_cb(): @@ -627,7 +615,6 @@ class _TransactionQueue(object): for p in data["pdus"]: if "age_ts" in p: p["age"] = now - int(p["age_ts"]) - data = sign_json(data, server_name, signing_key) return data code, response = yield self.transport_layer.send_transaction( diff --git a/synapse/federation/transport.py b/synapse/federation/transport.py index 1f864f5fa7..48fc9fbf5e 100644 --- a/synapse/federation/transport.py +++ b/synapse/federation/transport.py @@ -163,27 +163,15 @@ class TransportLayer(object): if transaction.destination == self.server_name: raise RuntimeError("Transport layer cannot send to itself!") - if json_data_callback is None: - def json_data_callback(): - return transaction.get_dict() - - # FIXME (erikj): This is a bit of a hack to make the Pdu age - # keys work - def cb(destination, method, path_bytes, producer): - json_data = json_data_callback() - del json_data["destination"] - del json_data["transaction_id"] - producer.reset(json_data) - + # FIXME: This is only used by the tests. The actual json sent is + # generated by the json_data_callback. json_data = transaction.get_dict() - del json_data["destination"] - del json_data["transaction_id"] code, response = yield self.client.put_json( transaction.destination, path=PREFIX + "/send/%s/" % transaction.transaction_id, data=json_data, - on_send_callback=cb, + json_data_callback=json_data_callback, ) logger.debug( diff --git a/synapse/federation/units.py b/synapse/federation/units.py index 1ca123d1bf..ecca35ac43 100644 --- a/synapse/federation/units.py +++ b/synapse/federation/units.py @@ -190,6 +190,11 @@ class Transaction(JsonEncodedObject): "destination", ] + internal_keys = [ + "transaction_id", + "destination", + ] + required_keys = [ "transaction_id", "origin", diff --git a/synapse/http/client.py b/synapse/http/client.py index 0e8fa2eb25..62fe14fa5e 100644 --- a/synapse/http/client.py +++ b/synapse/http/client.py @@ -26,6 +26,8 @@ from syutil.jsonutil import encode_canonical_json from synapse.api.errors import CodeMessageException, SynapseError +from syutil.crypto.jsonsign import sign_json + from StringIO import StringIO import json @@ -147,7 +149,7 @@ class BaseHttpClient(object): class MatrixHttpClient(BaseHttpClient): - """ Wrapper around the twisted HTTP client api. Implements + """ Wrapper around the twisted HTTP client api. Implements Attributes: agent (twisted.web.client.Agent): The twisted Agent used to send the @@ -156,8 +158,38 @@ class MatrixHttpClient(BaseHttpClient): RETRY_DNS_LOOKUP_FAILURES = "__retry_dns" + def __init__(self, hs): + self.signing_key = hs.config.signing_key[0] + self.server_name = hs.hostname + BaseHttpClient.__init__(self, hs) + + def sign_request(self, destination, method, url_bytes, headers_dict, + content=None): + request = { + "method": method, + "uri": url_bytes, + "origin": self.server_name, + "destination": destination, + } + + if content is not None: + request["content"] = content + + request = sign_json(request, self.server_name, self.signing_key) + + auth_headers = [] + + for key,sig in request["signatures"][self.server_name].items(): + auth_headers.append( + "X-Matrix origin=%s,key=\"%s\",sig=\"%s\"" % ( + self.server_name, key, sig, + ) + ) + + headers_dict["Authorization"] = auth_headers + @defer.inlineCallbacks - def put_json(self, destination, path, data, on_send_callback=None): + def put_json(self, destination, path, data={}, json_data_callback=None): """ Sends the specifed json data using PUT Args: @@ -166,6 +198,8 @@ class MatrixHttpClient(BaseHttpClient): path (str): The HTTP path. data (dict): A dict containing the data that will be used as the request body. This will be encoded as JSON. + json_data_callback (callable): A callable returning the dict to + use as the request body. Returns: Deferred: Succeeds when we get a 2xx HTTP response. The result @@ -173,13 +207,16 @@ class MatrixHttpClient(BaseHttpClient): CodeMessageException is raised. """ - if not on_send_callback: - def on_send_callback(destination, method, path_bytes, producer): - pass + if not json_data_callback: + def json_data_callback(): + return data def body_callback(method, url_bytes, headers_dict): - producer = _JsonProducer(data) - on_send_callback(destination, method, path, producer) + json_data = json_data_callback() + self.sign_request( + destination, method, url_bytes, headers_dict, json_data + ) + producer = _JsonProducer(json_data) return producer response = yield self._create_request( @@ -221,6 +258,7 @@ class MatrixHttpClient(BaseHttpClient): logger.debug("Query bytes: %s Retry DNS: %s", args, retry_on_dns_fail) def body_callback(method, url_bytes, headers_dict): + self.sign_request(destination, method, url_bytes, headers_dict) return None response = yield self._create_request( diff --git a/tests/federation/test_federation.py b/tests/federation/test_federation.py index 01f07ff36d..91edeaa4b9 100644 --- a/tests/federation/test_federation.py +++ b/tests/federation/test_federation.py @@ -186,7 +186,7 @@ class FederationTestCase(unittest.TestCase): }, ] }, - on_send_callback=ANY, + json_data_callback=ANY, ) @defer.inlineCallbacks @@ -218,7 +218,7 @@ class FederationTestCase(unittest.TestCase): } ], }, - on_send_callback=ANY, + json_data_callback=ANY, ) @defer.inlineCallbacks diff --git a/tests/handlers/test_presence.py b/tests/handlers/test_presence.py index c9406b70c4..15022b8d05 100644 --- a/tests/handlers/test_presence.py +++ b/tests/handlers/test_presence.py @@ -300,7 +300,7 @@ class PresenceInvitesTestCase(unittest.TestCase): "observed_user": "@cabbage:elsewhere", } ), - on_send_callback=ANY, + json_data_callback=ANY, ), defer.succeed((200, "OK")) ) @@ -329,7 +329,7 @@ class PresenceInvitesTestCase(unittest.TestCase): "observed_user": "@apple:test", } ), - on_send_callback=ANY, + json_data_callback=ANY, ), defer.succeed((200, "OK")) ) @@ -365,7 +365,7 @@ class PresenceInvitesTestCase(unittest.TestCase): "observed_user": "@durian:test", } ), - on_send_callback=ANY, + json_data_callback=ANY, ), defer.succeed((200, "OK")) ) @@ -786,7 +786,7 @@ class PresencePushTestCase(unittest.TestCase): ], } ), - on_send_callback=ANY, + json_data_callback=ANY, ), defer.succeed((200, "OK")) ) @@ -802,7 +802,7 @@ class PresencePushTestCase(unittest.TestCase): ], } ), - on_send_callback=ANY, + json_data_callback=ANY, ), defer.succeed((200, "OK")) ) @@ -928,7 +928,7 @@ class PresencePushTestCase(unittest.TestCase): ], } ), - on_send_callback=ANY, + json_data_callback=ANY, ), defer.succeed((200, "OK")) ) @@ -943,7 +943,7 @@ class PresencePushTestCase(unittest.TestCase): ], } ), - on_send_callback=ANY, + json_data_callback=ANY, ), defer.succeed((200, "OK")) ) @@ -973,7 +973,7 @@ class PresencePushTestCase(unittest.TestCase): ], } ), - on_send_callback=ANY, + json_data_callback=ANY, ), defer.succeed((200, "OK")) ) @@ -1175,7 +1175,7 @@ class PresencePollingTestCase(unittest.TestCase): "poll": [ "@potato:remote" ], }, ), - on_send_callback=ANY, + json_data_callback=ANY, ), defer.succeed((200, "OK")) ) @@ -1188,7 +1188,7 @@ class PresencePollingTestCase(unittest.TestCase): "push": [ {"user_id": "@clementine:test" }], }, ), - on_send_callback=ANY, + json_data_callback=ANY, ), defer.succeed((200, "OK")) ) @@ -1217,7 +1217,7 @@ class PresencePollingTestCase(unittest.TestCase): "push": [ {"user_id": "@fig:test" }], }, ), - on_send_callback=ANY, + json_data_callback=ANY, ), defer.succeed((200, "OK")) ) @@ -1250,7 +1250,7 @@ class PresencePollingTestCase(unittest.TestCase): "unpoll": [ "@potato:remote" ], }, ), - on_send_callback=ANY, + json_data_callback=ANY, ), defer.succeed((200, "OK")) ) @@ -1282,7 +1282,7 @@ class PresencePollingTestCase(unittest.TestCase): ], }, ), - on_send_callback=ANY, + json_data_callback=ANY, ), defer.succeed((200, "OK")) ) diff --git a/tests/handlers/test_typing.py b/tests/handlers/test_typing.py index ff40343b8c..064b04c217 100644 --- a/tests/handlers/test_typing.py +++ b/tests/handlers/test_typing.py @@ -175,7 +175,7 @@ class TypingNotificationsTestCase(unittest.TestCase): "typing": True, } ), - on_send_callback=ANY, + json_data_callback=ANY, ), defer.succeed((200, "OK")) ) @@ -226,7 +226,7 @@ class TypingNotificationsTestCase(unittest.TestCase): "typing": False, } ), - on_send_callback=ANY, + json_data_callback=ANY, ), defer.succeed((200, "OK")) ) -- cgit 1.4.1 From 9aed791fc38790eae6c24e154e7f82ac8509295d Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Tue, 14 Oct 2014 16:44:27 +0100 Subject: SYN-103: Ignore the 'origin' key in received EDUs. Instead take the origin from the transaction itself --- synapse/federation/replication.py | 2 +- synapse/federation/units.py | 8 ++++++-- tests/federation/test_federation.py | 1 + tests/handlers/test_presence.py | 1 + tests/handlers/test_typing.py | 1 + 5 files changed, 10 insertions(+), 3 deletions(-) (limited to 'tests/handlers') diff --git a/synapse/federation/replication.py b/synapse/federation/replication.py index 2346d55045..9363ac7300 100644 --- a/synapse/federation/replication.py +++ b/synapse/federation/replication.py @@ -319,7 +319,7 @@ class ReplicationLayer(object): if hasattr(transaction, "edus"): for edu in [Edu(**x) for x in transaction.edus]: - self.received_edu(edu.origin, edu.edu_type, edu.content) + self.received_edu(transaction.origin, edu.edu_type, edu.content) results = yield defer.DeferredList(dl) diff --git a/synapse/federation/units.py b/synapse/federation/units.py index ecca35ac43..d97aeb698e 100644 --- a/synapse/federation/units.py +++ b/synapse/federation/units.py @@ -156,11 +156,15 @@ class Edu(JsonEncodedObject): ] required_keys = [ - "origin", - "destination", "edu_type", ] +# TODO: SYN-103: Remove "origin" and "destination" keys. +# internal_keys = [ +# "origin", +# "destination", +# ] + class Transaction(JsonEncodedObject): """ A transaction is a list of Pdus and Edus to be sent to a remote home diff --git a/tests/federation/test_federation.py b/tests/federation/test_federation.py index 8d277d6612..d86ce83b28 100644 --- a/tests/federation/test_federation.py +++ b/tests/federation/test_federation.py @@ -211,6 +211,7 @@ class FederationTestCase(unittest.TestCase): "pdus": [], "edus": [ { + # TODO: SYN-103: Remove "origin" and "destination" "origin": "test", "destination": "remote", "edu_type": "m.test", diff --git a/tests/handlers/test_presence.py b/tests/handlers/test_presence.py index 15022b8d05..84985a8066 100644 --- a/tests/handlers/test_presence.py +++ b/tests/handlers/test_presence.py @@ -43,6 +43,7 @@ def _expect_edu(destination, edu_type, content, origin="test"): "pdus": [], "edus": [ { + # TODO: SYN-103: Remove "origin" and "destination" keys. "origin": origin, "destination": destination, "edu_type": edu_type, diff --git a/tests/handlers/test_typing.py b/tests/handlers/test_typing.py index 064b04c217..b685373deb 100644 --- a/tests/handlers/test_typing.py +++ b/tests/handlers/test_typing.py @@ -33,6 +33,7 @@ def _expect_edu(destination, edu_type, content, origin="test"): "pdus": [], "edus": [ { + # TODO: SYN-103: Remove "origin" and "destination" keys. "origin": origin, "destination": destination, "edu_type": edu_type, -- cgit 1.4.1 From f5cf7ac25b311fda8a2d553f07437b3648c66f6c Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Fri, 17 Oct 2014 17:12:25 +0100 Subject: SPEC-7: Rename 'ts' to 'origin_server_ts' --- synapse/api/events/factory.py | 4 ++-- synapse/federation/pdu_codec.py | 4 ++-- synapse/federation/persistence.py | 2 +- synapse/federation/replication.py | 4 ++-- synapse/federation/units.py | 16 ++++++++-------- synapse/storage/_base.py | 2 +- synapse/storage/pdu.py | 2 +- synapse/storage/schema/pdu.sql | 2 +- synapse/storage/schema/transactions.sql | 4 ++-- synapse/storage/transactions.py | 14 +++++++------- tests/federation/test_federation.py | 14 +++++++------- tests/federation/test_pdu_codec.py | 4 ++-- tests/handlers/test_federation.py | 6 +++--- tests/handlers/test_presence.py | 2 +- tests/handlers/test_typing.py | 2 +- tests/test_state.py | 2 +- 16 files changed, 42 insertions(+), 42 deletions(-) (limited to 'tests/handlers') diff --git a/synapse/api/events/factory.py b/synapse/api/events/factory.py index 0d94850cec..74d0ef77f4 100644 --- a/synapse/api/events/factory.py +++ b/synapse/api/events/factory.py @@ -58,8 +58,8 @@ class EventFactory(object): random_string(10), self.hs.hostname ) - if "ts" not in kwargs: - kwargs["ts"] = int(self.clock.time_msec()) + if "origin_server_ts" not in kwargs: + kwargs["origin_server_ts"] = int(self.clock.time_msec()) # The "age" key is a delta timestamp that should be converted into an # absolute timestamp the minute we see it. diff --git a/synapse/federation/pdu_codec.py b/synapse/federation/pdu_codec.py index cef61108dd..e8180d94fd 100644 --- a/synapse/federation/pdu_codec.py +++ b/synapse/federation/pdu_codec.py @@ -96,7 +96,7 @@ class PduCodec(object): if k not in ["event_id", "room_id", "type", "prev_events"] }) - if "ts" not in kwargs: - kwargs["ts"] = int(self.clock.time_msec()) + if "origin_server_ts" not in kwargs: + kwargs["origin_server_ts"] = int(self.clock.time_msec()) return Pdu(**kwargs) diff --git a/synapse/federation/persistence.py b/synapse/federation/persistence.py index de36a80e41..7043fcc504 100644 --- a/synapse/federation/persistence.py +++ b/synapse/federation/persistence.py @@ -157,7 +157,7 @@ class TransactionActions(object): transaction.prev_ids = yield self.store.prep_send_transaction( transaction.transaction_id, transaction.destination, - transaction.ts, + transaction.origin_server_ts, [(p["pdu_id"], p["origin"]) for p in transaction.pdus] ) diff --git a/synapse/federation/replication.py b/synapse/federation/replication.py index 9363ac7300..092411eaf9 100644 --- a/synapse/federation/replication.py +++ b/synapse/federation/replication.py @@ -421,7 +421,7 @@ class ReplicationLayer(object): return Transaction( origin=self.server_name, pdus=pdus, - ts=int(self._clock.time_msec()), + origin_server_ts=int(self._clock.time_msec()), destination=None, ) @@ -589,7 +589,7 @@ class _TransactionQueue(object): logger.debug("TX [%s] Persisting transaction...", destination) transaction = Transaction.create_new( - ts=self._clock.time_msec(), + origin_server_ts=self._clock.time_msec(), transaction_id=str(self._next_txn_id), origin=self.server_name, destination=destination, diff --git a/synapse/federation/units.py b/synapse/federation/units.py index d97aeb698e..dccac2aca7 100644 --- a/synapse/federation/units.py +++ b/synapse/federation/units.py @@ -40,7 +40,7 @@ class Pdu(JsonEncodedObject): { "pdu_id": "78c", - "ts": 1404835423000, + "origin_server_ts": 1404835423000, "origin": "bar", "prev_ids": [ ["23b", "foo"], @@ -55,7 +55,7 @@ class Pdu(JsonEncodedObject): "pdu_id", "context", "origin", - "ts", + "origin_server_ts", "pdu_type", "destinations", "transaction_id", @@ -82,7 +82,7 @@ class Pdu(JsonEncodedObject): "pdu_id", "context", "origin", - "ts", + "origin_server_ts", "pdu_type", "content", ] @@ -186,7 +186,7 @@ class Transaction(JsonEncodedObject): "transaction_id", "origin", "destination", - "ts", + "origin_server_ts", "previous_ids", "pdus", "edus", @@ -203,7 +203,7 @@ class Transaction(JsonEncodedObject): "transaction_id", "origin", "destination", - "ts", + "origin_server_ts", "pdus", ] @@ -225,10 +225,10 @@ class Transaction(JsonEncodedObject): @staticmethod def create_new(pdus, **kwargs): """ Used to create a new transaction. Will auto fill out - transaction_id and ts keys. + transaction_id and origin_server_ts keys. """ - if "ts" not in kwargs: - raise KeyError("Require 'ts' to construct a Transaction") + if "origin_server_ts" not in kwargs: + raise KeyError("Require 'origin_server_ts' to construct a Transaction") if "transaction_id" not in kwargs: raise KeyError( "Require 'transaction_id' to construct a Transaction" diff --git a/synapse/storage/_base.py b/synapse/storage/_base.py index dba50f1213..30c5103cdd 100644 --- a/synapse/storage/_base.py +++ b/synapse/storage/_base.py @@ -361,7 +361,7 @@ class SQLBaseStore(object): if "age_ts" not in d: # For compatibility - d["age_ts"] = d["ts"] if "ts" in d else 0 + d["age_ts"] = d["origin_server_ts"] if "origin_server_ts" in d else 0 return self.event_factory.create_event( etype=d["type"], diff --git a/synapse/storage/pdu.py b/synapse/storage/pdu.py index d70467dcd6..61ea979b8a 100644 --- a/synapse/storage/pdu.py +++ b/synapse/storage/pdu.py @@ -789,7 +789,7 @@ class PdusTable(Table): "origin", "context", "pdu_type", - "ts", + "origin_server_ts", "depth", "is_state", "content_json", diff --git a/synapse/storage/schema/pdu.sql b/synapse/storage/schema/pdu.sql index 16e111a56c..5cc8669912 100644 --- a/synapse/storage/schema/pdu.sql +++ b/synapse/storage/schema/pdu.sql @@ -18,7 +18,7 @@ CREATE TABLE IF NOT EXISTS pdus( origin TEXT, context TEXT, pdu_type TEXT, - ts INTEGER, + origin_server_ts INTEGER, depth INTEGER DEFAULT 0 NOT NULL, is_state BOOL, content_json TEXT, diff --git a/synapse/storage/schema/transactions.sql b/synapse/storage/schema/transactions.sql index 88e3e4e04d..5f8d01327a 100644 --- a/synapse/storage/schema/transactions.sql +++ b/synapse/storage/schema/transactions.sql @@ -16,7 +16,7 @@ CREATE TABLE IF NOT EXISTS received_transactions( transaction_id TEXT, origin TEXT, - ts INTEGER, + origin_server_ts INTEGER, response_code INTEGER, response_json TEXT, has_been_referenced BOOL default 0, -- Whether thishas been referenced by a prev_tx @@ -35,7 +35,7 @@ CREATE TABLE IF NOT EXISTS sent_transactions( destination TEXT, response_code INTEGER DEFAULT 0, response_json TEXT, - ts INTEGER + origin_server_ts INTEGER ); CREATE INDEX IF NOT EXISTS sent_transaction_dest ON sent_transactions(destination); diff --git a/synapse/storage/transactions.py b/synapse/storage/transactions.py index ab4599b468..a9fa959d49 100644 --- a/synapse/storage/transactions.py +++ b/synapse/storage/transactions.py @@ -87,7 +87,7 @@ class TransactionStore(SQLBaseStore): txn.execute(query, (code, response_json, transaction_id, origin)) - def prep_send_transaction(self, transaction_id, destination, ts, pdu_list): + def prep_send_transaction(self, transaction_id, destination, origin_server_ts, pdu_list): """Persists an outgoing transaction and calculates the values for the previous transaction id list. @@ -97,7 +97,7 @@ class TransactionStore(SQLBaseStore): Args: transaction_id (str) destination (str) - ts (int) + origin_server_ts (int) pdu_list (list) Returns: @@ -106,10 +106,10 @@ class TransactionStore(SQLBaseStore): return self.runInteraction( self._prep_send_transaction, - transaction_id, destination, ts, pdu_list + transaction_id, destination, origin_server_ts, pdu_list ) - def _prep_send_transaction(self, txn, transaction_id, destination, ts, + def _prep_send_transaction(self, txn, transaction_id, destination, origin_server_ts, pdu_list): # First we find out what the prev_txs should be. @@ -131,7 +131,7 @@ class TransactionStore(SQLBaseStore): None, transaction_id=transaction_id, destination=destination, - ts=ts, + origin_server_ts=origin_server_ts, response_code=0, response_json=None )) @@ -251,7 +251,7 @@ class ReceivedTransactionsTable(Table): fields = [ "transaction_id", "origin", - "ts", + "origin_server_ts", "response_code", "response_json", "has_been_referenced", @@ -267,7 +267,7 @@ class SentTransactions(Table): "id", "transaction_id", "destination", - "ts", + "origin_server_ts", "response_code", "response_json", ] diff --git a/tests/federation/test_federation.py b/tests/federation/test_federation.py index d86ce83b28..8b1202f6e4 100644 --- a/tests/federation/test_federation.py +++ b/tests/federation/test_federation.py @@ -99,7 +99,7 @@ class FederationTestCase(unittest.TestCase): origin="red", context="my-context", pdu_type="m.topic", - ts=123456789000, + origin_server_ts=123456789000, depth=1, is_state=True, content_json='{"topic":"The topic"}', @@ -134,7 +134,7 @@ class FederationTestCase(unittest.TestCase): origin="red", context="my-context", pdu_type="m.text", - ts=123456789001, + origin_server_ts=123456789001, depth=1, content_json='{"text":"Here is the message"}', ) @@ -158,7 +158,7 @@ class FederationTestCase(unittest.TestCase): origin="red", destinations=["remote"], context="my-context", - ts=123456789002, + origin_server_ts=123456789002, pdu_type="m.test", content={"testing": "content here"}, depth=1, @@ -170,14 +170,14 @@ class FederationTestCase(unittest.TestCase): "remote", path="/_matrix/federation/v1/send/1000000/", data={ - "ts": 1000000, + "origin_server_ts": 1000000, "origin": "test", "pdus": [ { "origin": "red", "pdu_id": "abc123def456", "prev_pdus": [], - "ts": 123456789002, + "origin_server_ts": 123456789002, "context": "my-context", "pdu_type": "m.test", "is_state": False, @@ -207,7 +207,7 @@ class FederationTestCase(unittest.TestCase): path="/_matrix/federation/v1/send/1000000/", data={ "origin": "test", - "ts": 1000000, + "origin_server_ts": 1000000, "pdus": [], "edus": [ { @@ -234,7 +234,7 @@ class FederationTestCase(unittest.TestCase): "/_matrix/federation/v1/send/1001000/", """{ "origin": "remote", - "ts": 1001000, + "origin_server_ts": 1001000, "pdus": [], "edus": [ { diff --git a/tests/federation/test_pdu_codec.py b/tests/federation/test_pdu_codec.py index 344e1baf60..0754ef92e8 100644 --- a/tests/federation/test_pdu_codec.py +++ b/tests/federation/test_pdu_codec.py @@ -68,7 +68,7 @@ class PduCodecTestCase(unittest.TestCase): context="rooooom", pdu_type="m.room.message", origin="bar.com", - ts=12345, + origin_server_ts=12345, depth=5, prev_pdus=[("alice", "bob.com")], is_state=False, @@ -123,7 +123,7 @@ class PduCodecTestCase(unittest.TestCase): context="rooooom", pdu_type="m.room.topic", origin="bar.com", - ts=12345, + origin_server_ts=12345, depth=5, prev_pdus=[("alice", "bob.com")], is_state=True, diff --git a/tests/handlers/test_federation.py b/tests/handlers/test_federation.py index 35c3a4df7b..219b2c4c5e 100644 --- a/tests/handlers/test_federation.py +++ b/tests/handlers/test_federation.py @@ -68,7 +68,7 @@ class FederationTestCase(unittest.TestCase): pdu_type=MessageEvent.TYPE, context="foo", content={"msgtype": u"fooo"}, - ts=0, + origin_server_ts=0, pdu_id="a", origin="b", ) @@ -95,7 +95,7 @@ class FederationTestCase(unittest.TestCase): target_host=self.hostname, context=room_id, content={}, - ts=0, + origin_server_ts=0, pdu_id="a", origin="b", ) @@ -127,7 +127,7 @@ class FederationTestCase(unittest.TestCase): state_key="@red:not%s" % self.hostname, context=room_id, content={}, - ts=0, + origin_server_ts=0, pdu_id="a", origin="b", ) diff --git a/tests/handlers/test_presence.py b/tests/handlers/test_presence.py index 84985a8066..1850deacf5 100644 --- a/tests/handlers/test_presence.py +++ b/tests/handlers/test_presence.py @@ -39,7 +39,7 @@ ONLINE = PresenceState.ONLINE def _expect_edu(destination, edu_type, content, origin="test"): return { "origin": origin, - "ts": 1000000, + "origin_server_ts": 1000000, "pdus": [], "edus": [ { diff --git a/tests/handlers/test_typing.py b/tests/handlers/test_typing.py index b685373deb..f1d3b27f74 100644 --- a/tests/handlers/test_typing.py +++ b/tests/handlers/test_typing.py @@ -29,7 +29,7 @@ from synapse.handlers.typing import TypingNotificationHandler def _expect_edu(destination, edu_type, content, origin="test"): return { "origin": origin, - "ts": 1000000, + "origin_server_ts": 1000000, "pdus": [], "edus": [ { diff --git a/tests/test_state.py b/tests/test_state.py index b1624f0b25..4b1feaf410 100644 --- a/tests/test_state.py +++ b/tests/test_state.py @@ -599,7 +599,7 @@ def new_fake_pdu(pdu_id, context, pdu_type, state_key, prev_state_id, prev_state_id=prev_state_id, origin="example.com", context="context", - ts=1405353060021, + origin_server_ts=1405353060021, depth=depth, content_json="{}", unrecognized_keys="{}", -- cgit 1.4.1