diff options
| author | Erik Johnston <erik@matrix.org> | 2014-10-17 21:02:16 +0100 |
|---|---|---|
| committer | Erik Johnston <erik@matrix.org> | 2014-10-17 21:02:16 +0100 |
| commit | 449739e6a364759a86f73417626af49d6a794e3d (patch) | |
| tree | 04b2f39c91ec6098ffe3b380442787ac3ce2908f /synapse | |
| parent | Merge branch 'release-v0.3.4' of github.com:matrix-org/synapse (diff) | |
| parent | Check that we have auth headers and fail nicely (diff) | |
| download | synapse-449739e6a364759a86f73417626af49d6a794e3d.tar.xz | |
Merge branch 'release-v0.4.0' of github.com:matrix-org/synapse
Diffstat (limited to 'synapse')
45 files changed, 1129 insertions, 700 deletions
diff --git a/synapse/__init__.py b/synapse/__init__.py index a340a5db66..979eac08a7 100644 --- a/synapse/__init__.py +++ b/synapse/__init__.py @@ -16,4 +16,4 @@ """ This is a reference implementation of a synapse home server. """ -__version__ = "0.3.4" +__version__ = "0.4.0" diff --git a/synapse/api/auth.py b/synapse/api/auth.py index 9bfd25c86e..e1b1823cd7 100644 --- a/synapse/api/auth.py +++ b/synapse/api/auth.py @@ -206,6 +206,7 @@ class Auth(object): defer.returnValue(True) + @defer.inlineCallbacks def get_user_by_req(self, request): """ Get a registered user's ID. @@ -218,7 +219,25 @@ class Auth(object): """ # Can optionally look elsewhere in the request (e.g. headers) try: - return self.get_user_by_token(request.args["access_token"][0]) + access_token = request.args["access_token"][0] + user_info = yield self.get_user_by_token(access_token) + user = user_info["user"] + + ip_addr = self.hs.get_ip_from_request(request) + user_agent = request.requestHeaders.getRawHeaders( + "User-Agent", + default=[""] + )[0] + if user and access_token and ip_addr: + self.store.insert_client_ip( + user=user, + access_token=access_token, + device_id=user_info["device_id"], + ip=ip_addr, + user_agent=user_agent + ) + + defer.returnValue(user) except KeyError: raise AuthError(403, "Missing access token.") @@ -227,21 +246,32 @@ class Auth(object): """ Get a registered user's ID. Args: - token (str)- The access token to get the user by. + token (str): The access token to get the user by. Returns: - UserID : User ID object of the user who has that access token. + dict : dict that includes the user, device_id, and whether the + user is a server admin. Raises: AuthError if no user by that token exists or the token is invalid. """ try: - user_id = yield self.store.get_user_by_token(token=token) - if not user_id: + ret = yield self.store.get_user_by_token(token=token) + if not ret: raise StoreError() - defer.returnValue(self.hs.parse_userid(user_id)) + + user_info = { + "admin": bool(ret.get("admin", False)), + "device_id": ret.get("device_id"), + "user": self.hs.parse_userid(ret.get("name")), + } + + defer.returnValue(user_info) except StoreError: raise AuthError(403, "Unrecognised access token.", errcode=Codes.UNKNOWN_TOKEN) + def is_server_admin(self, user): + return self.store.is_server_admin(user) + @defer.inlineCallbacks @log_function def _can_send_event(self, event): diff --git a/synapse/api/errors.py b/synapse/api/errors.py index 88175602c4..6d7d499fea 100644 --- a/synapse/api/errors.py +++ b/synapse/api/errors.py @@ -19,6 +19,7 @@ import logging class Codes(object): + UNAUTHORIZED = "M_UNAUTHORIZED" FORBIDDEN = "M_FORBIDDEN" BAD_JSON = "M_BAD_JSON" NOT_JSON = "M_NOT_JSON" diff --git a/synapse/api/events/factory.py b/synapse/api/events/factory.py index 0d94850cec..74d0ef77f4 100644 --- a/synapse/api/events/factory.py +++ b/synapse/api/events/factory.py @@ -58,8 +58,8 @@ class EventFactory(object): random_string(10), self.hs.hostname ) - if "ts" not in kwargs: - kwargs["ts"] = int(self.clock.time_msec()) + if "origin_server_ts" not in kwargs: + kwargs["origin_server_ts"] = int(self.clock.time_msec()) # The "age" key is a delta timestamp that should be converted into an # absolute timestamp the minute we see it. diff --git a/synapse/api/urls.py b/synapse/api/urls.py index 6314f31f7a..6dc19305b7 100644 --- a/synapse/api/urls.py +++ b/synapse/api/urls.py @@ -18,4 +18,5 @@ CLIENT_PREFIX = "/_matrix/client/api/v1" FEDERATION_PREFIX = "/_matrix/federation/v1" WEB_CLIENT_PREFIX = "/_matrix/client" -CONTENT_REPO_PREFIX = "/_matrix/content" \ No newline at end of file +CONTENT_REPO_PREFIX = "/_matrix/content" +SERVER_KEY_PREFIX = "/_matrix/key/v1" diff --git a/synapse/app/homeserver.py b/synapse/app/homeserver.py index 2f1b954902..6394bc27d1 100755 --- a/synapse/app/homeserver.py +++ b/synapse/app/homeserver.py @@ -25,9 +25,11 @@ from twisted.web.static import File from twisted.web.server import Site from synapse.http.server import JsonResource, RootRedirect from synapse.http.content_repository import ContentRepoResource -from synapse.http.client import TwistedHttpClient +from synapse.http.server_key_resource import LocalKey +from synapse.http.client import MatrixHttpClient from synapse.api.urls import ( - CLIENT_PREFIX, FEDERATION_PREFIX, WEB_CLIENT_PREFIX, CONTENT_REPO_PREFIX + CLIENT_PREFIX, FEDERATION_PREFIX, WEB_CLIENT_PREFIX, CONTENT_REPO_PREFIX, + SERVER_KEY_PREFIX, ) from synapse.config.homeserver import HomeServerConfig from synapse.crypto import context_factory @@ -47,7 +49,7 @@ logger = logging.getLogger(__name__) class SynapseHomeServer(HomeServer): def build_http_client(self): - return TwistedHttpClient(self) + return MatrixHttpClient(self) def build_resource_for_client(self): return JsonResource() @@ -63,6 +65,9 @@ class SynapseHomeServer(HomeServer): self, self.upload_dir, self.auth, self.content_addr ) + def build_resource_for_server_key(self): + return LocalKey(self) + def build_db_pool(self): return adbapi.ConnectionPool( "sqlite3", self.get_db_name(), @@ -88,7 +93,8 @@ class SynapseHomeServer(HomeServer): desired_tree = [ (CLIENT_PREFIX, self.get_resource_for_client()), (FEDERATION_PREFIX, self.get_resource_for_federation()), - (CONTENT_REPO_PREFIX, self.get_resource_for_content_repo()) + (CONTENT_REPO_PREFIX, self.get_resource_for_content_repo()), + (SERVER_KEY_PREFIX, self.get_resource_for_server_key()), ] if web_client: logger.info("Adding the web client.") diff --git a/synapse/config/_base.py b/synapse/config/_base.py index 35bcece2c0..b3aeff327c 100644 --- a/synapse/config/_base.py +++ b/synapse/config/_base.py @@ -123,6 +123,8 @@ class Config(object): # style mode markers into the file, to hint to people that # this is a YAML file. yaml.dump(config, config_file, default_flow_style=False) + print "A config file has been generated in %s for server name '%s') with corresponding SSL keys and self-signed certificates. Please review this file and customise it to your needs." % (config_args.config_path, config['server_name']) + print "If this server name is incorrect, you will need to regenerate the SSL certificates" sys.exit(0) return cls(args) diff --git a/synapse/config/repository.py b/synapse/config/repository.py index 407c8d6c24..b71d30227c 100644 --- a/synapse/config/repository.py +++ b/synapse/config/repository.py @@ -14,7 +14,6 @@ # limitations under the License. from ._base import Config -import os class ContentRepositoryConfig(Config): def __init__(self, args): diff --git a/synapse/config/server.py b/synapse/config/server.py index 516e4cf882..d9d8d0e14e 100644 --- a/synapse/config/server.py +++ b/synapse/config/server.py @@ -13,10 +13,9 @@ # See the License for the specific language governing permissions and # limitations under the License. -import nacl.signing import os -from ._base import Config -from syutil.base64util import encode_base64, decode_base64 +from ._base import Config, ConfigError +import syutil.crypto.signing_key class ServerConfig(Config): @@ -70,9 +69,16 @@ class ServerConfig(Config): "content repository") def read_signing_key(self, signing_key_path): - signing_key_base64 = self.read_file(signing_key_path, "signing_key") - signing_key_bytes = decode_base64(signing_key_base64) - return nacl.signing.SigningKey(signing_key_bytes) + signing_keys = self.read_file(signing_key_path, "signing_key") + try: + return syutil.crypto.signing_key.read_signing_keys( + signing_keys.splitlines(True) + ) + except Exception as e: + raise ConfigError( + "Error reading signing_key." + " Try running again with --generate-config" + ) @classmethod def generate_config(cls, args, config_dir_path): @@ -86,6 +92,21 @@ class ServerConfig(Config): if not os.path.exists(args.signing_key_path): with open(args.signing_key_path, "w") as signing_key_file: - key = nacl.signing.SigningKey.generate() - signing_key_file.write(encode_base64(key.encode())) - + syutil.crypto.signing_key.write_signing_keys( + signing_key_file, + (syutil.crypto.SigningKey.generate("auto"),), + ) + else: + signing_keys = cls.read_file(args.signing_key_path, "signing_key") + if len(signing_keys.split("\n")[0].split()) == 1: + # handle keys in the old format. + key = syutil.crypto.signing_key.decode_signing_key_base64( + syutil.crypto.signing_key.NACL_ED25519, + "auto", + signing_keys.split("\n")[0] + ) + with open(args.signing_key_path, "w") as signing_key_file: + syutil.crypto.signing_key.write_signing_keys( + signing_key_file, + (key,), + ) diff --git a/synapse/crypto/keyclient.py b/synapse/crypto/keyclient.py index c11df5c529..5949ea0573 100644 --- a/synapse/crypto/keyclient.py +++ b/synapse/crypto/keyclient.py @@ -15,9 +15,10 @@ from twisted.web.http import HTTPClient +from twisted.internet.protocol import Factory from twisted.internet import defer, reactor -from twisted.internet.protocol import ClientFactory -from twisted.names.srvconnect import SRVConnector +from twisted.internet.endpoints import connectProtocol +from synapse.http.endpoint import matrix_endpoint import json import logging @@ -30,15 +31,19 @@ def fetch_server_key(server_name, ssl_context_factory): """Fetch the keys for a remote server.""" factory = SynapseKeyClientFactory() + endpoint = matrix_endpoint( + reactor, server_name, ssl_context_factory, timeout=30 + ) - SRVConnector( - reactor, "matrix", server_name, factory, - protocol="tcp", connectFuncName="connectSSL", defaultPort=443, - connectFuncKwArgs=dict(contextFactory=ssl_context_factory)).connect() - - server_key, server_certificate = yield factory.remote_key - - defer.returnValue((server_key, server_certificate)) + for i in range(5): + try: + protocol = yield endpoint.connect(factory) + server_response, server_certificate = yield protocol.remote_key + defer.returnValue((server_response, server_certificate)) + return + except Exception as e: + logger.exception(e) + raise IOError("Cannot get key for %s" % server_name) class SynapseKeyClientError(Exception): @@ -51,69 +56,47 @@ class SynapseKeyClientProtocol(HTTPClient): the server and extracts the X.509 certificate for the remote peer from the SSL connection.""" + timeout = 30 + + def __init__(self): + self.remote_key = defer.Deferred() + def connectionMade(self): logger.debug("Connected to %s", self.transport.getHost()) - self.sendCommand(b"GET", b"/key") + self.sendCommand(b"GET", b"/_matrix/key/v1/") self.endHeaders() self.timer = reactor.callLater( - self.factory.timeout_seconds, + self.timeout, self.on_timeout ) def handleStatus(self, version, status, message): if status != b"200": - logger.info("Non-200 response from %s: %s %s", - self.transport.getHost(), status, message) + #logger.info("Non-200 response from %s: %s %s", + # self.transport.getHost(), status, message) self.transport.abortConnection() def handleResponse(self, response_body_bytes): try: json_response = json.loads(response_body_bytes) except ValueError: - logger.info("Invalid JSON response from %s", - self.transport.getHost()) + #logger.info("Invalid JSON response from %s", + # self.transport.getHost()) self.transport.abortConnection() return certificate = self.transport.getPeerCertificate() - self.factory.on_remote_key((json_response, certificate)) + self.remote_key.callback((json_response, certificate)) self.transport.abortConnection() self.timer.cancel() def on_timeout(self): logger.debug("Timeout waiting for response from %s", self.transport.getHost()) + self.remote_key.errback(IOError("Timeout waiting for response")) self.transport.abortConnection() -class SynapseKeyClientFactory(ClientFactory): +class SynapseKeyClientFactory(Factory): protocol = SynapseKeyClientProtocol - max_retries = 5 - timeout_seconds = 30 - - def __init__(self): - self.succeeded = False - self.retries = 0 - self.remote_key = defer.Deferred() - def on_remote_key(self, key): - self.succeeded = True - self.remote_key.callback(key) - - def retry_connection(self, connector): - self.retries += 1 - if self.retries < self.max_retries: - connector.connector = None - connector.connect() - else: - self.remote_key.errback( - SynapseKeyClientError("Max retries exceeded")) - - def clientConnectionFailed(self, connector, reason): - logger.info("Connection failed %s", reason) - self.retry_connection(connector) - - def clientConnectionLost(self, connector, reason): - logger.info("Connection lost %s", reason) - if not self.succeeded: - self.retry_connection(connector) diff --git a/synapse/crypto/keyring.py b/synapse/crypto/keyring.py new file mode 100644 index 0000000000..2440d604c3 --- /dev/null +++ b/synapse/crypto/keyring.py @@ -0,0 +1,155 @@ +# -*- coding: utf-8 -*- +# Copyright 2014 OpenMarket Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from synapse.crypto.keyclient import fetch_server_key +from twisted.internet import defer +from syutil.crypto.jsonsign import verify_signed_json, signature_ids +from syutil.crypto.signing_key import ( + is_signing_algorithm_supported, decode_verify_key_bytes +) +from syutil.base64util import decode_base64, encode_base64 +from synapse.api.errors import SynapseError, Codes + +from OpenSSL import crypto + +import logging + + +logger = logging.getLogger(__name__) + + +class Keyring(object): + def __init__(self, hs): + self.store = hs.get_datastore() + self.clock = hs.get_clock() + self.hs = hs + + @defer.inlineCallbacks + def verify_json_for_server(self, server_name, json_object): + logger.debug("Verifying for %s", server_name) + key_ids = signature_ids(json_object, server_name) + if not key_ids: + raise SynapseError( + 400, + "Not signed with a supported algorithm", + Codes.UNAUTHORIZED, + ) + try: + verify_key = yield self.get_server_verify_key(server_name, key_ids) + except IOError: + raise SynapseError( + 502, + "Error downloading keys for %s" % (server_name,), + Codes.UNAUTHORIZED, + ) + except: + raise SynapseError( + 401, + "No key for %s with id %s" % (server_name, key_ids), + Codes.UNAUTHORIZED, + ) + try: + verify_signed_json(json_object, server_name, verify_key) + except: + raise SynapseError( + 401, + "Invalid signature for server %s with key %s:%s" % ( + server_name, verify_key.alg, verify_key.version + ), + Codes.UNAUTHORIZED, + ) + + @defer.inlineCallbacks + def get_server_verify_key(self, server_name, key_ids): + """Finds a verification key for the server with one of the key ids. + Args: + server_name (str): The name of the server to fetch a key for. + keys_ids (list of str): The key_ids to check for. + """ + + # Check the datastore to see if we have one cached. + cached = yield self.store.get_server_verify_keys(server_name, key_ids) + + if cached: + defer.returnValue(cached[0]) + return + + # Try to fetch the key from the remote server. + # TODO(markjh): Ratelimit requests to a given server. + + (response, tls_certificate) = yield fetch_server_key( + server_name, self.hs.tls_context_factory + ) + + # Check the response. + + x509_certificate_bytes = crypto.dump_certificate( + crypto.FILETYPE_ASN1, tls_certificate + ) + + if ("signatures" not in response + or server_name not in response["signatures"]): + raise ValueError("Key response not signed by remote server") + + if "tls_certificate" not in response: + raise ValueError("Key response missing TLS certificate") + + tls_certificate_b64 = response["tls_certificate"] + + if encode_base64(x509_certificate_bytes) != tls_certificate_b64: + raise ValueError("TLS certificate doesn't match") + + verify_keys = {} + for key_id, key_base64 in response["verify_keys"].items(): + if is_signing_algorithm_supported(key_id): + key_bytes = decode_base64(key_base64) + verify_key = decode_verify_key_bytes(key_id, key_bytes) + verify_keys[key_id] = verify_key + + for key_id in response["signatures"][server_name]: + if key_id not in response["verify_keys"]: + raise ValueError( + "Key response must include verification keys for all" + " signatures" + ) + if key_id in verify_keys: + verify_signed_json( + response, + server_name, + verify_keys[key_id] + ) + + # Cache the result in the datastore. + + time_now_ms = self.clock.time_msec() + + self.store.store_server_certificate( + server_name, + server_name, + time_now_ms, + tls_certificate, + ) + + for key_id, key in verify_keys.items(): + self.store.store_server_verify_key( + server_name, server_name, time_now_ms, key + ) + + for key_id in key_ids: + if key_id in verify_keys: + defer.returnValue(verify_keys[key_id]) + return + + raise ValueError("No verification key found for given key ids") diff --git a/synapse/crypto/keyserver.py b/synapse/crypto/keyserver.py deleted file mode 100644 index a23484dbae..0000000000 --- a/synapse/crypto/keyserver.py +++ /dev/null @@ -1,111 +0,0 @@ -# -*- coding: utf-8 -*- -# Copyright 2014 OpenMarket Ltd -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - - -from twisted.internet import reactor, ssl -from twisted.web import server -from twisted.web.resource import Resource -from twisted.python.log import PythonLoggingObserver - -from synapse.crypto.resource.key import LocalKey -from synapse.crypto.config import load_config - -from syutil.base64util import decode_base64 - -from OpenSSL import crypto, SSL - -import logging -import nacl.signing -import sys - - -class KeyServerSSLContextFactory(ssl.ContextFactory): - """Factory for PyOpenSSL SSL contexts that are used to handle incoming - connections and to make connections to remote servers.""" - - def __init__(self, key_server): - self._context = SSL.Context(SSL.SSLv23_METHOD) - self.configure_context(self._context, key_server) - - @staticmethod - def configure_context(context, key_server): - context.set_options(SSL.OP_NO_SSLv2 | SSL.OP_NO_SSLv3) - context.use_certificate(key_server.tls_certificate) - context.use_privatekey(key_server.tls_private_key) - context.load_tmp_dh(key_server.tls_dh_params_path) - context.set_cipher_list("!ADH:HIGH+kEDH:!AECDH:HIGH+kEECDH") - - def getContext(self): - return self._context - - -class KeyServer(object): - """An HTTPS server serving LocalKey and RemoteKey resources.""" - - def __init__(self, server_name, tls_certificate_path, tls_private_key_path, - tls_dh_params_path, signing_key_path, bind_host, bind_port): - self.server_name = server_name - self.tls_certificate = self.read_tls_certificate(tls_certificate_path) - self.tls_private_key = self.read_tls_private_key(tls_private_key_path) - self.tls_dh_params_path = tls_dh_params_path - self.signing_key = self.read_signing_key(signing_key_path) - self.bind_host = bind_host - self.bind_port = int(bind_port) - self.ssl_context_factory = KeyServerSSLContextFactory(self) - - @staticmethod - def read_tls_certificate(cert_path): - with open(cert_path) as cert_file: - cert_pem = cert_file.read() - return crypto.load_certificate(crypto.FILETYPE_PEM, cert_pem) - - @staticmethod - def read_tls_private_key(private_key_path): - with open(private_key_path) as private_key_file: - private_key_pem = private_key_file.read() - return crypto.load_privatekey(crypto.FILETYPE_PEM, private_key_pem) - - @staticmethod - def read_signing_key(signing_key_path): - with open(signing_key_path) as signing_key_file: - signing_key_b64 = signing_key_file.read() - signing_key_bytes = decode_base64(signing_key_b64) - return nacl.signing.SigningKey(signing_key_bytes) - - def run(self): - root = Resource() - root.putChild("key", LocalKey(self)) - site = server.Site(root) - reactor.listenSSL( - self.bind_port, - site, - self.ssl_context_factory, - interface=self.bind_host - ) - - logging.basicConfig(level=logging.DEBUG) - observer = PythonLoggingObserver() - observer.start() - - reactor.run() - - -def main(): - key_server = KeyServer(**load_config(__doc__, sys.argv[1:])) - key_server.run() - - -if __name__ == "__main__": - main() diff --git a/synapse/crypto/resource/__init__.py b/synapse/crypto/resource/__init__.py deleted file mode 100644 index 9bff9ec169..0000000000 --- a/synapse/crypto/resource/__init__.py +++ /dev/null @@ -1,15 +0,0 @@ -# -*- coding: utf-8 -*- -# Copyright 2014 OpenMarket Ltd -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - diff --git a/synapse/crypto/resource/key.py b/synapse/crypto/resource/key.py deleted file mode 100644 index 48d14b9f4a..0000000000 --- a/synapse/crypto/resource/key.py +++ /dev/null @@ -1,161 +0,0 @@ -# -*- coding: utf-8 -*- -# Copyright 2014 OpenMarket Ltd -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - - -from twisted.web.resource import Resource -from twisted.web.server import NOT_DONE_YET -from twisted.internet import defer -from synapse.http.server import respond_with_json_bytes -from synapse.crypto.keyclient import fetch_server_key -from syutil.crypto.jsonsign import sign_json, verify_signed_json -from syutil.base64util import encode_base64, decode_base64 -from syutil.jsonutil import encode_canonical_json -from OpenSSL import crypto -from nacl.signing import VerifyKey -import logging - - -logger = logging.getLogger(__name__) - - -class LocalKey(Resource): - """HTTP resource containing encoding the TLS X.509 certificate and NACL - signature verification keys for this server:: - - GET /key HTTP/1.1 - - HTTP/1.1 200 OK - Content-Type: application/json - { - "server_name": "this.server.example.com" - "signature_verify_key": # base64 encoded NACL verification key. - "tls_certificate": # base64 ASN.1 DER encoded X.509 tls cert. - "signatures": { - "this.server.example.com": # NACL signature for this server. - } - } - """ - - def __init__(self, key_server): - self.key_server = key_server - self.response_body = encode_canonical_json( - self.response_json_object(key_server) - ) - Resource.__init__(self) - - @staticmethod - def response_json_object(key_server): - verify_key_bytes = key_server.signing_key.verify_key.encode() - x509_certificate_bytes = crypto.dump_certificate( - crypto.FILETYPE_ASN1, - key_server.tls_certificate - ) - json_object = { - u"server_name": key_server.server_name, - u"signature_verify_key": encode_base64(verify_key_bytes), - u"tls_certificate": encode_base64(x509_certificate_bytes) - } - signed_json = sign_json( - json_object, - key_server.server_name, - key_server.signing_key - ) - return signed_json - - def getChild(self, name, request): - logger.info("getChild %s %s", name, request) - if name == '': - return self - else: - return RemoteKey(name, self.key_server) - - def render_GET(self, request): - return respond_with_json_bytes(request, 200, self.response_body) - - -class RemoteKey(Resource): - """HTTP resource for retreiving the TLS certificate and NACL signature - verification keys for a another server. Checks that the reported X.509 TLS - certificate matches the one used in the HTTPS connection. Checks that the - NACL signature for the remote server is valid. Returns JSON signed by both - the remote server and by this server. - - GET /key/remote.server.example.com HTTP/1.1 - - HTTP/1.1 200 OK - Content-Type: application/json - { - "server_name": "remote.server.example.com" - "signature_verify_key": # base64 encoded NACL verification key. - "tls_certificate": # base64 ASN.1 DER encoded X.509 tls cert. - "signatures": { - "remote.server.example.com": # NACL signature for remote server. - "this.server.example.com": # NACL signature for this server. - } - } - """ - - isLeaf = True - - def __init__(self, server_name, key_server): - self.server_name = server_name - self.key_server = key_server - Resource.__init__(self) - - def render_GET(self, request): - self._async_render_GET(request) - return NOT_DONE_YET - - @defer.inlineCallbacks - def _async_render_GET(self, request): - try: - server_keys, certificate = yield fetch_server_key( - self.server_name, - self.key_server.ssl_context_factory - ) - - resp_server_name = server_keys[u"server_name"] - verify_key_b64 = server_keys[u"signature_verify_key"] - tls_certificate_b64 = server_keys[u"tls_certificate"] - verify_key = VerifyKey(decode_base64(verify_key_b64)) - - if resp_server_name != self.server_name: - raise ValueError("Wrong server name '%s' != '%s'" % - (resp_server_name, self.server_name)) - - x509_certificate_bytes = crypto.dump_certificate( - crypto.FILETYPE_ASN1, - certificate - ) - - if encode_base64(x509_certificate_bytes) != tls_certificate_b64: - raise ValueError("TLS certificate doesn't match") - - verify_signed_json(server_keys, self.server_name, verify_key) - - signed_json = sign_json( - server_keys, - self.key_server.server_name, - self.key_server.signing_key - ) - - json_bytes = encode_canonical_json(signed_json) - respond_with_json_bytes(request, 200, json_bytes) - - except Exception as e: - json_bytes = encode_canonical_json({ - u"error": {u"code": 502, u"message": e.message} - }) - respond_with_json_bytes(request, 502, json_bytes) diff --git a/synapse/federation/__init__.py b/synapse/federation/__init__.py index 1351b68fd6..0112588656 100644 --- a/synapse/federation/__init__.py +++ b/synapse/federation/__init__.py @@ -22,6 +22,7 @@ from .transport import TransportLayer def initialize_http_replication(homeserver): transport = TransportLayer( + homeserver, homeserver.hostname, server=homeserver.get_resource_for_federation(), client=homeserver.get_http_client() diff --git a/synapse/federation/pdu_codec.py b/synapse/federation/pdu_codec.py index cef61108dd..e8180d94fd 100644 --- a/synapse/federation/pdu_codec.py +++ b/synapse/federation/pdu_codec.py @@ -96,7 +96,7 @@ class PduCodec(object): if k not in ["event_id", "room_id", "type", "prev_events"] }) - if "ts" not in kwargs: - kwargs["ts"] = int(self.clock.time_msec()) + if "origin_server_ts" not in kwargs: + kwargs["origin_server_ts"] = int(self.clock.time_msec()) return Pdu(**kwargs) diff --git a/synapse/federation/persistence.py b/synapse/federation/persistence.py index de36a80e41..7043fcc504 100644 --- a/synapse/federation/persistence.py +++ b/synapse/federation/persistence.py @@ -157,7 +157,7 @@ class TransactionActions(object): transaction.prev_ids = yield self.store.prep_send_transaction( transaction.transaction_id, transaction.destination, - transaction.ts, + transaction.origin_server_ts, [(p["pdu_id"], p["origin"]) for p in transaction.pdus] ) diff --git a/synapse/federation/replication.py b/synapse/federation/replication.py index 96b82f00cb..092411eaf9 100644 --- a/synapse/federation/replication.py +++ b/synapse/federation/replication.py @@ -159,7 +159,8 @@ class ReplicationLayer(object): return defer.succeed(None) @log_function - def make_query(self, destination, query_type, args): + def make_query(self, destination, query_type, args, + retry_on_dns_fail=True): """Sends a federation Query to a remote homeserver of the given type and arguments. @@ -174,7 +175,9 @@ class ReplicationLayer(object): a Deferred which will eventually yield a JSON object from the response """ - return self.transport_layer.make_query(destination, query_type, args) + return self.transport_layer.make_query( + destination, query_type, args, retry_on_dns_fail=retry_on_dns_fail + ) @defer.inlineCallbacks @log_function @@ -316,7 +319,7 @@ class ReplicationLayer(object): if hasattr(transaction, "edus"): for edu in [Edu(**x) for x in transaction.edus]: - self.received_edu(edu.origin, edu.edu_type, edu.content) + self.received_edu(transaction.origin, edu.edu_type, edu.content) results = yield defer.DeferredList(dl) @@ -418,7 +421,7 @@ class ReplicationLayer(object): return Transaction( origin=self.server_name, pdus=pdus, - ts=int(self._clock.time_msec()), + origin_server_ts=int(self._clock.time_msec()), destination=None, ) @@ -489,7 +492,6 @@ class _TransactionQueue(object): """ def __init__(self, hs, transaction_actions, transport_layer): - self.server_name = hs.hostname self.transaction_actions = transaction_actions self.transport_layer = transport_layer @@ -587,8 +589,8 @@ class _TransactionQueue(object): logger.debug("TX [%s] Persisting transaction...", destination) transaction = Transaction.create_new( - ts=self._clock.time_msec(), - transaction_id=self._next_txn_id, + origin_server_ts=self._clock.time_msec(), + transaction_id=str(self._next_txn_id), origin=self.server_name, destination=destination, pdus=pdus, @@ -606,18 +608,17 @@ class _TransactionQueue(object): # FIXME (erikj): This is a bit of a hack to make the Pdu age # keys work - def cb(transaction): + def json_data_cb(): + data = transaction.get_dict() now = int(self._clock.time_msec()) - if "pdus" in transaction: - for p in transaction["pdus"]: + if "pdus" in data: + for p in data["pdus"]: if "age_ts" in p: p["age"] = now - int(p["age_ts"]) - - return transaction + return data code, response = yield self.transport_layer.send_transaction( - transaction, - on_send_callback=cb, + transaction, json_data_cb ) logger.debug("TX [%s] Sent transaction", destination) diff --git a/synapse/federation/transport.py b/synapse/federation/transport.py index afc777ec9e..e7517cac4d 100644 --- a/synapse/federation/transport.py +++ b/synapse/federation/transport.py @@ -24,6 +24,7 @@ over a different (albeit still reliable) protocol. from twisted.internet import defer from synapse.api.urls import FEDERATION_PREFIX as PREFIX +from synapse.api.errors import Codes, SynapseError from synapse.util.logutils import log_function import logging @@ -54,7 +55,7 @@ class TransportLayer(object): we receive data. """ - def __init__(self, server_name, server, client): + def __init__(self, homeserver, server_name, server, client): """ Args: server_name (str): Local home server host @@ -63,6 +64,7 @@ class TransportLayer(object): client (synapse.protocol.http.HttpClient): the http client used to send requests """ + self.keyring = homeserver.get_keyring() self.server_name = server_name self.server = server self.client = client @@ -144,7 +146,7 @@ class TransportLayer(object): @defer.inlineCallbacks @log_function - def send_transaction(self, transaction, on_send_callback=None): + def send_transaction(self, transaction, json_data_callback=None): """ Sends the given Transaction to it's destination Args: @@ -163,25 +165,15 @@ class TransportLayer(object): if transaction.destination == self.server_name: raise RuntimeError("Transport layer cannot send to itself!") - data = transaction.get_dict() - - # FIXME (erikj): This is a bit of a hack to make the Pdu age - # keys work - def cb(destination, method, path_bytes, producer): - if not on_send_callback: - return - - transaction = json.loads(producer.body) - - new_transaction = on_send_callback(transaction) - - producer.reset(new_transaction) + # FIXME: This is only used by the tests. The actual json sent is + # generated by the json_data_callback. + json_data = transaction.get_dict() code, response = yield self.client.put_json( transaction.destination, path=PREFIX + "/send/%s/" % transaction.transaction_id, - data=data, - on_send_callback=cb, + data=json_data, + json_data_callback=json_data_callback, ) logger.debug( @@ -193,17 +185,93 @@ class TransportLayer(object): @defer.inlineCallbacks @log_function - def make_query(self, destination, query_type, args): + def make_query(self, destination, query_type, args, retry_on_dns_fail): path = PREFIX + "/query/%s" % query_type response = yield self.client.get_json( destination=destination, path=path, - args=args + args=args, + retry_on_dns_fail=retry_on_dns_fail, ) defer.returnValue(response) + @defer.inlineCallbacks + def _authenticate_request(self, request): + json_request = { + "method": request.method, + "uri": request.uri, + "destination": self.server_name, + "signatures": {}, + } + + content = None + origin = None + + if request.method == "PUT": + #TODO: Handle other method types? other content types? + try: + content_bytes = request.content.read() + content = json.loads(content_bytes) + json_request["content"] = content + except: + raise SynapseError(400, "Unable to parse JSON", Codes.BAD_JSON) + + def parse_auth_header(header_str): + try: + params = auth.split(" ")[1].split(",") + param_dict = dict(kv.split("=") for kv in params) + def strip_quotes(value): + if value.startswith("\""): + return value[1:-1] + else: + return value + origin = strip_quotes(param_dict["origin"]) + key = strip_quotes(param_dict["key"]) + sig = strip_quotes(param_dict["sig"]) + return (origin, key, sig) + except: + raise SynapseError( + 400, "Malformed Authorization header", Codes.UNAUTHORIZED + ) + + auth_headers = request.requestHeaders.getRawHeaders(b"Authorization") + + if not auth_headers: + raise SynapseError( + 401, "Missing Authorization headers", Codes.UNAUTHORIZED, + ) + + for auth in auth_headers: + if auth.startswith("X-Matrix"): + (origin, key, sig) = parse_auth_header(auth) + json_request["origin"] = origin + json_request["signatures"].setdefault(origin,{})[key] = sig + + if not json_request["signatures"]: + raise SynapseError( + 401, "Missing Authorization headers", Codes.UNAUTHORIZED, + ) + + yield self.keyring.verify_json_for_server(origin, json_request) + + defer.returnValue((origin, content)) + + def _with_authentication(self, handler): + @defer.inlineCallbacks + def new_handler(request, *args, **kwargs): + try: + (origin, content) = yield self._authenticate_request(request) + response = yield handler( + origin, content, request.args, *args, **kwargs + ) + except: + logger.exception("_authenticate_request failed") + raise + defer.returnValue(response) + return new_handler + @log_function def register_received_handler(self, handler): """ Register a handler that will be fired when we receive data. @@ -217,7 +285,7 @@ class TransportLayer(object): self.server.register_path( "PUT", re.compile("^" + PREFIX + "/send/([^/]*)/$"), - self._on_send_request + self._with_authentication(self._on_send_request) ) @log_function @@ -235,9 +303,9 @@ class TransportLayer(object): self.server.register_path( "GET", re.compile("^" + PREFIX + "/pull/$"), - lambda request: handler.on_pull_request( - request.args["origin"][0], - request.args["v"] + self._with_authentication( + lambda origin, content, query: + handler.on_pull_request(query["origin"][0], query["v"]) ) ) @@ -246,8 +314,9 @@ class TransportLayer(object): self.server.register_path( "GET", re.compile("^" + PREFIX + "/pdu/([^/]*)/([^/]*)/$"), - lambda request, pdu_origin, pdu_id: handler.on_pdu_request( - pdu_origin, pdu_id + self._with_authentication( + lambda origin, content, query, pdu_origin, pdu_id: + handler.on_pdu_request(pdu_origin, pdu_id) ) ) @@ -255,38 +324,47 @@ class TransportLayer(object): self.server.register_path( "GET", re.compile("^" + PREFIX + "/state/([^/]*)/$"), - lambda request, context: handler.on_context_state_request( - context + self._with_authentication( + lambda origin, content, query, context: + handler.on_context_state_request(context) ) ) self.server.register_path( "GET", re.compile("^" + PREFIX + "/backfill/([^/]*)/$"), - lambda request, context: self._on_backfill_request( - context, request.args["v"], - request.args["limit"] + self._with_authentication( + lambda origin, content, query, context: + self._on_backfill_request( + context, query["v"], query["limit"] + ) ) ) self.server.register_path( "GET", re.compile("^" + PREFIX + "/context/([^/]*)/$"), - lambda request, context: handler.on_context_pdus_request(context) + self._with_authentication( + lambda origin, content, query, context: + handler.on_context_pdus_request(context) + ) ) # This is when we receive a server-server Query self.server.register_path( "GET", re.compile("^" + PREFIX + "/query/([^/]*)$"), - lambda request, query_type: handler.on_query_request( - query_type, {k: v[0] for k, v in request.args.items()} + self._with_authentication( + lambda origin, content, query, query_type: + handler.on_query_request( + query_type, {k: v[0] for k, v in query.items()} + ) ) ) @defer.inlineCallbacks @log_function - def _on_send_request(self, request, transaction_id): + def _on_send_request(self, origin, content, query, transaction_id): """ Called on PUT /send/<transaction_id>/ Args: @@ -301,12 +379,7 @@ class TransportLayer(object): """ # Parse the request try: - data = request.content.read() - - l = data[:20].encode("string_escape") - logger.debug("Got data: \"%s\"", l) - - transaction_data = json.loads(data) + transaction_data = content logger.debug( "Decoded %s: %s", @@ -328,9 +401,13 @@ class TransportLayer(object): defer.returnValue((400, {"error": "Invalid transaction"})) return - code, response = yield self.received_handler.on_incoming_transaction( - transaction_data - ) + try: + code, response = yield self.received_handler.on_incoming_transaction( + transaction_data + ) + except: + logger.exception("on_incoming_transaction failed") + raise defer.returnValue((code, response)) diff --git a/synapse/federation/units.py b/synapse/federation/units.py index 622fe66a8f..b2fb964180 100644 --- a/synapse/federation/units.py +++ b/synapse/federation/units.py @@ -40,7 +40,7 @@ class Pdu(JsonEncodedObject): { "pdu_id": "78c", - "ts": 1404835423000, + "origin_server_ts": 1404835423000, "origin": "bar", "prev_ids": [ ["23b", "foo"], @@ -55,7 +55,7 @@ class Pdu(JsonEncodedObject): "pdu_id", "context", "origin", - "ts", + "origin_server_ts", "pdu_type", "destinations", "transaction_id", @@ -82,7 +82,7 @@ class Pdu(JsonEncodedObject): "pdu_id", "context", "origin", - "ts", + "origin_server_ts", "pdu_type", "content", ] @@ -118,6 +118,7 @@ class Pdu(JsonEncodedObject): """ if pdu_tuple: d = copy.copy(pdu_tuple.pdu_entry._asdict()) + d["origin_server_ts"] = d.pop("ts") d["content"] = json.loads(d["content_json"]) del d["content_json"] @@ -156,11 +157,15 @@ class Edu(JsonEncodedObject): ] required_keys = [ - "origin", - "destination", "edu_type", ] +# TODO: SYN-103: Remove "origin" and "destination" keys. +# internal_keys = [ +# "origin", +# "destination", +# ] + class Transaction(JsonEncodedObject): """ A transaction is a list of Pdus and Edus to be sent to a remote home @@ -182,10 +187,12 @@ class Transaction(JsonEncodedObject): "transaction_id", "origin", "destination", - "ts", + "origin_server_ts", "previous_ids", "pdus", "edus", + "transaction_id", + "destination", ] internal_keys = [ @@ -197,7 +204,7 @@ class Transaction(JsonEncodedObject): "transaction_id", "origin", "destination", - "ts", + "origin_server_ts", "pdus", ] @@ -219,10 +226,10 @@ class Transaction(JsonEncodedObject): @staticmethod def create_new(pdus, **kwargs): """ Used to create a new transaction. Will auto fill out - transaction_id and ts keys. + transaction_id and origin_server_ts keys. """ - if "ts" not in kwargs: - raise KeyError("Require 'ts' to construct a Transaction") + if "origin_server_ts" not in kwargs: + raise KeyError("Require 'origin_server_ts' to construct a Transaction") if "transaction_id" not in kwargs: raise KeyError( "Require 'transaction_id' to construct a Transaction" diff --git a/synapse/handlers/__init__.py b/synapse/handlers/__init__.py index 5308e2c8e1..d5df3c630b 100644 --- a/synapse/handlers/__init__.py +++ b/synapse/handlers/__init__.py @@ -25,6 +25,7 @@ from .profile import ProfileHandler from .presence import PresenceHandler from .directory import DirectoryHandler from .typing import TypingNotificationHandler +from .admin import AdminHandler class Handlers(object): @@ -49,3 +50,4 @@ class Handlers(object): self.login_handler = LoginHandler(hs) self.directory_handler = DirectoryHandler(hs) self.typing_notification_handler = TypingNotificationHandler(hs) + self.admin_handler = AdminHandler(hs) diff --git a/synapse/handlers/admin.py b/synapse/handlers/admin.py new file mode 100644 index 0000000000..687b343a1d --- /dev/null +++ b/synapse/handlers/admin.py @@ -0,0 +1,62 @@ +# -*- coding: utf-8 -*- +# Copyright 2014 OpenMarket Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from twisted.internet import defer + +from ._base import BaseHandler + +import logging + + +logger = logging.getLogger(__name__) + + +class AdminHandler(BaseHandler): + + def __init__(self, hs): + super(AdminHandler, self).__init__(hs) + + @defer.inlineCallbacks + def get_whois(self, user): + res = yield self.store.get_user_ip_and_agents(user) + + d = {} + for r in res: + device = d.setdefault(r["device_id"], {}) + session = device.setdefault(r["access_token"], []) + session.append({ + "ip": r["ip"], + "user_agent": r["user_agent"], + "last_seen": r["last_seen"], + }) + + ret = { + "user_id": user.to_string(), + "devices": [ + { + "device_id": k, + "sessions": [ + { + # "access_token": x, TODO (erikj) + "connections": y, + } + for x, y in v.items() + ] + } + for k, v in d.items() + ], + } + + defer.returnValue(ret) diff --git a/synapse/handlers/directory.py b/synapse/handlers/directory.py index 4ab00a761a..a56830d520 100644 --- a/synapse/handlers/directory.py +++ b/synapse/handlers/directory.py @@ -18,7 +18,6 @@ from twisted.internet import defer from ._base import BaseHandler from synapse.api.errors import SynapseError -from synapse.http.client import HttpClient from synapse.api.events.room import RoomAliasesEvent import logging @@ -57,7 +56,6 @@ class DirectoryHandler(BaseHandler): if not servers: raise SynapseError(400, "Failed to get server list") - try: yield self.store.create_room_alias_association( room_alias, @@ -68,25 +66,19 @@ class DirectoryHandler(BaseHandler): defer.returnValue("Already exists") # TODO: Send the room event. + yield self._update_room_alias_events(user_id, room_id) - aliases = yield self.store.get_aliases_for_room(room_id) - - event = self.event_factory.create_event( - etype=RoomAliasesEvent.TYPE, - state_key=self.hs.hostname, - room_id=room_id, - user_id=user_id, - content={"aliases": aliases}, - ) + @defer.inlineCallbacks + def delete_association(self, user_id, room_alias): + # TODO Check if server admin - snapshot = yield self.store.snapshot_room( - room_id=room_id, - user_id=user_id, - ) + if not room_alias.is_mine: + raise SynapseError(400, "Room alias must be local") - yield self.state_handler.handle_new_event(event, snapshot) - yield self._on_new_room_event(event, snapshot, extra_users=[user_id]) + room_id = yield self.store.delete_room_alias(room_alias) + if room_id: + yield self._update_room_alias_events(user_id, room_id) @defer.inlineCallbacks def get_association(self, room_alias): @@ -105,8 +97,8 @@ class DirectoryHandler(BaseHandler): query_type="directory", args={ "room_alias": room_alias.to_string(), - HttpClient.RETRY_DNS_LOOKUP_FAILURES: False - } + }, + retry_on_dns_fail=False, ) if result and "room_id" in result and "servers" in result: @@ -142,3 +134,23 @@ class DirectoryHandler(BaseHandler): "room_id": result.room_id, "servers": result.servers, }) + + @defer.inlineCallbacks + def _update_room_alias_events(self, user_id, room_id): + aliases = yield self.store.get_aliases_for_room(room_id) + + event = self.event_factory.create_event( + etype=RoomAliasesEvent.TYPE, + state_key=self.hs.hostname, + room_id=room_id, + user_id=user_id, + content={"aliases": aliases}, + ) + + snapshot = yield self.store.snapshot_room( + room_id=room_id, + user_id=user_id, + ) + + yield self.state_handler.handle_new_event(event, snapshot) + yield self._on_new_room_event(event, snapshot, extra_users=[user_id]) diff --git a/synapse/handlers/login.py b/synapse/handlers/login.py index 80ffdd2726..3f152e18f0 100644 --- a/synapse/handlers/login.py +++ b/synapse/handlers/login.py @@ -17,7 +17,7 @@ from twisted.internet import defer from ._base import BaseHandler from synapse.api.errors import LoginError, Codes -from synapse.http.client import PlainHttpClient +from synapse.http.client import IdentityServerHttpClient from synapse.util.emailutils import EmailException import synapse.util.emailutils as emailutils @@ -97,10 +97,10 @@ class LoginHandler(BaseHandler): @defer.inlineCallbacks def _query_email(self, email): - httpCli = PlainHttpClient(self.hs) + httpCli = IdentityServerHttpClient(self.hs) data = yield httpCli.get_json( 'matrix.org:8090', # TODO FIXME This should be configurable. "/_matrix/identity/api/v1/lookup?medium=email&address=" + "%s" % urllib.quote(email) ) - defer.returnValue(data) \ No newline at end of file + defer.returnValue(data) diff --git a/synapse/handlers/message.py b/synapse/handlers/message.py index 317ef2c80c..7b2b8549ed 100644 --- a/synapse/handlers/message.py +++ b/synapse/handlers/message.py @@ -64,7 +64,7 @@ class MessageHandler(BaseHandler): defer.returnValue(None) @defer.inlineCallbacks - def send_message(self, event=None, suppress_auth=False, stamp_event=True): + def send_message(self, event=None, suppress_auth=False): """ Send a message. Args: @@ -72,7 +72,6 @@ class MessageHandler(BaseHandler): suppress_auth (bool) : True to suppress auth for this message. This is primarily so the home server can inject messages into rooms at will. - stamp_event (bool) : True to stamp event content with server keys. Raises: SynapseError if something went wrong. """ @@ -82,9 +81,6 @@ class MessageHandler(BaseHandler): user = self.hs.parse_userid(event.user_id) assert user.is_mine, "User must be our own: %s" % (user,) - if stamp_event: - event.content["hsob_ts"] = int(self.clock.time_msec()) - snapshot = yield self.store.snapshot_room(event.room_id, event.user_id) if not suppress_auth: @@ -132,7 +128,7 @@ class MessageHandler(BaseHandler): defer.returnValue(chunk) @defer.inlineCallbacks - def store_room_data(self, event=None, stamp_event=True): + def store_room_data(self, event=None): """ Stores data for a room. Args: @@ -151,9 +147,6 @@ class MessageHandler(BaseHandler): yield self.auth.check(event, snapshot, raises=True) - if stamp_event: - event.content["hsob_ts"] = int(self.clock.time_msec()) - yield self.state_handler.handle_new_event(event, snapshot) yield self._on_new_room_event(event, snapshot) @@ -221,10 +214,7 @@ class MessageHandler(BaseHandler): defer.returnValue(None) @defer.inlineCallbacks - def send_feedback(self, event, stamp_event=True): - if stamp_event: - event.content["hsob_ts"] = int(self.clock.time_msec()) - + def send_feedback(self, event): snapshot = yield self.store.snapshot_room(event.room_id, event.user_id) yield self.auth.check(event, snapshot, raises=True) diff --git a/synapse/handlers/register.py b/synapse/handlers/register.py index a019d770d4..df562aa762 100644 --- a/synapse/handlers/register.py +++ b/synapse/handlers/register.py @@ -22,7 +22,8 @@ from synapse.api.errors import ( ) from ._base import BaseHandler import synapse.util.stringutils as stringutils -from synapse.http.client import PlainHttpClient +from synapse.http.client import IdentityServerHttpClient +from synapse.http.client import CaptchaServerHttpClient import base64 import bcrypt @@ -154,7 +155,9 @@ class RegistrationHandler(BaseHandler): @defer.inlineCallbacks def _threepid_from_creds(self, creds): - httpCli = PlainHttpClient(self.hs) + # TODO: get this from the homeserver rather than creating a new one for + # each request + httpCli = IdentityServerHttpClient(self.hs) # XXX: make this configurable! trustedIdServers = ['matrix.org:8090'] if not creds['idServer'] in trustedIdServers: @@ -173,7 +176,7 @@ class RegistrationHandler(BaseHandler): @defer.inlineCallbacks def _bind_threepid(self, creds, mxid): - httpCli = PlainHttpClient(self.hs) + httpCli = IdentityServerHttpClient(self.hs) data = yield httpCli.post_urlencoded_get_json( creds['idServer'], "/_matrix/identity/api/v1/3pid/bind", @@ -203,7 +206,9 @@ class RegistrationHandler(BaseHandler): @defer.inlineCallbacks def _submit_captcha(self, ip_addr, private_key, challenge, response): - client = PlainHttpClient(self.hs) + # TODO: get this from the homeserver rather than creating a new one for + # each request + client = CaptchaServerHttpClient(self.hs) data = yield client.post_urlencoded_get_raw( "www.google.com:80", "/recaptcha/api/verify", diff --git a/synapse/http/client.py b/synapse/http/client.py index eb11bfd4d5..316ca1ccb9 100644 --- a/synapse/http/client.py +++ b/synapse/http/client.py @@ -26,65 +26,18 @@ from syutil.jsonutil import encode_canonical_json from synapse.api.errors import CodeMessageException, SynapseError +from syutil.crypto.jsonsign import sign_json + from StringIO import StringIO import json import logging import urllib +import urlparse logger = logging.getLogger(__name__) -# FIXME: SURELY these should be killed?! -_destination_mappings = { - "red": "localhost:8080", - "blue": "localhost:8081", - "green": "localhost:8082", -} - - -class HttpClient(object): - """ Interface for talking json over http - """ - RETRY_DNS_LOOKUP_FAILURES = "__retry_dns" - - def put_json(self, destination, path, data): - """ Sends the specifed json data using PUT - - Args: - destination (str): The remote server to send the HTTP request - to. - path (str): The HTTP path. - data (dict): A dict containing the data that will be used as - the request body. This will be encoded as JSON. - - Returns: - Deferred: Succeeds when we get a 2xx HTTP response. The result - will be the decoded JSON body. On a 4xx or 5xx error response a - CodeMessageException is raised. - """ - pass - - def get_json(self, destination, path, args=None): - """ Get's some json from the given host homeserver and path - - Args: - destination (str): The remote server to send the HTTP request - to. - path (str): The HTTP path. - args (dict): A dictionary used to create query strings, defaults to - None. - **Note**: The value of each key is assumed to be an iterable - and *not* a string. - - Returns: - Deferred: Succeeds when we get *any* HTTP response. - - The result of the deferred is a tuple of `(code, response)`, - where `response` is a dict representing the decoded JSON body. - """ - pass - class MatrixHttpAgent(_AgentBase): @@ -109,12 +62,8 @@ class MatrixHttpAgent(_AgentBase): parsed_URI.originForm) -class TwistedHttpClient(HttpClient): - """ Wrapper around the twisted HTTP client api. - - Attributes: - agent (twisted.web.client.Agent): The twisted Agent used to send the - requests. +class BaseHttpClient(object): + """Base class for HTTP clients using twisted. """ def __init__(self, hs): @@ -122,111 +71,20 @@ class TwistedHttpClient(HttpClient): self.hs = hs @defer.inlineCallbacks - def put_json(self, destination, path, data, on_send_callback=None): - if destination in _destination_mappings: - destination = _destination_mappings[destination] - - response = yield self._create_request( - destination.encode("ascii"), - "PUT", - path.encode("ascii"), - producer=_JsonProducer(data), - headers_dict={"Content-Type": ["application/json"]}, - on_send_callback=on_send_callback, - ) - - logger.debug("Getting resp body") - body = yield readBody(response) - logger.debug("Got resp body") - - defer.returnValue((response.code, body)) - - @defer.inlineCallbacks - def get_json(self, destination, path, args={}): - if destination in _destination_mappings: - destination = _destination_mappings[destination] - - logger.debug("get_json args: %s", args) - - retry_on_dns_fail = True - if HttpClient.RETRY_DNS_LOOKUP_FAILURES in args: - # FIXME: This isn't ideal, but the interface exposed in get_json - # isn't comprehensive enough to give caller's any control over - # their connection mechanics. - retry_on_dns_fail = args.pop(HttpClient.RETRY_DNS_LOOKUP_FAILURES) - - query_bytes = urllib.urlencode(args, True) - logger.debug("Query bytes: %s Retry DNS: %s", args, retry_on_dns_fail) - - response = yield self._create_request( - destination.encode("ascii"), - "GET", - path.encode("ascii"), - query_bytes=query_bytes, - retry_on_dns_fail=retry_on_dns_fail - ) - - body = yield readBody(response) - - defer.returnValue(json.loads(body)) - - @defer.inlineCallbacks - def post_urlencoded_get_json(self, destination, path, args={}): - if destination in _destination_mappings: - destination = _destination_mappings[destination] - - logger.debug("post_urlencoded_get_json args: %s", args) - query_bytes = urllib.urlencode(args, True) - - response = yield self._create_request( - destination.encode("ascii"), - "POST", - path.encode("ascii"), - producer=FileBodyProducer(StringIO(urllib.urlencode(args))), - headers_dict={"Content-Type": ["application/x-www-form-urlencoded"]} - ) - - body = yield readBody(response) - - defer.returnValue(json.loads(body)) - - # XXX FIXME : I'm so sorry. - @defer.inlineCallbacks - def post_urlencoded_get_raw(self, destination, path, accept_partial=False, args={}): - if destination in _destination_mappings: - destination = _destination_mappings[destination] - - query_bytes = urllib.urlencode(args, True) - - response = yield self._create_request( - destination.encode("ascii"), - "POST", - path.encode("ascii"), - producer=FileBodyProducer(StringIO(urllib.urlencode(args))), - headers_dict={"Content-Type": ["application/x-www-form-urlencoded"]} - ) - - try: - body = yield readBody(response) - defer.returnValue(body) - except PartialDownloadError as e: - if accept_partial: - defer.returnValue(e.response) - else: - raise e - - - @defer.inlineCallbacks - def _create_request(self, destination, method, path_bytes, param_bytes=b"", - query_bytes=b"", producer=None, headers_dict={}, - retry_on_dns_fail=True, on_send_callback=None): + def _create_request(self, destination, method, path_bytes, + body_callback, headers_dict={}, param_bytes=b"", + query_bytes=b"", retry_on_dns_fail=True): """ Creates and sends a request to the given url """ headers_dict[b"User-Agent"] = [b"Synapse"] headers_dict[b"Host"] = [destination] - logger.debug("Sending request to %s: %s %s;%s?%s", - destination, method, path_bytes, param_bytes, query_bytes) + url_bytes = urlparse.urlunparse( + ("", "", path_bytes, param_bytes, query_bytes, "",) + ) + + logger.debug("Sending request to %s: %s %s", + destination, method, url_bytes) logger.debug( "Types: %s", @@ -239,12 +97,11 @@ class TwistedHttpClient(HttpClient): retries_left = 5 - # TODO: setup and pass in an ssl_context to enable TLS endpoint = self._getEndpoint(reactor, destination); while True: - if on_send_callback: - on_send_callback(destination, method, path_bytes, producer) + + producer = body_callback(method, url_bytes, headers_dict) try: response = yield self.agent.request( @@ -290,6 +147,134 @@ class TwistedHttpClient(HttpClient): defer.returnValue(response) + +class MatrixHttpClient(BaseHttpClient): + """ Wrapper around the twisted HTTP client api. Implements + + Attributes: + agent (twisted.web.client.Agent): The twisted Agent used to send the + requests. + """ + + RETRY_DNS_LOOKUP_FAILURES = "__retry_dns" + + def __init__(self, hs): + self.signing_key = hs.config.signing_key[0] + self.server_name = hs.hostname + BaseHttpClient.__init__(self, hs) + + def sign_request(self, destination, method, url_bytes, headers_dict, + content=None): + request = { + "method": method, + "uri": url_bytes, + "origin": self.server_name, + "destination": destination, + } + + if content is not None: + request["content"] = content + + request = sign_json(request, self.server_name, self.signing_key) + + auth_headers = [] + + for key,sig in request["signatures"][self.server_name].items(): + auth_headers.append(bytes( + "X-Matrix origin=%s,key=\"%s\",sig=\"%s\"" % ( + self.server_name, key, sig, + ) + )) + + headers_dict[b"Authorization"] = auth_headers + + @defer.inlineCallbacks + def put_json(self, destination, path, data={}, json_data_callback=None): + """ Sends the specifed json data using PUT + + Args: + destination (str): The remote server to send the HTTP request + to. + path (str): The HTTP path. + data (dict): A dict containing the data that will be used as + the request body. This will be encoded as JSON. + json_data_callback (callable): A callable returning the dict to + use as the request body. + + Returns: + Deferred: Succeeds when we get a 2xx HTTP response. The result + will be the decoded JSON body. On a 4xx or 5xx error response a + CodeMessageException is raised. + """ + + if not json_data_callback: + def json_data_callback(): + return data + + def body_callback(method, url_bytes, headers_dict): + json_data = json_data_callback() + self.sign_request( + destination, method, url_bytes, headers_dict, json_data + ) + producer = _JsonProducer(json_data) + return producer + + response = yield self._create_request( + destination.encode("ascii"), + "PUT", + path.encode("ascii"), + body_callback=body_callback, + headers_dict={"Content-Type": ["application/json"]}, + ) + + logger.debug("Getting resp body") + body = yield readBody(response) + logger.debug("Got resp body") + + defer.returnValue((response.code, body)) + + @defer.inlineCallbacks + def get_json(self, destination, path, args={}, retry_on_dns_fail=True): + """ Get's some json from the given host homeserver and path + + Args: + destination (str): The remote server to send the HTTP request + to. + path (str): The HTTP path. + args (dict): A dictionary used to create query strings, defaults to + None. + **Note**: The value of each key is assumed to be an iterable + and *not* a string. + + Returns: + Deferred: Succeeds when we get *any* HTTP response. + + The result of the deferred is a tuple of `(code, response)`, + where `response` is a dict representing the decoded JSON body. + """ + logger.debug("get_json args: %s", args) + + query_bytes = urllib.urlencode(args, True) + logger.debug("Query bytes: %s Retry DNS: %s", args, retry_on_dns_fail) + + def body_callback(method, url_bytes, headers_dict): + self.sign_request(destination, method, url_bytes, headers_dict) + return None + + response = yield self._create_request( + destination.encode("ascii"), + "GET", + path.encode("ascii"), + query_bytes=query_bytes, + body_callback=body_callback, + retry_on_dns_fail=retry_on_dns_fail + ) + + body = yield readBody(response) + + defer.returnValue(json.loads(body)) + + def _getEndpoint(self, reactor, destination): return matrix_endpoint( reactor, destination, timeout=10, @@ -297,10 +282,69 @@ class TwistedHttpClient(HttpClient): ) -class PlainHttpClient(TwistedHttpClient): +class IdentityServerHttpClient(BaseHttpClient): + """Separate HTTP client for talking to the Identity servers since they + don't use SRV records and talk x-www-form-urlencoded rather than JSON. + """ + def _getEndpoint(self, reactor, destination): + #TODO: This should be talking TLS + return matrix_endpoint(reactor, destination, timeout=10) + + @defer.inlineCallbacks + def post_urlencoded_get_json(self, destination, path, args={}): + logger.debug("post_urlencoded_get_json args: %s", args) + query_bytes = urllib.urlencode(args, True) + + def body_callback(method, url_bytes, headers_dict): + return FileBodyProducer(StringIO(query_bytes)) + + response = yield self._create_request( + destination.encode("ascii"), + "POST", + path.encode("ascii"), + body_callback=body_callback, + headers_dict={ + "Content-Type": ["application/x-www-form-urlencoded"] + } + ) + + body = yield readBody(response) + + defer.returnValue(json.loads(body)) + + +class CaptchaServerHttpClient(MatrixHttpClient): + """Separate HTTP client for talking to google's captcha servers""" + def _getEndpoint(self, reactor, destination): return matrix_endpoint(reactor, destination, timeout=10) - + + @defer.inlineCallbacks + def post_urlencoded_get_raw(self, destination, path, accept_partial=False, + args={}): + query_bytes = urllib.urlencode(args, True) + + def body_callback(method, url_bytes, headers_dict): + return FileBodyProducer(StringIO(query_bytes)) + + response = yield self._create_request( + destination.encode("ascii"), + "POST", + path.encode("ascii"), + body_callback=body_callback, + headers_dict={ + "Content-Type": ["application/x-www-form-urlencoded"] + } + ) + + try: + body = yield readBody(response) + defer.returnValue(body) + except PartialDownloadError as e: + if accept_partial: + defer.returnValue(e.response) + else: + raise e def _print_ex(e): if hasattr(e, "reasons") and e.reasons: diff --git a/synapse/http/server_key_resource.py b/synapse/http/server_key_resource.py new file mode 100644 index 0000000000..b30ecead27 --- /dev/null +++ b/synapse/http/server_key_resource.py @@ -0,0 +1,89 @@ +# -*- coding: utf-8 -*- +# Copyright 2014 OpenMarket Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +from twisted.web.resource import Resource +from synapse.http.server import respond_with_json_bytes +from syutil.crypto.jsonsign import sign_json +from syutil.base64util import encode_base64 +from syutil.jsonutil import encode_canonical_json +from OpenSSL import crypto +import logging + + +logger = logging.getLogger(__name__) + + +class LocalKey(Resource): + """HTTP resource containing encoding the TLS X.509 certificate and NACL + signature verification keys for this server:: + + GET /key HTTP/1.1 + + HTTP/1.1 200 OK + Content-Type: application/json + { + "server_name": "this.server.example.com" + "verify_keys": { + "algorithm:version": # base64 encoded NACL verification key. + }, + "tls_certificate": # base64 ASN.1 DER encoded X.509 tls cert. + "signatures": { + "this.server.example.com": { + "algorithm:version": # NACL signature for this server. + } + } + } + """ + + def __init__(self, hs): + self.hs = hs + self.response_body = encode_canonical_json( + self.response_json_object(hs.config) + ) + Resource.__init__(self) + + @staticmethod + def response_json_object(server_config): + verify_keys = {} + for key in server_config.signing_key: + verify_key_bytes = key.verify_key.encode() + key_id = "%s:%s" % (key.alg, key.version) + verify_keys[key_id] = encode_base64(verify_key_bytes) + + x509_certificate_bytes = crypto.dump_certificate( + crypto.FILETYPE_ASN1, + server_config.tls_certificate + ) + json_object = { + u"server_name": server_config.server_name, + u"verify_keys": verify_keys, + u"tls_certificate": encode_base64(x509_certificate_bytes) + } + for key in server_config.signing_key: + json_object = sign_json( + json_object, + server_config.server_name, + key, + ) + + return json_object + + def render_GET(self, request): + return respond_with_json_bytes(request, 200, self.response_body) + + def getChild(self, name, request): + if name == '': + return self diff --git a/synapse/rest/__init__.py b/synapse/rest/__init__.py index 3b9aa59733..e391e5678d 100644 --- a/synapse/rest/__init__.py +++ b/synapse/rest/__init__.py @@ -15,7 +15,8 @@ from . import ( - room, events, register, login, profile, presence, initial_sync, directory, voip + room, events, register, login, profile, presence, initial_sync, directory, + voip, admin, ) @@ -43,3 +44,4 @@ class RestServletFactory(object): initial_sync.register_servlets(hs, client_resource) directory.register_servlets(hs, client_resource) voip.register_servlets(hs, client_resource) + admin.register_servlets(hs, client_resource) diff --git a/synapse/rest/admin.py b/synapse/rest/admin.py new file mode 100644 index 0000000000..ed9b484623 --- /dev/null +++ b/synapse/rest/admin.py @@ -0,0 +1,47 @@ +# -*- coding: utf-8 -*- +# Copyright 2014 OpenMarket Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from twisted.internet import defer + +from synapse.api.errors import AuthError, SynapseError +from base import RestServlet, client_path_pattern + +import logging + +logger = logging.getLogger(__name__) + + +class WhoisRestServlet(RestServlet): + PATTERN = client_path_pattern("/admin/whois/(?P<user_id>[^/]*)") + + @defer.inlineCallbacks + def on_GET(self, request, user_id): + target_user = self.hs.parse_userid(user_id) + auth_user = yield self.auth.get_user_by_req(request) + is_admin = yield self.auth.is_server_admin(auth_user) + + if not is_admin and target_user != auth_user: + raise AuthError(403, "You are not a server admin") + + if not target_user.is_mine: + raise SynapseError(400, "Can only whois a local user") + + ret = yield self.handlers.admin_handler.get_whois(target_user) + + defer.returnValue((200, ret)) + + +def register_servlets(hs, http_server): + WhoisRestServlet(hs).register(http_server) diff --git a/synapse/rest/directory.py b/synapse/rest/directory.py index 31849246a1..6c260e7102 100644 --- a/synapse/rest/directory.py +++ b/synapse/rest/directory.py @@ -16,7 +16,7 @@ from twisted.internet import defer -from synapse.api.errors import SynapseError, Codes +from synapse.api.errors import AuthError, SynapseError, Codes from base import RestServlet, client_path_pattern import json @@ -81,6 +81,24 @@ class ClientDirectoryServer(RestServlet): defer.returnValue((200, {})) + @defer.inlineCallbacks + def on_DELETE(self, request, room_alias): + user = yield self.auth.get_user_by_req(request) + + is_admin = yield self.auth.is_server_admin(user) + if not is_admin: + raise AuthError(403, "You need to be a server admin") + + dir_handler = self.handlers.directory_handler + + room_alias = self.hs.parse_roomalias(urllib.unquote(room_alias)) + + yield dir_handler.delete_association( + user.to_string(), room_alias + ) + + defer.returnValue((200, {})) + def _parse_json(request): try: diff --git a/synapse/rest/presence.py b/synapse/rest/presence.py index 7fc8ce4404..138cc88a05 100644 --- a/synapse/rest/presence.py +++ b/synapse/rest/presence.py @@ -68,7 +68,7 @@ class PresenceStatusRestServlet(RestServlet): yield self.handlers.presence_handler.set_state( target_user=user, auth_user=auth_user, state=state) - defer.returnValue((200, "")) + defer.returnValue((200, {})) def on_OPTIONS(self, request): return (200, {}) @@ -141,7 +141,7 @@ class PresenceListRestServlet(RestServlet): yield defer.DeferredList(deferreds) - defer.returnValue((200, "")) + defer.returnValue((200, {})) def on_OPTIONS(self, request): return (200, {}) diff --git a/synapse/rest/register.py b/synapse/rest/register.py index 4935e323d9..804117ee09 100644 --- a/synapse/rest/register.py +++ b/synapse/rest/register.py @@ -195,13 +195,7 @@ class RegisterRestServlet(RestServlet): raise SynapseError(400, "Captcha response is required", errcode=Codes.CAPTCHA_NEEDED) - # May be an X-Forwarding-For header depending on config - ip_addr = request.getClientIP() - if self.hs.config.captcha_ip_origin_is_x_forwarded: - # use the header - if request.requestHeaders.hasHeader("X-Forwarded-For"): - ip_addr = request.requestHeaders.getRawHeaders( - "X-Forwarded-For")[0] + ip_addr = self.hs.get_ip_from_request(request) handler = self.handlers.registration_handler yield handler.check_recaptcha( diff --git a/synapse/server.py b/synapse/server.py index cdea49e6ab..a4d2d4aba5 100644 --- a/synapse/server.py +++ b/synapse/server.py @@ -34,6 +34,7 @@ from synapse.util.distributor import Distributor from synapse.util.lockutils import LockManager from synapse.streams.events import EventSources from synapse.api.ratelimiting import Ratelimiter +from synapse.crypto.keyring import Keyring class BaseHomeServer(object): @@ -75,8 +76,10 @@ class BaseHomeServer(object): 'resource_for_federation', 'resource_for_web_client', 'resource_for_content_repo', + 'resource_for_server_key', 'event_sources', 'ratelimiter', + 'keyring', ] def __init__(self, hostname, **kwargs): @@ -143,6 +146,18 @@ class BaseHomeServer(object): def serialize_event(self, e): return serialize_event(self, e) + def get_ip_from_request(self, request): + # May be an X-Forwarding-For header depending on config + ip_addr = request.getClientIP() + if self.config.captcha_ip_origin_is_x_forwarded: + # use the header + if request.requestHeaders.hasHeader("X-Forwarded-For"): + ip_addr = request.requestHeaders.getRawHeaders( + "X-Forwarded-For" + )[0] + + return ip_addr + # Build magic accessors for every dependency for depname in BaseHomeServer.DEPENDENCIES: BaseHomeServer._make_dependency_method(depname) @@ -200,6 +215,9 @@ class HomeServer(BaseHomeServer): def build_ratelimiter(self): return Ratelimiter() + def build_keyring(self): + return Keyring(self) + def register_servlets(self): """ Register all servlets associated with this HomeServer. """ diff --git a/synapse/storage/__init__.py b/synapse/storage/__init__.py index 15919eb580..3aa6345a7f 100644 --- a/synapse/storage/__init__.py +++ b/synapse/storage/__init__.py @@ -57,13 +57,14 @@ SCHEMAS = [ "presence", "im", "room_aliases", + "keys", "redactions", ] # Remember to update this number every time an incompatible change is made to # database schema files, so the users will be informed on server restarts. -SCHEMA_VERSION = 4 +SCHEMA_VERSION = 6 class _RollbackButIsFineException(Exception): @@ -105,7 +106,7 @@ class DataStore(RoomMemberStore, RoomStore, stream_ordering=stream_ordering, is_new_state=is_new_state, ) - except _RollbackButIsFineException as e: + except _RollbackButIsFineException: pass @defer.inlineCallbacks @@ -154,6 +155,8 @@ class DataStore(RoomMemberStore, RoomStore, cols["unrecognized_keys"] = json.dumps(unrec_keys) + cols["ts"] = cols.pop("origin_server_ts") + logger.debug("Persisting: %s", repr(cols)) if pdu.is_state: @@ -294,6 +297,28 @@ class DataStore(RoomMemberStore, RoomStore, defer.returnValue(self.min_token) + def insert_client_ip(self, user, access_token, device_id, ip, user_agent): + return self._simple_insert( + "user_ips", + { + "user": user.to_string(), + "access_token": access_token, + "device_id": device_id, + "ip": ip, + "user_agent": user_agent, + "last_seen": int(self._clock.time_msec()), + } + ) + + def get_user_ip_and_agents(self, user): + return self._simple_select_list( + table="user_ips", + keyvalues={"user": user.to_string()}, + retcols=[ + "device_id", "access_token", "ip", "user_agent", "last_seen" + ], + ) + def snapshot_room(self, room_id, user_id, state_type=None, state_key=None): """Snapshot the room for an update by a user Args: diff --git a/synapse/storage/_base.py b/synapse/storage/_base.py index 889de2bedc..65a86e9056 100644 --- a/synapse/storage/_base.py +++ b/synapse/storage/_base.py @@ -121,7 +121,7 @@ class SQLBaseStore(object): # "Simple" SQL API methods that operate on a single table with no JOINs, # no complex WHERE clauses, just a dict of values for columns. - def _simple_insert(self, table, values, or_replace=False): + def _simple_insert(self, table, values, or_replace=False, or_ignore=False): """Executes an INSERT query on the named table. Args: @@ -130,13 +130,16 @@ class SQLBaseStore(object): or_replace : bool; if True performs an INSERT OR REPLACE """ return self.runInteraction( - self._simple_insert_txn, table, values, or_replace=or_replace + self._simple_insert_txn, table, values, or_replace=or_replace, + or_ignore=or_ignore, ) @log_function - def _simple_insert_txn(self, txn, table, values, or_replace=False): + def _simple_insert_txn(self, txn, table, values, or_replace=False, + or_ignore=False): sql = "%s INTO %s (%s) VALUES(%s)" % ( - ("INSERT OR REPLACE" if or_replace else "INSERT"), + ("INSERT OR REPLACE" if or_replace else + "INSERT OR IGNORE" if or_ignore else "INSERT"), table, ", ".join(k for k in values), ", ".join("?" for k in values) @@ -351,6 +354,7 @@ class SQLBaseStore(object): d.pop("stream_ordering", None) d.pop("topological_ordering", None) d.pop("processed", None) + d["origin_server_ts"] = d.pop("ts", 0) d.update(json.loads(row_dict["unrecognized_keys"])) d["content"] = json.loads(d["content"]) @@ -358,7 +362,7 @@ class SQLBaseStore(object): if "age_ts" not in d: # For compatibility - d["age_ts"] = d["ts"] if "ts" in d else 0 + d["age_ts"] = d.get("origin_server_ts", 0) return self.event_factory.create_event( etype=d["type"], diff --git a/synapse/storage/directory.py b/synapse/storage/directory.py index 540eb4c2c4..52373a28a6 100644 --- a/synapse/storage/directory.py +++ b/synapse/storage/directory.py @@ -93,6 +93,36 @@ class DirectoryStore(SQLBaseStore): } ) + def delete_room_alias(self, room_alias): + return self.runInteraction( + self._delete_room_alias_txn, + room_alias, + ) + + def _delete_room_alias_txn(self, txn, room_alias): + cursor = txn.execute( + "SELECT room_id FROM room_aliases WHERE room_alias = ?", + (room_alias.to_string(),) + ) + + res = cursor.fetchone() + if res: + room_id = res[0] + else: + return None + + txn.execute( + "DELETE FROM room_aliases WHERE room_alias = ?", + (room_alias.to_string(),) + ) + + txn.execute( + "DELETE FROM room_alias_servers WHERE room_alias = ?", + (room_alias.to_string(),) + ) + + return room_id + def get_aliases_for_room(self, room_id): return self._simple_select_onecol( "room_aliases", diff --git a/synapse/storage/keys.py b/synapse/storage/keys.py index 5a38c3e8f2..8189e071a3 100644 --- a/synapse/storage/keys.py +++ b/synapse/storage/keys.py @@ -18,7 +18,8 @@ from _base import SQLBaseStore from twisted.internet import defer import OpenSSL -import nacl.signing +from syutil.crypto.signing_key import decode_verify_key_bytes +import hashlib class KeyStore(SQLBaseStore): """Persistence for signature verification keys and tls X.509 certificates @@ -42,62 +43,76 @@ class KeyStore(SQLBaseStore): ) defer.returnValue(tls_certificate) - def store_server_certificate(self, server_name, key_server, ts_now_ms, + def store_server_certificate(self, server_name, from_server, time_now_ms, tls_certificate): """Stores the TLS X.509 certificate for the given server Args: - server_name (bytes): The name of the server. - key_server (bytes): Where the certificate was looked up - ts_now_ms (int): The time now in milliseconds + server_name (str): The name of the server. + from_server (str): Where the certificate was looked up + time_now_ms (int): The time now in milliseconds tls_certificate (OpenSSL.crypto.X509): The X.509 certificate. """ tls_certificate_bytes = OpenSSL.crypto.dump_certificate( OpenSSL.crypto.FILETYPE_ASN1, tls_certificate ) + fingerprint = hashlib.sha256(tls_certificate_bytes).hexdigest() return self._simple_insert( table="server_tls_certificates", - keyvalues={ + values={ "server_name": server_name, - "key_server": key_server, - "ts_added_ms": ts_now_ms, - "tls_certificate": tls_certificate_bytes, + "fingerprint": fingerprint, + "from_server": from_server, + "ts_added_ms": time_now_ms, + "tls_certificate": buffer(tls_certificate_bytes), }, + or_ignore=True, ) @defer.inlineCallbacks - def get_server_verification_key(self, server_name): - """Retrieve the NACL verification key for a given server + def get_server_verify_keys(self, server_name, key_ids): + """Retrieve the NACL verification key for a given server for the given + key_ids Args: - server_name (bytes): The name of the server. + server_name (str): The name of the server. + key_ids (list of str): List of key_ids to try and look up. Returns: - (nacl.signing.VerifyKey): The verification key. + (list of VerifyKey): The verification keys. """ - verification_key_bytes, = yield self._simple_select_one( - table="server_signature_keys", - key_values={"server_name": server_name}, - retcols=("tls_certificate",), + sql = ( + "SELECT key_id, verify_key FROM server_signature_keys" + " WHERE server_name = ?" + " AND key_id in (" + ",".join("?" for key_id in key_ids) + ")" ) - verification_key = nacl.signing.VerifyKey(verification_key_bytes) - defer.returnValue(verification_key) - def store_server_verification_key(self, server_name, key_version, - key_server, ts_now_ms, verification_key): + rows = yield self._execute_and_decode(sql, server_name, *key_ids) + + keys = [] + for row in rows: + key_id = row["key_id"] + key_bytes = row["verify_key"] + key = decode_verify_key_bytes(key_id, str(key_bytes)) + keys.append(key) + defer.returnValue(keys) + + def store_server_verify_key(self, server_name, from_server, time_now_ms, + verify_key): """Stores a NACL verification key for the given server. Args: - server_name (bytes): The name of the server. - key_version (bytes): The version of the key for the server. - key_server (bytes): Where the verification key was looked up + server_name (str): The name of the server. + key_id (str): The version of the key for the server. + from_server (str): Where the verification key was looked up ts_now_ms (int): The time now in milliseconds - verification_key (nacl.signing.VerifyKey): The NACL verify key. + verification_key (VerifyKey): The NACL verify key. """ - verification_key_bytes = verification_key.encode() + verify_key_bytes = verify_key.encode() return self._simple_insert( table="server_signature_keys", - key_values={ + values={ "server_name": server_name, - "key_version": key_version, - "key_server": key_server, - "ts_added_ms": ts_now_ms, - "verification_key": verification_key_bytes, + "key_id": "%s:%s" % (verify_key.alg, verify_key.version), + "from_server": from_server, + "ts_added_ms": time_now_ms, + "verify_key": buffer(verify_key.encode()), }, + or_ignore=True, ) diff --git a/synapse/storage/registration.py b/synapse/storage/registration.py index db20b1daa0..719806f82b 100644 --- a/synapse/storage/registration.py +++ b/synapse/storage/registration.py @@ -88,27 +88,40 @@ class RegistrationStore(SQLBaseStore): query, user_id ) - @defer.inlineCallbacks def get_user_by_token(self, token): """Get a user from the given access token. Args: token (str): The access token of a user. Returns: - str: The user ID of the user. + dict: Including the name (user_id), device_id and whether they are + an admin. Raises: StoreError if no user was found. """ - user_id = yield self.runInteraction(self._query_for_auth, - token) - defer.returnValue(user_id) + return self.runInteraction( + self._query_for_auth, + token + ) + + def is_server_admin(self, user): + return self._simple_select_one_onecol( + table="users", + keyvalues={"name": user.to_string()}, + retcol="admin", + ) def _query_for_auth(self, txn, token): - txn.execute("SELECT users.name FROM access_tokens LEFT JOIN users" + - " ON users.id = access_tokens.user_id WHERE token = ?", - [token]) - row = txn.fetchone() - if row: - return row[0] + sql = ( + "SELECT users.name, users.admin, access_tokens.device_id " + "FROM users " + "INNER JOIN access_tokens on users.id = access_tokens.user_id " + "WHERE token = ?" + ) + + cursor = txn.execute(sql, (token,)) + rows = self.cursor_to_dict(cursor) + if rows: + return rows[0] raise StoreError(404, "Token not found.") diff --git a/synapse/storage/roommember.py b/synapse/storage/roommember.py index 958e730591..ceeef5880e 100644 --- a/synapse/storage/roommember.py +++ b/synapse/storage/roommember.py @@ -18,7 +18,6 @@ from twisted.internet import defer from ._base import SQLBaseStore from synapse.api.constants import Membership -from synapse.util.logutils import log_function import logging diff --git a/synapse/storage/schema/delta/v5.sql b/synapse/storage/schema/delta/v5.sql new file mode 100644 index 0000000000..af9df11aa9 --- /dev/null +++ b/synapse/storage/schema/delta/v5.sql @@ -0,0 +1,16 @@ + +CREATE TABLE IF NOT EXISTS user_ips ( + user TEXT NOT NULL, + access_token TEXT NOT NULL, + device_id TEXT, + ip TEXT NOT NULL, + user_agent TEXT NOT NULL, + last_seen INTEGER NOT NULL, + CONSTRAINT user_ip UNIQUE (user, access_token, ip, user_agent) ON CONFLICT REPLACE +); + +CREATE INDEX IF NOT EXISTS user_ips_user ON user_ips(user); + +ALTER TABLE users ADD COLUMN admin BOOL DEFAULT 0 NOT NULL; + +PRAGMA user_version = 5; diff --git a/synapse/storage/schema/delta/v6.sql b/synapse/storage/schema/delta/v6.sql new file mode 100644 index 0000000000..9bf2068d84 --- /dev/null +++ b/synapse/storage/schema/delta/v6.sql @@ -0,0 +1,31 @@ +/* Copyright 2014 OpenMarket Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +CREATE TABLE IF NOT EXISTS server_tls_certificates( + server_name TEXT, -- Server name. + fingerprint TEXT, -- Certificate fingerprint. + from_server TEXT, -- Which key server the certificate was fetched from. + ts_added_ms INTEGER, -- When the certifcate was added. + tls_certificate BLOB, -- DER encoded x509 certificate. + CONSTRAINT uniqueness UNIQUE (server_name, fingerprint) +); + +CREATE TABLE IF NOT EXISTS server_signature_keys( + server_name TEXT, -- Server name. + key_id TEXT, -- Key version. + from_server TEXT, -- Which key server the key was fetched form. + ts_added_ms INTEGER, -- When the key was added. + verify_key BLOB, -- NACL verification key. + CONSTRAINT uniqueness UNIQUE (server_name, key_id) +); diff --git a/synapse/storage/schema/keys.sql b/synapse/storage/schema/keys.sql index 706a1a03ff..9bf2068d84 100644 --- a/synapse/storage/schema/keys.sql +++ b/synapse/storage/schema/keys.sql @@ -14,17 +14,18 @@ */ CREATE TABLE IF NOT EXISTS server_tls_certificates( server_name TEXT, -- Server name. - key_server TEXT, -- Which key server the certificate was fetched from. + fingerprint TEXT, -- Certificate fingerprint. + from_server TEXT, -- Which key server the certificate was fetched from. ts_added_ms INTEGER, -- When the certifcate was added. tls_certificate BLOB, -- DER encoded x509 certificate. - CONSTRAINT uniqueness UNIQUE (server_name) + CONSTRAINT uniqueness UNIQUE (server_name, fingerprint) ); CREATE TABLE IF NOT EXISTS server_signature_keys( server_name TEXT, -- Server name. - key_version TEXT, -- Key version. - key_server TEXT, -- Which key server the key was fetched form. + key_id TEXT, -- Key version. + from_server TEXT, -- Which key server the key was fetched form. ts_added_ms INTEGER, -- When the key was added. - verification_key BLOB, -- NACL verification key. - CONSTRAINT uniqueness UNIQUE (server_name, key_version) + verify_key BLOB, -- NACL verification key. + CONSTRAINT uniqueness UNIQUE (server_name, key_id) ); diff --git a/synapse/storage/schema/users.sql b/synapse/storage/schema/users.sql index 2519702971..8244f733bd 100644 --- a/synapse/storage/schema/users.sql +++ b/synapse/storage/schema/users.sql @@ -17,6 +17,7 @@ CREATE TABLE IF NOT EXISTS users( name TEXT, password_hash TEXT, creation_ts INTEGER, + admin BOOL DEFAULT 0 NOT NULL, UNIQUE(name) ON CONFLICT ROLLBACK ); @@ -29,3 +30,16 @@ CREATE TABLE IF NOT EXISTS access_tokens( FOREIGN KEY(user_id) REFERENCES users(id), UNIQUE(token) ON CONFLICT ROLLBACK ); + +CREATE TABLE IF NOT EXISTS user_ips ( + user TEXT NOT NULL, + access_token TEXT NOT NULL, + device_id TEXT, + ip TEXT NOT NULL, + user_agent TEXT NOT NULL, + last_seen INTEGER NOT NULL, + CONSTRAINT user_ip UNIQUE (user, access_token, ip, user_agent) ON CONFLICT REPLACE +); + +CREATE INDEX IF NOT EXISTS user_ips_user ON user_ips(user); + diff --git a/synapse/storage/transactions.py b/synapse/storage/transactions.py index ab4599b468..2ba8e30efe 100644 --- a/synapse/storage/transactions.py +++ b/synapse/storage/transactions.py @@ -87,7 +87,8 @@ class TransactionStore(SQLBaseStore): txn.execute(query, (code, response_json, transaction_id, origin)) - def prep_send_transaction(self, transaction_id, destination, ts, pdu_list): + def prep_send_transaction(self, transaction_id, destination, + origin_server_ts, pdu_list): """Persists an outgoing transaction and calculates the values for the previous transaction id list. @@ -97,7 +98,7 @@ class TransactionStore(SQLBaseStore): Args: transaction_id (str) destination (str) - ts (int) + origin_server_ts (int) pdu_list (list) Returns: @@ -106,11 +107,11 @@ class TransactionStore(SQLBaseStore): return self.runInteraction( self._prep_send_transaction, - transaction_id, destination, ts, pdu_list + transaction_id, destination, origin_server_ts, pdu_list ) - def _prep_send_transaction(self, txn, transaction_id, destination, ts, - pdu_list): + def _prep_send_transaction(self, txn, transaction_id, destination, + origin_server_ts, pdu_list): # First we find out what the prev_txs should be. # Since we know that we are only sending one transaction at a time, @@ -131,7 +132,7 @@ class TransactionStore(SQLBaseStore): None, transaction_id=transaction_id, destination=destination, - ts=ts, + ts=origin_server_ts, response_code=0, response_json=None )) |