diff options
| -rw-r--r-- | CHANGES.rst | 24 | ||||
| -rw-r--r-- | UPGRADE.rst | 36 | ||||
| -rw-r--r-- | VERSION | 2 | ||||
| -rwxr-xr-x | database-prepare-for-0.5.0.sh | 21 | ||||
| -rwxr-xr-x | setup.py | 8 | ||||
| -rw-r--r-- | synapse/__init__.py | 2 | ||||
| -rwxr-xr-x | synapse/app/homeserver.py | 4 | ||||
| -rw-r--r-- | synapse/handlers/directory.py | 2 | ||||
| -rw-r--r-- | synapse/handlers/login.py | 4 | ||||
| -rw-r--r-- | synapse/handlers/register.py | 6 | ||||
| -rw-r--r-- | synapse/http/client.py | 154 |
11 files changed, 170 insertions, 93 deletions
diff --git a/CHANGES.rst b/CHANGES.rst index 78c178bafd..5a284c3853 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -1,3 +1,27 @@ +Changes in synapse 0.5.0 (2014-11-19) +===================================== +This release includes changes to the federation protocol and client-server API +that is not backwards compatible. + +This release also changes the internal database schemas and so requires servers to +drop their current history. See UPGRADES.rst for details. + +Homeserver: + * Add authentication and authorization to the federation protocol. Events are + now signed by their originating homeservers. + * Implement the new authorization model for rooms. + * Split out web client into a seperate repository: matrix-angular-sdk. + * Change the structure of PDUs. + * Fix bug where user could not join rooms via an alias containing 4-byte + UTF-8 characters. + * Merge concept of PDUs and Events internally. + * Improve logging by adding request ids to log lines. + * Implement a very basic room initial sync API. + * Implement the new invite/join federation APIs. + +Webclient: + * The webclient has been moved to a seperate repository. + Changes in synapse 0.4.2 (2014-10-31) ===================================== diff --git a/UPGRADE.rst b/UPGRADE.rst index 99ce1a2d3d..961f4da31c 100644 --- a/UPGRADE.rst +++ b/UPGRADE.rst @@ -1,3 +1,39 @@ +Upgrading to v0.5.0 +=================== + +The webclient has been split out into a seperate repository/pacakage in this +release. Before you restart your homeserver you will need to pull in the +webclient package by running:: + + python setup.py develop --user + +This release completely changes the database schema and so requires upgrading +it before starting the new version of the homeserver. + +The script "database-prepare-for-0.5.0.sh" should be used to upgrade the +database. This will save all user information, such as logins and profiles, +but will otherwise purge the database. This includes messages, which +rooms the home server was a member of and room alias mappings. + +If you would like to keep your history, please take a copy of your database +file and ask for help in #matrix:matrix.org. The upgrade process is, +unfortunately, non trivial and requires human intervention to resolve any +resulting conflicts during the upgrade process. + +Before running the command the homeserver should be first completely +shutdown. To run it, simply specify the location of the database, e.g.: + + ./database-prepare-for-0.5.0.sh "homeserver.db" + +Once this has successfully completed it will be safe to restart the +homeserver. You may notice that the homeserver takes a few seconds longer to +restart than usual as it reinitializes the database. + +On startup of the new version, users can either rejoin remote rooms using room +aliases or by being reinvited. Alternatively, if any other homeserver sends a +message to a room that the homeserver was previously in the local HS will +automatically rejoin the room. + Upgrading to v0.4.0 =================== diff --git a/VERSION b/VERSION index 2b7c5ae018..8f0916f768 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -0.4.2 +0.5.0 diff --git a/database-prepare-for-0.5.0.sh b/database-prepare-for-0.5.0.sh new file mode 100755 index 0000000000..e824cb583e --- /dev/null +++ b/database-prepare-for-0.5.0.sh @@ -0,0 +1,21 @@ +#!/bin/bash + +# This is will prepare a synapse database for running with v0.5.0 of synapse. +# It will store all the user information, but will *delete* all messages and +# room data. + +set -e + +cp "$1" "$1.bak" + +DUMP=$(sqlite3 "$1" << 'EOF' +.dump users +.dump access_tokens +.dump presence +.dump profiles +EOF +) + +rm "$1" + +sqlite3 "$1" <<< "$DUMP" diff --git a/setup.py b/setup.py index ed21799e72..d0d649612d 100755 --- a/setup.py +++ b/setup.py @@ -26,13 +26,13 @@ def read(fname): return open(os.path.join(os.path.dirname(__file__), fname)).read() setup( - name="synapse", - version=read("VERSION"), + name="matrix-synapse", + version=read("VERSION").strip(), packages=find_packages(exclude=["tests", "tests.*"]), description="Reference Synapse Home Server", install_requires=[ "syutil==0.0.2", - "syweb==0.0.1", + "matrix_angular_sdk==0.5.0", "Twisted>=14.0.0", "service_identity>=1.0.0", "pyopenssl>=0.14", @@ -45,7 +45,7 @@ setup( dependency_links=[ "https://github.com/matrix-org/syutil/tarball/v0.0.2#egg=syutil-0.0.2", "https://github.com/pyca/pynacl/tarball/52dbe2dc33f1#egg=pynacl-0.3.0", - "https://github.com/matrix-org/matrix-angular-sdk/tarball/master/#egg=syweb-0.0.1", + "https://github.com/matrix-org/matrix-angular-sdk/tarball/v0.5.0/#egg=matrix_angular_sdk-0.5.0", ], setup_requires=[ "setuptools_trial", diff --git a/synapse/__init__.py b/synapse/__init__.py index 23ae5f003f..14564e735e 100644 --- a/synapse/__init__.py +++ b/synapse/__init__.py @@ -16,4 +16,4 @@ """ This is a reference implementation of a synapse home server. """ -__version__ = "0.4.2" +__version__ = "0.5.0" diff --git a/synapse/app/homeserver.py b/synapse/app/homeserver.py index de16d8a2e1..8f8d566191 100755 --- a/synapse/app/homeserver.py +++ b/synapse/app/homeserver.py @@ -26,7 +26,7 @@ from twisted.web.server import Site from synapse.http.server import JsonResource, RootRedirect from synapse.http.content_repository import ContentRepoResource from synapse.http.server_key_resource import LocalKey -from synapse.http.client import MatrixHttpClient +from synapse.http.client import MatrixFederationHttpClient from synapse.api.urls import ( CLIENT_PREFIX, FEDERATION_PREFIX, WEB_CLIENT_PREFIX, CONTENT_REPO_PREFIX, SERVER_KEY_PREFIX, @@ -51,7 +51,7 @@ logger = logging.getLogger(__name__) class SynapseHomeServer(HomeServer): def build_http_client(self): - return MatrixHttpClient(self) + return MatrixFederationHttpClient(self) def build_resource_for_client(self): return JsonResource() diff --git a/synapse/handlers/directory.py b/synapse/handlers/directory.py index ed9b0f8551..05e5c6ecfc 100644 --- a/synapse/handlers/directory.py +++ b/synapse/handlers/directory.py @@ -128,7 +128,7 @@ class DirectoryHandler(BaseHandler): "servers": result.servers, }) else: - raise SynapseError(404, "Room alias \"%s\" not found", room_alias) + raise SynapseError(404, "Room alias \"%s\" not found" % (room_alias,)) @defer.inlineCallbacks diff --git a/synapse/handlers/login.py b/synapse/handlers/login.py index 99d15261d4..1204dc3b8f 100644 --- a/synapse/handlers/login.py +++ b/synapse/handlers/login.py @@ -17,7 +17,7 @@ from twisted.internet import defer from ._base import BaseHandler from synapse.api.errors import LoginError, Codes -from synapse.http.client import IdentityServerHttpClient +from synapse.http.client import SimpleHttpClient from synapse.util.emailutils import EmailException import synapse.util.emailutils as emailutils @@ -97,7 +97,7 @@ class LoginHandler(BaseHandler): @defer.inlineCallbacks def _query_email(self, email): - httpCli = IdentityServerHttpClient(self.hs) + httpCli = SimpleHttpClient(self.hs) data = yield httpCli.get_json( 'matrix.org:8090', # TODO FIXME This should be configurable. "/_matrix/identity/api/v1/lookup?medium=email&address=" + diff --git a/synapse/handlers/register.py b/synapse/handlers/register.py index 7df9d9b82d..a39230bc76 100644 --- a/synapse/handlers/register.py +++ b/synapse/handlers/register.py @@ -22,7 +22,7 @@ from synapse.api.errors import ( ) from ._base import BaseHandler import synapse.util.stringutils as stringutils -from synapse.http.client import IdentityServerHttpClient +from synapse.http.client import SimpleHttpClient from synapse.http.client import CaptchaServerHttpClient import base64 @@ -159,7 +159,7 @@ class RegistrationHandler(BaseHandler): def _threepid_from_creds(self, creds): # TODO: get this from the homeserver rather than creating a new one for # each request - httpCli = IdentityServerHttpClient(self.hs) + httpCli = SimpleHttpClient(self.hs) # XXX: make this configurable! trustedIdServers = ['matrix.org:8090'] if not creds['idServer'] in trustedIdServers: @@ -178,7 +178,7 @@ class RegistrationHandler(BaseHandler): @defer.inlineCallbacks def _bind_threepid(self, creds, mxid): - httpCli = IdentityServerHttpClient(self.hs) + httpCli = SimpleHttpClient(self.hs) data = yield httpCli.post_urlencoded_get_json( creds['idServer'], "/_matrix/identity/api/v1/3pid/bind", diff --git a/synapse/http/client.py b/synapse/http/client.py index dea61ba1e0..6361ac55f9 100644 --- a/synapse/http/client.py +++ b/synapse/http/client.py @@ -154,16 +154,81 @@ class BaseHttpClient(object): defer.returnValue(response) -class MatrixHttpClient(BaseHttpClient): - """ Wrapper around the twisted HTTP client api. Implements +class SimpleHttpClient(BaseHttpClient): + """ + A simple, no-frills HTTP client with methods that wrap up common ways of using HTTP in Matrix + """ + def _getEndpoint(self, reactor, destination): + return matrix_endpoint(reactor, destination, timeout=10) + + @defer.inlineCallbacks + def post_urlencoded_get_json(self, destination, path, args={}): + logger.debug("post_urlencoded_get_json args: %s", args) + query_bytes = urllib.urlencode(args, True) + + def body_callback(method, url_bytes, headers_dict): + return FileBodyProducer(StringIO(query_bytes)) + + response = yield self._create_request( + destination.encode("ascii"), + "POST", + path.encode("ascii"), + body_callback=body_callback, + headers_dict={ + "Content-Type": ["application/x-www-form-urlencoded"] + } + ) + + body = yield readBody(response) + + defer.returnValue(json.loads(body)) + + @defer.inlineCallbacks + def get_json(self, destination, path, args={}, retry_on_dns_fail=True): + """ Get's some json from the given host and path + + Args: + destination (str): The remote server to send the HTTP request to. + path (str): The HTTP path. + args (dict): A dictionary used to create query strings, defaults to + None. + **Note**: The value of each key is assumed to be an iterable + and *not* a string. + + Returns: + Deferred: Succeeds when we get *any* HTTP response. + + The result of the deferred is a tuple of `(code, response)`, + where `response` is a dict representing the decoded JSON body. + """ + logger.debug("get_json args: %s", args) + + query_bytes = urllib.urlencode(args, True) + logger.debug("Query bytes: %s Retry DNS: %s", args, retry_on_dns_fail) + + response = yield self._create_request( + destination.encode("ascii"), + "GET", + path.encode("ascii"), + query_bytes=query_bytes, + retry_on_dns_fail=retry_on_dns_fail, + body_callback=None + ) + + body = yield readBody(response) + + defer.returnValue(json.loads(body)) + + +class MatrixFederationHttpClient(BaseHttpClient): + """HTTP client used to talk to other homeservers over the federation protocol. + Send client certificates and signs requests. Attributes: agent (twisted.web.client.Agent): The twisted Agent used to send the requests. """ - RETRY_DNS_LOOKUP_FAILURES = "__retry_dns" - def __init__(self, hs): self.signing_key = hs.config.signing_key[0] self.server_name = hs.hostname @@ -293,83 +358,17 @@ class MatrixHttpClient(BaseHttpClient): ) -class IdentityServerHttpClient(BaseHttpClient): - """Separate HTTP client for talking to the Identity servers since they - don't use SRV records and talk x-www-form-urlencoded rather than JSON. +class CaptchaServerHttpClient(BaseHttpClient): + """ + Separate HTTP client for talking to google's captcha servers + Only slightly special because accepts partial download responses """ - def _getEndpoint(self, reactor, destination): - #TODO: This should be talking TLS - return matrix_endpoint(reactor, destination, timeout=10) - - @defer.inlineCallbacks - def post_urlencoded_get_json(self, destination, path, args={}): - logger.debug("post_urlencoded_get_json args: %s", args) - query_bytes = urllib.urlencode(args, True) - - def body_callback(method, url_bytes, headers_dict): - return FileBodyProducer(StringIO(query_bytes)) - - response = yield self._create_request( - destination.encode("ascii"), - "POST", - path.encode("ascii"), - body_callback=body_callback, - headers_dict={ - "Content-Type": ["application/x-www-form-urlencoded"] - } - ) - - body = yield readBody(response) - - defer.returnValue(json.loads(body)) - - @defer.inlineCallbacks - def get_json(self, destination, path, args={}, retry_on_dns_fail=True): - """ Get's some json from the given host homeserver and path - - Args: - destination (str): The remote server to send the HTTP request - to. - path (str): The HTTP path. - args (dict): A dictionary used to create query strings, defaults to - None. - **Note**: The value of each key is assumed to be an iterable - and *not* a string. - - Returns: - Deferred: Succeeds when we get *any* HTTP response. - - The result of the deferred is a tuple of `(code, response)`, - where `response` is a dict representing the decoded JSON body. - """ - logger.debug("get_json args: %s", args) - - query_bytes = urllib.urlencode(args, True) - logger.debug("Query bytes: %s Retry DNS: %s", args, retry_on_dns_fail) - - response = yield self._create_request( - destination.encode("ascii"), - "GET", - path.encode("ascii"), - query_bytes=query_bytes, - retry_on_dns_fail=retry_on_dns_fail, - body_callback=None - ) - - body = yield readBody(response) - - defer.returnValue(json.loads(body)) - - -class CaptchaServerHttpClient(MatrixHttpClient): - """Separate HTTP client for talking to google's captcha servers""" def _getEndpoint(self, reactor, destination): return matrix_endpoint(reactor, destination, timeout=10) @defer.inlineCallbacks - def post_urlencoded_get_raw(self, destination, path, accept_partial=False, - args={}): + def post_urlencoded_get_raw(self, destination, path, args={}): query_bytes = urllib.urlencode(args, True) def body_callback(method, url_bytes, headers_dict): @@ -389,10 +388,7 @@ class CaptchaServerHttpClient(MatrixHttpClient): body = yield readBody(response) defer.returnValue(body) except PartialDownloadError as e: - if accept_partial: - defer.returnValue(e.response) - else: - raise e + defer.returnValue(e.response) def _print_ex(e): |