diff options
Diffstat (limited to '')
36 files changed, 310 insertions, 509 deletions
diff --git a/.dockerignore b/.dockerignore index 0180602e56..3c3996eb4c 100644 --- a/.dockerignore +++ b/.dockerignore @@ -5,3 +5,5 @@ demo/etc tox.ini .git/* .tox/* +debian/matrix-synapse/ +debian/matrix-synapse-*/ diff --git a/.gitignore b/.gitignore index 3b2252ad8a..1b632646bb 100644 --- a/.gitignore +++ b/.gitignore @@ -18,7 +18,7 @@ homeserver*.db homeserver*.log homeserver*.log.* homeserver*.pid -homeserver*.yaml +/homeserver*.yaml *.signing.key *.tls.crt @@ -26,6 +26,8 @@ homeserver*.yaml *.tls.key .coverage +.coverage.* +!.coverage.rc htmlcov demo/*/*.db @@ -57,3 +59,7 @@ env/ .vscode/ .ropeproject/ + +*.deb + +/debs diff --git a/MANIFEST.in b/MANIFEST.in index ec18819bc9..29303cc8b5 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -36,6 +36,7 @@ prune demo/etc prune docker prune .circleci prune .coveragerc +prune debian exclude jenkins* recursive-exclude jenkins *.sh diff --git a/changelog.d/4212.misc b/changelog.d/4212.misc new file mode 100644 index 0000000000..42f2546cf9 --- /dev/null +++ b/changelog.d/4212.misc @@ -0,0 +1 @@ +Debian packages utilising a virtualenv with bundled dependencies can now be built. diff --git a/debian/.gitignore b/debian/.gitignore new file mode 100644 index 0000000000..f027374ae2 --- /dev/null +++ b/debian/.gitignore @@ -0,0 +1,7 @@ +/matrix-synapse-py3.*.debhelper +/matrix-synapse-py3.debhelper.log +/matrix-synapse-py3.substvars +/matrix-synapse-*/ +/files +/debhelper-build-stamp +/.debhelper diff --git a/debian/NEWS b/debian/NEWS new file mode 100644 index 0000000000..367e08f851 --- /dev/null +++ b/debian/NEWS @@ -0,0 +1,22 @@ +matrix-synapse-py3 (0.34.0) stable; urgency=medium + + matrix-synapse-py3 is intended as a drop-in replacement for the existing + matrix-synapse package. The replacement should be relatively seamless, + however, please note the following important differences to matrix-synapse: + + * Most importantly, the matrix-synapse service now runs under Python 3 rather + than Python 2.7. + + * Synapse is installed into its own virtualenv (in /opt/venvs/matrix-synapse) + instead of using the system python libraries. (This may mean that you can + remove a number of old dependencies with `apt-get autoremove`). + + matrix-synapse-py3 will take over responsibility for the existing + configuration files, including the matrix-synapse systemd service. + + Beware, however, that `apt-get purge matrix-synapse` will *disable* the + matrix-synapse service (so that it will not be started on reboot), even + though that service is no longer being provided by the matrix-synapse + package. It can be re-enabled with `systemctl enable matrix-synapse`. + + -- Richard van der Hoff <richard@matrix.org> Wed, 19 Dec 2018 14:00:00 +0000 diff --git a/debian/build_virtualenv b/debian/build_virtualenv new file mode 100755 index 0000000000..61ffb13192 --- /dev/null +++ b/debian/build_virtualenv @@ -0,0 +1,48 @@ +#!/bin/bash +# +# runs dh_virtualenv to build the virtualenv in the build directory, +# and then runs the trial tests against the installed synapse. + +set -e + +export DH_VIRTUALENV_INSTALL_ROOT=/opt/venvs +SNAKE=/usr/bin/python3 + +# try to set the CFLAGS so any compiled C extensions are compiled with the most +# generic as possible x64 instructions, so that compiling it on a new Intel chip +# doesn't enable features not available on older ones or AMD. +# +# TODO: add similar things for non-amd64, or figure out a more generic way to +# do this. + +case `dpkg-architecture -q DEB_HOST_ARCH` in + amd64) + export CFLAGS=-march=x86-64 + ;; +esac + +# Use --builtin-venv to use the better `venv` module from CPython 3.4+ rather +# than the 2/3 compatible `virtualenv`. + +dh_virtualenv \ + --install-suffix "matrix-synapse" \ + --builtin-venv \ + --setuptools \ + --python "$SNAKE" \ + --upgrade-pip \ + --preinstall="lxml" \ + --preinstall="mock" \ + --extra-pip-arg="--no-cache-dir" \ + --extra-pip-arg="--compile" + +# we copy the tests to a temporary directory so that we can put them on the +# PYTHONPATH without putting the uninstalled synapse on the pythonpath. +tmpdir=`mktemp -d` +trap "rm -r $tmpdir" EXIT + +cp -r tests "$tmpdir" +cd debian/matrix-synapse-py3 + +PYTHONPATH="$tmpdir" \ + ./opt/venvs/matrix-synapse/bin/python \ + -B -m twisted.trial --reporter=text -j2 tests diff --git a/debian/changelog b/debian/changelog index 20167978cf..040c8e7cd3 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,3 +1,14 @@ +matrix-synapse-py3 (0.34.0) stable; urgency=medium + + * New synapse release 0.34.0. + * Synapse is now installed into a Python 3 virtual environment with + up-to-date dependencies. + * The matrix-synapse service will now be restarted when the package is + upgraded. + (Fixes https://github.com/matrix-org/package-synapse-debian/issues/18) + + -- Synapse packaging team <packages@matrix.org> Wed, 19 Dec 2018 14:00:00 +0000 + matrix-synapse (0.33.9-1matrix1) stretch; urgency=medium [ Erik Johnston ] diff --git a/debian/control b/debian/control index 854d9688f3..552a81dcb0 100644 --- a/debian/control +++ b/debian/control @@ -1,77 +1,37 @@ -Source: matrix-synapse -Maintainer: Erik Johnston <erikj@matrix.org> -Section: python -Priority: optional +Source: matrix-synapse-py3 +Section: contrib/python +Priority: extra +Maintainer: Synapse Packaging team <packages@matrix.org> Build-Depends: debhelper (>= 9), - dh-python, - dh-systemd (>= 1.5), - po-debconf, - python (>= 2.6.6-3), - python-bcrypt, - python-blist, - python-canonicaljson (>=1.1.3), - python-daemonize, - python-frozendict (>= 0.4), - python-lxml, - python-mock, - python-msgpack (>=0.3.0), - python-nacl (>= 0.3.0), - python-netaddr, - python-openssl (>= 0.14), - python-pil, - python-psutil, - python-pyasn1, - python-pydenticon, - python-pymacaroons-pynacl, - python-pysaml2, - python-service-identity (>= 1.0.0), - python-setuptools (>= 0.6b3), - python-signedjson (>= 1.0.0), - python-sortedcontainers, - python-syutil (>= 0.0.7), - python-treq (>= 15.1.0), - python-twisted (>= 17.1.0), - python-unpaddedbase64 (>= 1.0.1), - python-yaml, - python-phonenumbers (>= 8.2.0), - python-jsonschema (>=2.5.1), - python-prometheus-client, - python-attr -Standards-Version: 3.9.8 -X-Python-Version: >= 2.7 + dh-systemd, + dh-virtualenv (>= 1.0), + lsb-release, + python3-dev, + python3, + python3-setuptools, + python3-pip, + python3-venv, + tar, +Standards-Version: 3.9.5 +Homepage: https://github.com/matrix-org/synapse -Package: matrix-synapse -Architecture: all +Package: matrix-synapse-py3 +Architecture: amd64 +Conflicts: matrix-synapse +Pre-Depends: dpkg (>= 1.16.1) Depends: - ${misc:Depends}, - ${python:Depends}, adduser, debconf, - lsb-base (>= 3.0-6), - python-attr (>= 16.0.0), - python-twisted (>= 17.1.0), - python-canonicaljson (>=1.1.3), - python-prometheus-client (>=0.0.14), + python3-distutils|libpython3-stdlib (<< 3.6), + python3, + ${misc:Depends}, +# some of our scripts use perl, but none of them are important, +# so we put perl:Depends in Suggests rather than Depends. Suggests: - python-bleach (>= 1.4.2), - python-jinja2 (>= 2.8), -Recommends: - python-psycopg2, - python-lxml, + sqlite3, + ${perl:Depends}, Description: Open federated Instant Messaging and VoIP server Matrix is an ambitious new ecosystem for open federated Instant Messaging and VoIP. Synapse is a reference Matrix server implementation. - . - Everything in Matrix happens in a room. Rooms are distributed and do - not exist on any single server. Rooms can be located using - convenience aliases like #matrix:matrix.org or #test:localhost:8448. - . - Matrix user IDs look like @matthew:matrix.org (although in the future - you will normally refer to yourself and others using a 3PID: email - address, phone number, etc rather than manipulating Matrix user IDs) - . - The overall architecture is: - client <------> homeserver <=============> homeserver <------> client - https://a.org/_matrix https://b.net/_matrix diff --git a/debian/copyright b/debian/copyright index 35597e0804..95c21ea12a 100644 --- a/debian/copyright +++ b/debian/copyright @@ -3,7 +3,7 @@ Upstream-Name: synapse Source: https://github.com/matrix-org/synapse Files: * -Copyright: 2014-2017, OpenMarket Ltd +Copyright: 2014-2017, OpenMarket Ltd, 2017-2018 New Vector Ltd License: Apache-2.0 Files: synapse/config/saml2.py diff --git a/debian/gbp.conf b/debian/gbp.conf deleted file mode 100644 index 0432accfa0..0000000000 --- a/debian/gbp.conf +++ /dev/null @@ -1,5 +0,0 @@ -[DEFAULT] -debian-branch = debian - -[dch] -distribution = stable diff --git a/debian/homeserver.yaml b/debian/homeserver.yaml index 04ece25d49..188a2d5483 100644 --- a/debian/homeserver.yaml +++ b/debian/homeserver.yaml @@ -70,13 +70,9 @@ pid_file: "/var/run/matrix-synapse.pid" # # cpu_affinity: 0xFFFFFFFF -# Whether to serve a web client from the HTTP/HTTPS root resource. -web_client: False - -# The root directory to server for the above web client. -# If left undefined, synapse will serve the matrix-angular-sdk web client. -# Make sure matrix-angular-sdk is installed with pip if web_client is True -# and web_client_location is undefined +# The path to the web client which will be served at /_matrix/client/ +# if 'webclient' is configured under the 'listeners' configuration. +# # web_client_location: "/path/to/web/root" # The public-facing base URL for the client API (not including _matrix/...) diff --git a/debian/matrix-synapse-py3.links b/debian/matrix-synapse-py3.links new file mode 100644 index 0000000000..bf19efa562 --- /dev/null +++ b/debian/matrix-synapse-py3.links @@ -0,0 +1,4 @@ +opt/venvs/matrix-synapse/bin/hash_password usr/bin/hash_password +opt/venvs/matrix-synapse/bin/register_new_matrix_user usr/bin/register_new_matrix_user +opt/venvs/matrix-synapse/bin/synapse_port_db usr/bin/synapse_port_db +opt/venvs/matrix-synapse/bin/synctl usr/bin/synctl diff --git a/debian/postinst b/debian/matrix-synapse-py3.postinst index 0509acd0a4..0509acd0a4 100755..100644 --- a/debian/postinst +++ b/debian/matrix-synapse-py3.postinst diff --git a/debian/matrix-synapse-py3.preinst b/debian/matrix-synapse-py3.preinst new file mode 100644 index 0000000000..4b5612f050 --- /dev/null +++ b/debian/matrix-synapse-py3.preinst @@ -0,0 +1,31 @@ +#!/bin/sh -e + +# Attempt to undo some of the braindamage caused by +# https://github.com/matrix-org/package-synapse-debian/issues/18. +# +# Due to reasons [1], the old python2 matrix-synapse package will not stop the +# service when the package is uninstalled. Our maintainer scripts will do the +# right thing in terms of ensuring the service is enabled and unmasked, but +# then do a `systemctl start matrix-synapse`, which of course does nothing - +# leaving the old (py2) service running. +# +# There should normally be no reason for the service to be running during our +# preinst, so we assume that if it *is* running, it's due to that situation, +# and stop it. +# +# [1] dh_systemd_start doesn't do anything because it sees that there is an +# init.d script with the same name, so leaves it to dh_installinit. +# +# dh_installinit doesn't do anything because somebody gave it a --no-start +# for unknown reasons. + +if [ -x /bin/systemctl ]; then + if /bin/systemctl --quiet is-active -- matrix-synapse; then + echo >&2 "stopping existing matrix-synapse service" + /bin/systemctl stop matrix-synapse || true + fi +fi + +#DEBHELPER# + +exit 0 diff --git a/debian/matrix-synapse-py3.triggers b/debian/matrix-synapse-py3.triggers new file mode 100644 index 0000000000..f8c1fdb021 --- /dev/null +++ b/debian/matrix-synapse-py3.triggers @@ -0,0 +1,9 @@ +# Register interest in Python interpreter changes and +# don't make the Python package dependent on the virtualenv package +# processing (noawait) +interest-noawait /usr/bin/python3.5 +interest-noawait /usr/bin/python3.6 +interest-noawait /usr/bin/python3.7 + +# Also provide a symbolic trigger for all dh-virtualenv packages +interest dh-virtualenv-interpreter-update diff --git a/debian/matrix-synapse.init b/debian/matrix-synapse.init deleted file mode 100755 index 7a9e8b3296..0000000000 --- a/debian/matrix-synapse.init +++ /dev/null @@ -1,184 +0,0 @@ -#!/bin/sh -### BEGIN INIT INFO -# Provides: matrix-synapse -# Required-Start: $local_fs $network $remote_fs $syslog -# Required-Stop: $local_fs $network $remote_fs $syslog -# Default-Start: 2 3 4 5 -# Default-Stop: 0 1 6 -# Short-Description: <Enter a short description of the software> -# Description: <Enter a long description of the software> -# <...> -# <...> -### END INIT INFO - -# Author: Paul "LeoNerd" Evans <paul@matrix.org> - -# Do NOT "set -e" - -# PATH should only include /usr/* if it runs after the mountnfs.sh script -PATH=/sbin:/usr/sbin:/bin:/usr/bin -DESC="matrix-synapse" -NAME=matrix-synapse -SCRIPTNAME=/etc/init.d/$NAME - -PYTHON="/usr/bin/python" -CONFIGS="--config-path /etc/matrix-synapse/homeserver.yaml --config-path /etc/matrix-synapse/conf.d/" -USER="matrix-synapse" -SHAREDIR=/var/lib/$NAME - -# Exit if the package is not installed -[ -f "/etc/matrix-synapse/homeserver.yaml" ] || exit 0 - -# Read configuration variable file if it is present -[ -r /etc/default/$NAME ] && . /etc/default/$NAME - -# Load the VERBOSE setting and other rcS variables -. /lib/init/vars.sh - -# Define LSB log_* functions. -# Depend on lsb-base (>= 3.2-14) to ensure that this file is present -# and status_of_proc is working. -. /lib/lsb/init-functions - -get_config_key() -{ - python -m synapse.config read "$1" $CONFIGS || return 2 -} - -# -# Function that starts the daemon/service -# -do_start() -{ - # Running --generate-config to create keys if any are absent. - # Doesn't matter if not - $PYTHON -m "synapse.app.homeserver" $CONFIGS --generate-keys || return 2 - - # Return - # 0 if daemon has been started - # 1 if daemon was already running - # 2 if daemon could not be started - PIDFILE=`get_config_key "pid_file"` - RETVAL=$? - if [ "$RETVAL" != 0 ]; then - return $RETVAL - fi - if [ -r "$PIDFILE" ]; then - kill -0 `cat $PIDFILE` && return 1 - fi - - export PYTHONPATH - - # Create the PID file so that synapse can write to it as nonroot - touch $PIDFILE - chown $USER:nogroup $PIDFILE - chown $USER:nogroup $SHAREDIR/media/ - chown $USER:nogroup $SHAREDIR/uploads/ - - start-stop-daemon --start --pidfile $PIDFILE --chuid $USER \ - --exec $PYTHON -- -m "synapse.app.homeserver" $CONFIGS --daemonize || return 2 - - return 0 -} - -# -# Function that stops the daemon/service -# -do_stop() -{ - # Return - # 0 if daemon has been stopped - # 1 if daemon was already stopped - # 2 if daemon could not be stopped - # other if a failure occurred - PIDFILE=`get_config_key "pid_file"` - RETVAL=$? - if [ "$RETVAL" != 0 ]; then - return $RETVAL - fi - - start-stop-daemon --stop --quiet --retry=TERM/30/KILL/5 --pidfile $PIDFILE --user $USER --exec $PYTHON - RETVAL="$?" - [ "$RETVAL" = 2 ] && return 2 - - # Many daemons don't delete their pidfiles when they exit. - rm -f $PIDFILE - return "$RETVAL" -} - -# -# Function that sends a SIGHUP to the daemon/service -# -do_reload() { - # - # If the daemon can reload its configuration without - # restarting (for example, when it is sent a SIGHUP), - # then implement that here. - # - return 1 -} - -case "$1" in - start) - [ "$VERBOSE" != no ] && log_daemon_msg "Starting $DESC" "$NAME" - do_start - case "$?" in - 0|1) [ "$VERBOSE" != no ] && log_end_msg 0 ;; - 2) [ "$VERBOSE" != no ] && log_end_msg 1 ;; - esac - ;; - stop) - [ "$VERBOSE" != no ] && log_daemon_msg "Stopping $DESC" "$NAME" - do_stop - case "$?" in - 0|1) [ "$VERBOSE" != no ] && log_end_msg 0 ;; - 2) [ "$VERBOSE" != no ] && log_end_msg 1 ;; - esac - ;; - status) - PIDFILE=`get_config_key "pid_file"` - RETVAL=$? - if [ "$RETVAL" != 0 ]; then - return $RETVAL - fi - status_of_proc -p "$PIDFILE" "$PYTHON" "$NAME" && exit 0 || exit $? - ;; - #reload|force-reload) - # - # If do_reload() is not implemented then leave this commented out - # and leave 'force-reload' as an alias for 'restart'. - # - #log_daemon_msg "Reloading $DESC" "$NAME" - #do_reload - #log_end_msg $? - #;; - restart|force-reload) - # - # If the "reload" option is implemented then remove the - # 'force-reload' alias - # - log_daemon_msg "Restarting $DESC" "$NAME" - do_stop - case "$?" in - 0|1) - do_start - case "$?" in - 0) log_end_msg 0 ;; - 1) log_end_msg 1 ;; # Old process is still running - *) log_end_msg 1 ;; # Failed to start - esac - ;; - *) - # Failed to stop - log_end_msg 1 - ;; - esac - ;; - *) - #echo "Usage: $SCRIPTNAME {start|stop|restart|reload|force-reload}" >&2 - echo "Usage: $SCRIPTNAME {start|stop|status|restart|force-reload}" >&2 - exit 3 - ;; -esac - -: diff --git a/debian/matrix-synapse.service b/debian/matrix-synapse.service index ab94e073a6..2e9cd83b5f 100644 --- a/debian/matrix-synapse.service +++ b/debian/matrix-synapse.service @@ -6,8 +6,8 @@ Type=simple User=matrix-synapse WorkingDirectory=/var/lib/matrix-synapse EnvironmentFile=/etc/default/matrix-synapse -ExecStartPre=/usr/bin/python -m synapse.app.homeserver --config-path=/etc/matrix-synapse/homeserver.yaml --config-path=/etc/matrix-synapse/conf.d/ --generate-keys -ExecStart=/usr/bin/python -m synapse.app.homeserver --config-path=/etc/matrix-synapse/homeserver.yaml --config-path=/etc/matrix-synapse/conf.d/ +ExecStartPre=/opt/venvs/matrix-synapse/bin/python -m synapse.app.homeserver --config-path=/etc/matrix-synapse/homeserver.yaml --config-path=/etc/matrix-synapse/conf.d/ --generate-keys +ExecStart=/opt/venvs/matrix-synapse/bin/python -m synapse.app.homeserver --config-path=/etc/matrix-synapse/homeserver.yaml --config-path=/etc/matrix-synapse/conf.d/ Restart=always RestartSec=3 diff --git a/debian/patches/0001-tox.patch b/debian/patches/0001-tox.patch deleted file mode 100644 index 2cf3ec0fe8..0000000000 --- a/debian/patches/0001-tox.patch +++ /dev/null @@ -1,19 +0,0 @@ -From: Erik Johnston <erikj@matrix.org> -Date: Fri, 10 Jun 2016 10:57:07 +0100 -Subject: tox - ---- - tox.ini | 1 + - 1 file changed, 1 insertion(+) - -Index: package-synapse-debian/tox.ini -=================================================================== ---- package-synapse-debian.orig/tox.ini -+++ package-synapse-debian/tox.ini -@@ -1,5 +1,6 @@ - [tox] - envlist = packaging, py27, py36, pep8, check_isort -+sitepackages = True - - [base] - deps = diff --git a/debian/patches/0002-change_instructions.patch b/debian/patches/0002-change_instructions.patch deleted file mode 100644 index 933de3ab94..0000000000 --- a/debian/patches/0002-change_instructions.patch +++ /dev/null @@ -1,34 +0,0 @@ -From: Erik Johnston <erikj@matrix.org> -Date: Fri, 10 Jun 2016 10:57:07 +0100 -Subject: change_instructions - ---- - synapse/config/_base.py | 10 ++++++++++ - 1 file changed, 10 insertions(+) - ---- a/synapse/config/_base.py -+++ b/synapse/config/_base.py -@@ -31,6 +31,11 @@ - MISSING_REPORT_STATS_CONFIG_INSTRUCTIONS = """\ - Please opt in or out of reporting anonymized homeserver usage statistics, by - setting the `report_stats` key in your config file to either True or False. -+ -+To set it run: -+ -+ dpkg-reconfigure matrix-synapse -+ - """ - - MISSING_REPORT_STATS_SPIEL = """\ -@@ -45,6 +50,11 @@ - - MISSING_SERVER_NAME = """\ - Missing mandatory `server_name` config option. -+ -+To set it run: -+ -+ dpkg-reconfigure matrix-synapse -+ - """ - - diff --git a/debian/patches/0004-webclient-instructions.patch b/debian/patches/0004-webclient-instructions.patch deleted file mode 100644 index e2e61a8446..0000000000 --- a/debian/patches/0004-webclient-instructions.patch +++ /dev/null @@ -1,27 +0,0 @@ -From: Erik Johnston <erikj@matrix.org> -Date: Fri, 10 Jun 2016 10:57:07 +0100 -Subject: webclient-instructions - ---- - synapse/app/homeserver.py | 8 +++----- - 1 file changed, 3 insertions(+), 5 deletions(-) - -Index: package-synapse-debian/synapse/app/homeserver.py -=================================================================== ---- package-synapse-debian.orig/synapse/app/homeserver.py -+++ package-synapse-debian/synapse/app/homeserver.py -@@ -86,12 +86,11 @@ def build_resource_for_web_client(hs): - "Please either install the matrix-angular-sdk or configure\n" - "the location of the source to serve via the configuration\n" - "option `web_client_location`\n\n" -- "To install the `matrix-angular-sdk` via pip, run:\n\n" -- " pip install '%(dep)s'\n" -+ "To install the `matrix-angular-sdk` via apt-get, run:\n\n" -+ " apt-get install matrix-synapse-angular-client\n" - "\n" - "You can also disable hosting of the webclient via the\n" - "configuration option `web_client`\n" -- % {"dep": CONDITIONAL_REQUIREMENTS["web_client"].keys()[0]} - ) - syweb_path = os.path.dirname(syweb.__file__) - webclient_path = os.path.join(syweb_path, "webclient") diff --git a/debian/patches/0006-Don-t-require-strict-nacl-0.3.0-requirement.patch b/debian/patches/0006-Don-t-require-strict-nacl-0.3.0-requirement.patch deleted file mode 100644 index 8370c96166..0000000000 --- a/debian/patches/0006-Don-t-require-strict-nacl-0.3.0-requirement.patch +++ /dev/null @@ -1,21 +0,0 @@ -From: Erik Johnston <erikj@matrix.org> -Date: Mon, 20 Jun 2016 13:20:37 +0100 -Subject: Don't require strict nacl==0.3.0 requirement - ---- - synapse/python_dependencies.py | 2 +- - 1 file changed, 1 insertion(+), 1 deletion(-) - -Index: package-synapse-debian/synapse/python_dependencies.py -=================================================================== ---- package-synapse-debian.orig/synapse/python_dependencies.py -+++ package-synapse-debian/synapse/python_dependencies.py -@@ -37,7 +37,7 @@ REQUIREMENTS = { - "unpaddedbase64>=1.1.0": ["unpaddedbase64>=1.1.0"], - "canonicaljson>=1.1.3": ["canonicaljson>=1.1.3"], - "signedjson>=1.0.0": ["signedjson>=1.0.0"], -- "pynacl>=1.2.1": ["nacl>=1.2.1", "nacl.bindings"], -+ "pynacl>=0.3.0": ["nacl>=0.3.0", "nacl.bindings"], - "service_identity>=16.0.0": ["service_identity>=16.0.0"], - "Twisted>=17.1.0": ["twisted>=17.1.0"], - "treq>=15.1": ["treq>=15.1"], diff --git a/debian/patches/bcrypt.patch b/debian/patches/bcrypt.patch deleted file mode 100644 index a962949920..0000000000 --- a/debian/patches/bcrypt.patch +++ /dev/null @@ -1,30 +0,0 @@ -Index: package-synapse-debian/synapse/handlers/auth.py -=================================================================== ---- package-synapse-debian.orig/synapse/handlers/auth.py -+++ package-synapse-debian/synapse/handlers/auth.py -@@ -921,10 +921,10 @@ class AuthHandler(BaseHandler): - # Normalise the Unicode in the password - pw = unicodedata.normalize("NFKC", password) - -- return bcrypt.checkpw( -+ return bcrypt.hashpw( - pw.encode('utf8') + self.hs.config.password_pepper.encode("utf8"), - stored_hash -- ) -+ ) == stored_hash - - if stored_hash: - if not isinstance(stored_hash, bytes): -Index: package-synapse-debian/synapse/python_dependencies.py -=================================================================== ---- package-synapse-debian.orig/synapse/python_dependencies.py -+++ package-synapse-debian/synapse/python_dependencies.py -@@ -49,7 +49,7 @@ REQUIREMENTS = { - "pyasn1>=0.1.9": ["pyasn1"], - "pyasn1-modules>=0.0.7": ["pyasn1_modules"], - "daemonize>=2.3.1": ["daemonize"], -- "bcrypt>=3.1.0": ["bcrypt>=3.1.0"], -+ "bcrypt": ["bcrypt"], - "pillow>=3.1.2": ["PIL"], - "sortedcontainers>=1.4.4": ["sortedcontainers"], - "psutil>=2.0.0": ["psutil>=2.0.0"], diff --git a/debian/patches/no_install_with_pip b/debian/patches/no_install_with_pip deleted file mode 100644 index 654656f596..0000000000 --- a/debian/patches/no_install_with_pip +++ /dev/null @@ -1,43 +0,0 @@ -Index: package-synapse-debian/synapse/app/__init__.py -=================================================================== ---- package-synapse-debian.orig/synapse/app/__init__.py -+++ package-synapse-debian/synapse/app/__init__.py -@@ -25,8 +25,8 @@ try: - except python_dependencies.MissingRequirementError as e: - message = "\n".join([ - "Missing Requirement: %s" % (str(e),), -- "To install run:", -- " pip install --upgrade --force \"%s\"" % (e.dependency,), -+ "To install, try:", -+ " sudo apt-get install python-%s" % (e.dependency,), - "", - ]) - sys.stderr.writelines(message) -Index: package-synapse-debian/synapse/config/jwt_config.py -=================================================================== ---- package-synapse-debian.orig/synapse/config/jwt_config.py -+++ package-synapse-debian/synapse/config/jwt_config.py -@@ -19,7 +19,7 @@ MISSING_JWT = ( - """Missing jwt library. This is required for jwt login. - - Install by running: -- pip install pyjwt -+ sudo apt-get install python-jwt - """ - ) - -Index: package-synapse-debian/synapse/config/repository.py -=================================================================== ---- package-synapse-debian.orig/synapse/config/repository.py -+++ package-synapse-debian/synapse/config/repository.py -@@ -27,9 +27,7 @@ MISSING_LXML = ( - """Missing lxml library. This is required for URL preview API. - - Install by running: -- pip install lxml -- -- Requires libxslt1-dev system package. -+ sudo apt-get install python-lxml - """ - ) - diff --git a/debian/patches/remove-webclient.patch b/debian/patches/remove-webclient.patch deleted file mode 100644 index bfd2fce576..0000000000 --- a/debian/patches/remove-webclient.patch +++ /dev/null @@ -1,31 +0,0 @@ -Index: package-synapse-debian/synapse/app/homeserver.py -=================================================================== ---- package-synapse-debian.orig/synapse/app/homeserver.py -+++ package-synapse-debian/synapse/app/homeserver.py -@@ -124,7 +124,7 @@ class SynapseHomeServer(HomeServer): - for res in listener_config["resources"]: - for name in res["names"]: - resources.update(self._configure_named_resource( -- name, res.get("compress", False), -+ config, name, res.get("compress", False), - )) - - additional_resources = listener_config.get("additional_resources", {}) -@@ -171,7 +171,7 @@ class SynapseHomeServer(HomeServer): - ) - logger.info("Synapse now listening on port %d", port) - -- def _configure_named_resource(self, name, compress=False): -+ def _configure_named_resource(self, config, name, compress=False): - """Build a resource map for a named resource - - Args: -@@ -235,7 +235,7 @@ class SynapseHomeServer(HomeServer): - if name in ["keys", "federation"]: - resources[SERVER_KEY_V2_PREFIX] = KeyApiV2Resource(self) - -- if name == "webclient": -+ if name == "webclient" and config.web_client: - resources[WEB_CLIENT_PREFIX] = build_resource_for_web_client(self) - - if name == "metrics" and self.get_config().enable_metrics: diff --git a/debian/patches/series b/debian/patches/series deleted file mode 100644 index ae46209e92..0000000000 --- a/debian/patches/series +++ /dev/null @@ -1,7 +0,0 @@ -0001-tox.patch -0002-change_instructions.patch -0004-webclient-instructions.patch -0006-Don-t-require-strict-nacl-0.3.0-requirement.patch -remove-webclient.patch -bcrypt.patch -no_install_with_pip diff --git a/debian/pydist-overrides b/debian/pydist-overrides deleted file mode 100644 index e2e34b7d30..0000000000 --- a/debian/pydist-overrides +++ /dev/null @@ -1,5 +0,0 @@ -matrix-angular-sdk -jinja2 -bleach -ldap3 -matrix-synapse-ldap3 diff --git a/debian/rules b/debian/rules index ad0e95c83d..05cbbdde08 100755 --- a/debian/rules +++ b/debian/rules @@ -1,18 +1,22 @@ #!/usr/bin/make -f +# +# Build Debian package using https://github.com/spotify/dh-virtualenv +# -# This file was automatically generated by stdeb 0.8.2 at -# Fri, 12 Jun 2015 14:32:03 +0100 -export PYBUILD_NAME=matrix-synapse -%: - dh $@ --with python2 --with systemd --buildsystem=pybuild --no-guessing-deps +override_dh_systemd_enable: + dh_systemd_enable --name=matrix-synapse -override_dh_auto_install: - python setup.py install --root=debian/matrix-synapse --install-layout=deb +override_dh_installinit: + dh_installinit --name=matrix-synapse -override_dh_auto_build: +override_dh_strip: -override_dh_installinit: - dh_installinit --no-start +override_dh_shlibdeps: -override_dh_auto_test: - PYTHONPATH=. trial tests +override_dh_virtualenv: + ./debian/build_virtualenv + +# We are restricted to compat level 9 (because xenial), so have to +# enable the systemd bits manually. +%: + dh $@ --with python-virtualenv --with systemd diff --git a/debian/source/format b/debian/source/format index 163aaf8d82..89ae9db8f8 100644 --- a/debian/source/format +++ b/debian/source/format @@ -1 +1 @@ -3.0 (quilt) +3.0 (native) diff --git a/debian/watch b/debian/watch deleted file mode 100644 index b1b10d4cd0..0000000000 --- a/debian/watch +++ /dev/null @@ -1,11 +0,0 @@ -# Example watch control file for uscan -# Rename this file to "watch" and then you can run the "uscan" command -# to check for upstream updates and more. -# See uscan(1) for format - -# Compulsory line, this is a version 3 file -version=3 - - -opts=filenamemangle=s/.+\/v?(\d\S*)\.tar\.gz/matrix-synapse-$1\.tar\.gz/,uversionmangle=s/-?rc/~rc/ \ - https://github.com/matrix-org/synapse/tags .*/v?(\d[^\s\-]*)\.tar\.gz debian uupdate diff --git a/docker/Dockerfile-dhvirtualenv b/docker/Dockerfile-dhvirtualenv new file mode 100644 index 0000000000..ea6b650af2 --- /dev/null +++ b/docker/Dockerfile-dhvirtualenv @@ -0,0 +1,35 @@ +# A dockerfile which builds a docker image for building a debian package for +# synapse. The distro to build for is passed as a docker build var. +# +# The default entrypoint expects the synapse source to be mounted as a +# (read-only) volume at /synapse/source, and an output directory at /debs. +# +# A pair of environment variables (TARGET_USERID and TARGET_GROUPID) can be +# passed to the docker container; if these are set, the build script will chown +# the build products accordingly, to avoid ending up with things owned by root +# in the host filesystem. + +# Get the distro we want to pull from as a dynamic build variable +ARG distro="" +FROM ${distro} + +# Install the build dependencies +RUN apt-get update -qq -o Acquire::Languages=none \ + && env DEBIAN_FRONTEND=noninteractive apt-get install \ + -yqq --no-install-recommends -o Dpkg::Options::=--force-unsafe-io \ + build-essential \ + debhelper \ + devscripts \ + dh-systemd \ + dh-virtualenv \ + equivs \ + lsb-release \ + python3-dev \ + python3-pip \ + python3-setuptools \ + python3-venv \ + sqlite3 \ + wget + +WORKDIR /synapse/source +ENTRYPOINT ["bash","/synapse/source/docker/build_debian.sh"] diff --git a/docker/build_debian.sh b/docker/build_debian.sh new file mode 100644 index 0000000000..cea5067fe9 --- /dev/null +++ b/docker/build_debian.sh @@ -0,0 +1,41 @@ +#!/bin/bash + +# The script to build the Debian package, as ran inside the Docker image. + +set -ex + +DIST=`lsb_release -c -s` + +# We need to build a newer dh_virtualenv on older OSes like Xenial. +if [ "$DIST" = 'xenial' ]; then + mkdir -p /tmp/dhvenv + cd /tmp/dhvenv + wget https://github.com/spotify/dh-virtualenv/archive/1.1.tar.gz + tar xvf 1.1.tar.gz + cd dh-virtualenv-1.1/ + env DEBIAN_FRONTEND=noninteractive mk-build-deps -ri -t "apt-get -yqq --no-install-recommends -o Dpkg::Options::=--force-unsafe-io" + dpkg-buildpackage -us -uc -b + cd /tmp/dhvenv + apt-get install -yqq ./dh-virtualenv_1.1-1_all.deb +fi + + +# we get a read-only copy of the source: make a writeable copy +cp -aT /synapse/source /synapse/build +cd /synapse/build + +# add an entry to the changelog for this distribution +dch -M -l "+$DIST" "build for $DIST" +dch -M -r "" --force-distribution --distribution "$DIST" + +dpkg-buildpackage -us -uc + +ls -l .. + +# copy the build results out, setting perms if necessary +shopt -s nullglob +for i in ../*.deb ../*.dsc ../*.tar.xz ../*.changes ../*.buildinfo; do + [ -z "$TARGET_USERID" ] || chown "$TARGET_USERID" "$i" + [ -z "$TARGET_GROUPID" ] || chgrp "$TARGET_GROUPID" "$i" + mv "$i" /debs +done diff --git a/docker/build_debian_packages.sh b/docker/build_debian_packages.sh new file mode 100755 index 0000000000..eafed4ac41 --- /dev/null +++ b/docker/build_debian_packages.sh @@ -0,0 +1,39 @@ +#!/bin/bash + +# Build the Debian packages using Docker images. +# +# This script builds the Docker images and then executes them sequentially, each +# one building a Debian package for the targeted operating system. It is +# designed to be a "single command" to produce all the images. +# +# By default, builds for all known distributions, but a list of distributions +# can be passed on the commandline for debugging. + +set -ex + +cd `dirname $0` + +if [ $# -lt 1 ]; then + DISTS=(debian:stretch debian:sid ubuntu:xenial ubuntu:bionic ubuntu:cosmic) +else + DISTS=("$@") +fi + +# Make the dir where the debs will live. +# +# Note that we deliberately put this outside the source tree, otherwise we tend +# to get source packages which are full of debs. (We could hack around that +# with more magic in the build_debian.sh script, but that doesn't solve the +# problem for natively-run dpkg-buildpakage). + +mkdir -p ../../debs + +# Build each OS image; +for i in "${DISTS[@]}"; do + TAG=$(echo ${i} | cut -d ":" -f 2) + docker build --tag dh-venv-builder:${TAG} --build-arg distro=${i} -f Dockerfile-dhvirtualenv . + docker run -it --rm --volume=$(pwd)/../\:/synapse/source:ro --volume=$(pwd)/../../debs:/debs \ + -e TARGET_USERID=$(id -u) \ + -e TARGET_GROUPID=$(id -g) \ + dh-venv-builder:${TAG} +done diff --git a/synapse/python_dependencies.py b/synapse/python_dependencies.py index 92422c6ffc..96cd154234 100644 --- a/synapse/python_dependencies.py +++ b/synapse/python_dependencies.py @@ -78,7 +78,7 @@ CONDITIONAL_REQUIREMENTS = { }, "postgres": { "psycopg2>=2.6": ["psycopg2"] - } + }, } diff --git a/synapse/storage/e2e_room_keys.py b/synapse/storage/e2e_room_keys.py index 16b7f005aa..45cebe61d1 100644 --- a/synapse/storage/e2e_room_keys.py +++ b/synapse/storage/e2e_room_keys.py @@ -182,7 +182,7 @@ class EndToEndRoomKeyStore(SQLBaseStore): keyvalues = { "user_id": user_id, - "version": version, + "version": int(version), } if room_id: keyvalues['room_id'] = room_id diff --git a/tox.ini b/tox.ini index 731094b5da..44371f211f 100644 --- a/tox.ini +++ b/tox.ini @@ -119,6 +119,7 @@ setenv = [testenv:packaging] +skip_install=True deps = check-manifest commands = |