From 78ba0e7ab8f1d57eee6aa56e9d496d838a24c6f3 Mon Sep 17 00:00:00 2001 From: Neil Johnson Date: Tue, 20 Nov 2018 11:29:25 +0000 Subject: Remove riot.im from the list of trusted Identity Servers in the default configuration (#4207) --- docker/conf/homeserver.yaml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) (limited to 'docker') diff --git a/docker/conf/homeserver.yaml b/docker/conf/homeserver.yaml index 1b0f655d26..d5074be6dd 100644 --- a/docker/conf/homeserver.yaml +++ b/docker/conf/homeserver.yaml @@ -150,10 +150,12 @@ enable_group_creation: true # The list of identity servers trusted to verify third party # identifiers by this server. +# +# Also defines the ID server which will be called when an account is +# deactivated (one will be picked arbitrarily). trusted_third_party_id_servers: - matrix.org - vector.im - - riot.im ## Metrics ### -- cgit 1.4.1 From 75937e90338f93dfb9cf1c528c66f20fe213b0ad Mon Sep 17 00:00:00 2001 From: Richard van der Hoff Date: Tue, 4 Dec 2018 12:29:55 +0000 Subject: Remove obsolete settings from docker homeserver.yaml These aren't used, because we have a `log_config` setting. --- changelog.d/4261.misc | 1 + docker/conf/homeserver.yaml | 4 +--- 2 files changed, 2 insertions(+), 3 deletions(-) create mode 100644 changelog.d/4261.misc (limited to 'docker') diff --git a/changelog.d/4261.misc b/changelog.d/4261.misc new file mode 100644 index 0000000000..2a88c8c856 --- /dev/null +++ b/changelog.d/4261.misc @@ -0,0 +1 @@ +Remove obsolete `verbose` and `log_file` settings from `homeserver.yaml` for Docker image. \ No newline at end of file diff --git a/docker/conf/homeserver.yaml b/docker/conf/homeserver.yaml index d5074be6dd..c2b8576a32 100644 --- a/docker/conf/homeserver.yaml +++ b/docker/conf/homeserver.yaml @@ -14,6 +14,7 @@ server_name: "{{ SYNAPSE_SERVER_NAME }}" pid_file: /homeserver.pid web_client: False soft_file_limit: 0 +log_config: "/compiled/log.config" ## Ports ## @@ -67,9 +68,6 @@ database: ## Performance ## event_cache_size: "{{ SYNAPSE_EVENT_CACHE_SIZE or "10K" }}" -verbose: 0 -log_file: "/data/homeserver.log" -log_config: "/compiled/log.config" ## Ratelimiting ## -- cgit 1.4.1 From 1a6d5bfa08b4ba618dcab68e4ed524c4db07da2b Mon Sep 17 00:00:00 2001 From: Richard van der Hoff Date: Thu, 20 Dec 2018 11:33:29 +0000 Subject: Debian packaging via dh_virtualenv (#4285) --- .dockerignore | 2 + .gitignore | 8 +- MANIFEST.in | 1 + changelog.d/4212.misc | 1 + debian/.gitignore | 7 + debian/NEWS | 22 +++ debian/build_virtualenv | 48 ++++++ debian/changelog | 11 ++ debian/control | 92 +++-------- debian/copyright | 2 +- debian/gbp.conf | 5 - debian/homeserver.yaml | 10 +- debian/matrix-synapse-py3.links | 4 + debian/matrix-synapse-py3.postinst | 39 +++++ debian/matrix-synapse-py3.preinst | 31 ++++ debian/matrix-synapse-py3.triggers | 9 + debian/matrix-synapse.init | 184 --------------------- debian/matrix-synapse.service | 4 +- debian/patches/0001-tox.patch | 19 --- debian/patches/0002-change_instructions.patch | 34 ---- debian/patches/0004-webclient-instructions.patch | 27 --- ...n-t-require-strict-nacl-0.3.0-requirement.patch | 21 --- debian/patches/bcrypt.patch | 30 ---- debian/patches/no_install_with_pip | 43 ----- debian/patches/remove-webclient.patch | 31 ---- debian/patches/series | 7 - debian/postinst | 39 ----- debian/pydist-overrides | 5 - debian/rules | 28 ++-- debian/source/format | 2 +- debian/watch | 11 -- docker/Dockerfile-dhvirtualenv | 35 ++++ docker/build_debian.sh | 41 +++++ docker/build_debian_packages.sh | 39 +++++ synapse/python_dependencies.py | 2 +- synapse/storage/e2e_room_keys.py | 2 +- tox.ini | 1 + 37 files changed, 349 insertions(+), 548 deletions(-) create mode 100644 changelog.d/4212.misc create mode 100644 debian/.gitignore create mode 100644 debian/NEWS create mode 100755 debian/build_virtualenv delete mode 100644 debian/gbp.conf create mode 100644 debian/matrix-synapse-py3.links create mode 100644 debian/matrix-synapse-py3.postinst create mode 100644 debian/matrix-synapse-py3.preinst create mode 100644 debian/matrix-synapse-py3.triggers delete mode 100755 debian/matrix-synapse.init delete mode 100644 debian/patches/0001-tox.patch delete mode 100644 debian/patches/0002-change_instructions.patch delete mode 100644 debian/patches/0004-webclient-instructions.patch delete mode 100644 debian/patches/0006-Don-t-require-strict-nacl-0.3.0-requirement.patch delete mode 100644 debian/patches/bcrypt.patch delete mode 100644 debian/patches/no_install_with_pip delete mode 100644 debian/patches/remove-webclient.patch delete mode 100644 debian/patches/series delete mode 100755 debian/postinst delete mode 100644 debian/pydist-overrides delete mode 100644 debian/watch create mode 100644 docker/Dockerfile-dhvirtualenv create mode 100644 docker/build_debian.sh create mode 100755 docker/build_debian_packages.sh (limited to 'docker') diff --git a/.dockerignore b/.dockerignore index 0180602e56..3c3996eb4c 100644 --- a/.dockerignore +++ b/.dockerignore @@ -5,3 +5,5 @@ demo/etc tox.ini .git/* .tox/* +debian/matrix-synapse/ +debian/matrix-synapse-*/ diff --git a/.gitignore b/.gitignore index 3b2252ad8a..1b632646bb 100644 --- a/.gitignore +++ b/.gitignore @@ -18,7 +18,7 @@ homeserver*.db homeserver*.log homeserver*.log.* homeserver*.pid -homeserver*.yaml +/homeserver*.yaml *.signing.key *.tls.crt @@ -26,6 +26,8 @@ homeserver*.yaml *.tls.key .coverage +.coverage.* +!.coverage.rc htmlcov demo/*/*.db @@ -57,3 +59,7 @@ env/ .vscode/ .ropeproject/ + +*.deb + +/debs diff --git a/MANIFEST.in b/MANIFEST.in index ec18819bc9..29303cc8b5 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -36,6 +36,7 @@ prune demo/etc prune docker prune .circleci prune .coveragerc +prune debian exclude jenkins* recursive-exclude jenkins *.sh diff --git a/changelog.d/4212.misc b/changelog.d/4212.misc new file mode 100644 index 0000000000..42f2546cf9 --- /dev/null +++ b/changelog.d/4212.misc @@ -0,0 +1 @@ +Debian packages utilising a virtualenv with bundled dependencies can now be built. diff --git a/debian/.gitignore b/debian/.gitignore new file mode 100644 index 0000000000..f027374ae2 --- /dev/null +++ b/debian/.gitignore @@ -0,0 +1,7 @@ +/matrix-synapse-py3.*.debhelper +/matrix-synapse-py3.debhelper.log +/matrix-synapse-py3.substvars +/matrix-synapse-*/ +/files +/debhelper-build-stamp +/.debhelper diff --git a/debian/NEWS b/debian/NEWS new file mode 100644 index 0000000000..367e08f851 --- /dev/null +++ b/debian/NEWS @@ -0,0 +1,22 @@ +matrix-synapse-py3 (0.34.0) stable; urgency=medium + + matrix-synapse-py3 is intended as a drop-in replacement for the existing + matrix-synapse package. The replacement should be relatively seamless, + however, please note the following important differences to matrix-synapse: + + * Most importantly, the matrix-synapse service now runs under Python 3 rather + than Python 2.7. + + * Synapse is installed into its own virtualenv (in /opt/venvs/matrix-synapse) + instead of using the system python libraries. (This may mean that you can + remove a number of old dependencies with `apt-get autoremove`). + + matrix-synapse-py3 will take over responsibility for the existing + configuration files, including the matrix-synapse systemd service. + + Beware, however, that `apt-get purge matrix-synapse` will *disable* the + matrix-synapse service (so that it will not be started on reboot), even + though that service is no longer being provided by the matrix-synapse + package. It can be re-enabled with `systemctl enable matrix-synapse`. + + -- Richard van der Hoff Wed, 19 Dec 2018 14:00:00 +0000 diff --git a/debian/build_virtualenv b/debian/build_virtualenv new file mode 100755 index 0000000000..61ffb13192 --- /dev/null +++ b/debian/build_virtualenv @@ -0,0 +1,48 @@ +#!/bin/bash +# +# runs dh_virtualenv to build the virtualenv in the build directory, +# and then runs the trial tests against the installed synapse. + +set -e + +export DH_VIRTUALENV_INSTALL_ROOT=/opt/venvs +SNAKE=/usr/bin/python3 + +# try to set the CFLAGS so any compiled C extensions are compiled with the most +# generic as possible x64 instructions, so that compiling it on a new Intel chip +# doesn't enable features not available on older ones or AMD. +# +# TODO: add similar things for non-amd64, or figure out a more generic way to +# do this. + +case `dpkg-architecture -q DEB_HOST_ARCH` in + amd64) + export CFLAGS=-march=x86-64 + ;; +esac + +# Use --builtin-venv to use the better `venv` module from CPython 3.4+ rather +# than the 2/3 compatible `virtualenv`. + +dh_virtualenv \ + --install-suffix "matrix-synapse" \ + --builtin-venv \ + --setuptools \ + --python "$SNAKE" \ + --upgrade-pip \ + --preinstall="lxml" \ + --preinstall="mock" \ + --extra-pip-arg="--no-cache-dir" \ + --extra-pip-arg="--compile" + +# we copy the tests to a temporary directory so that we can put them on the +# PYTHONPATH without putting the uninstalled synapse on the pythonpath. +tmpdir=`mktemp -d` +trap "rm -r $tmpdir" EXIT + +cp -r tests "$tmpdir" +cd debian/matrix-synapse-py3 + +PYTHONPATH="$tmpdir" \ + ./opt/venvs/matrix-synapse/bin/python \ + -B -m twisted.trial --reporter=text -j2 tests diff --git a/debian/changelog b/debian/changelog index 20167978cf..040c8e7cd3 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,3 +1,14 @@ +matrix-synapse-py3 (0.34.0) stable; urgency=medium + + * New synapse release 0.34.0. + * Synapse is now installed into a Python 3 virtual environment with + up-to-date dependencies. + * The matrix-synapse service will now be restarted when the package is + upgraded. + (Fixes https://github.com/matrix-org/package-synapse-debian/issues/18) + + -- Synapse packaging team Wed, 19 Dec 2018 14:00:00 +0000 + matrix-synapse (0.33.9-1matrix1) stretch; urgency=medium [ Erik Johnston ] diff --git a/debian/control b/debian/control index 854d9688f3..552a81dcb0 100644 --- a/debian/control +++ b/debian/control @@ -1,77 +1,37 @@ -Source: matrix-synapse -Maintainer: Erik Johnston -Section: python -Priority: optional +Source: matrix-synapse-py3 +Section: contrib/python +Priority: extra +Maintainer: Synapse Packaging team Build-Depends: debhelper (>= 9), - dh-python, - dh-systemd (>= 1.5), - po-debconf, - python (>= 2.6.6-3), - python-bcrypt, - python-blist, - python-canonicaljson (>=1.1.3), - python-daemonize, - python-frozendict (>= 0.4), - python-lxml, - python-mock, - python-msgpack (>=0.3.0), - python-nacl (>= 0.3.0), - python-netaddr, - python-openssl (>= 0.14), - python-pil, - python-psutil, - python-pyasn1, - python-pydenticon, - python-pymacaroons-pynacl, - python-pysaml2, - python-service-identity (>= 1.0.0), - python-setuptools (>= 0.6b3), - python-signedjson (>= 1.0.0), - python-sortedcontainers, - python-syutil (>= 0.0.7), - python-treq (>= 15.1.0), - python-twisted (>= 17.1.0), - python-unpaddedbase64 (>= 1.0.1), - python-yaml, - python-phonenumbers (>= 8.2.0), - python-jsonschema (>=2.5.1), - python-prometheus-client, - python-attr -Standards-Version: 3.9.8 -X-Python-Version: >= 2.7 + dh-systemd, + dh-virtualenv (>= 1.0), + lsb-release, + python3-dev, + python3, + python3-setuptools, + python3-pip, + python3-venv, + tar, +Standards-Version: 3.9.5 +Homepage: https://github.com/matrix-org/synapse -Package: matrix-synapse -Architecture: all +Package: matrix-synapse-py3 +Architecture: amd64 +Conflicts: matrix-synapse +Pre-Depends: dpkg (>= 1.16.1) Depends: - ${misc:Depends}, - ${python:Depends}, adduser, debconf, - lsb-base (>= 3.0-6), - python-attr (>= 16.0.0), - python-twisted (>= 17.1.0), - python-canonicaljson (>=1.1.3), - python-prometheus-client (>=0.0.14), + python3-distutils|libpython3-stdlib (<< 3.6), + python3, + ${misc:Depends}, +# some of our scripts use perl, but none of them are important, +# so we put perl:Depends in Suggests rather than Depends. Suggests: - python-bleach (>= 1.4.2), - python-jinja2 (>= 2.8), -Recommends: - python-psycopg2, - python-lxml, + sqlite3, + ${perl:Depends}, Description: Open federated Instant Messaging and VoIP server Matrix is an ambitious new ecosystem for open federated Instant Messaging and VoIP. Synapse is a reference Matrix server implementation. - . - Everything in Matrix happens in a room. Rooms are distributed and do - not exist on any single server. Rooms can be located using - convenience aliases like #matrix:matrix.org or #test:localhost:8448. - . - Matrix user IDs look like @matthew:matrix.org (although in the future - you will normally refer to yourself and others using a 3PID: email - address, phone number, etc rather than manipulating Matrix user IDs) - . - The overall architecture is: - client <------> homeserver <=============> homeserver <------> client - https://a.org/_matrix https://b.net/_matrix diff --git a/debian/copyright b/debian/copyright index 35597e0804..95c21ea12a 100644 --- a/debian/copyright +++ b/debian/copyright @@ -3,7 +3,7 @@ Upstream-Name: synapse Source: https://github.com/matrix-org/synapse Files: * -Copyright: 2014-2017, OpenMarket Ltd +Copyright: 2014-2017, OpenMarket Ltd, 2017-2018 New Vector Ltd License: Apache-2.0 Files: synapse/config/saml2.py diff --git a/debian/gbp.conf b/debian/gbp.conf deleted file mode 100644 index 0432accfa0..0000000000 --- a/debian/gbp.conf +++ /dev/null @@ -1,5 +0,0 @@ -[DEFAULT] -debian-branch = debian - -[dch] -distribution = stable diff --git a/debian/homeserver.yaml b/debian/homeserver.yaml index 04ece25d49..188a2d5483 100644 --- a/debian/homeserver.yaml +++ b/debian/homeserver.yaml @@ -70,13 +70,9 @@ pid_file: "/var/run/matrix-synapse.pid" # # cpu_affinity: 0xFFFFFFFF -# Whether to serve a web client from the HTTP/HTTPS root resource. -web_client: False - -# The root directory to server for the above web client. -# If left undefined, synapse will serve the matrix-angular-sdk web client. -# Make sure matrix-angular-sdk is installed with pip if web_client is True -# and web_client_location is undefined +# The path to the web client which will be served at /_matrix/client/ +# if 'webclient' is configured under the 'listeners' configuration. +# # web_client_location: "/path/to/web/root" # The public-facing base URL for the client API (not including _matrix/...) diff --git a/debian/matrix-synapse-py3.links b/debian/matrix-synapse-py3.links new file mode 100644 index 0000000000..bf19efa562 --- /dev/null +++ b/debian/matrix-synapse-py3.links @@ -0,0 +1,4 @@ +opt/venvs/matrix-synapse/bin/hash_password usr/bin/hash_password +opt/venvs/matrix-synapse/bin/register_new_matrix_user usr/bin/register_new_matrix_user +opt/venvs/matrix-synapse/bin/synapse_port_db usr/bin/synapse_port_db +opt/venvs/matrix-synapse/bin/synctl usr/bin/synctl diff --git a/debian/matrix-synapse-py3.postinst b/debian/matrix-synapse-py3.postinst new file mode 100644 index 0000000000..0509acd0a4 --- /dev/null +++ b/debian/matrix-synapse-py3.postinst @@ -0,0 +1,39 @@ +#!/bin/sh -e + +. /usr/share/debconf/confmodule + +CONFIGFILE_SERVERNAME="/etc/matrix-synapse/conf.d/server_name.yaml" +CONFIGFILE_REPORTSTATS="/etc/matrix-synapse/conf.d/report_stats.yaml" +USER="matrix-synapse" + +case "$1" in + configure|reconfigure) + # Set server name in config file + mkdir -p "/etc/matrix-synapse/conf.d/" + db_get matrix-synapse/server-name + + if [ "$RET" ]; then + echo "server_name: $RET" > $CONFIGFILE_SERVERNAME + fi + + db_get matrix-synapse/report-stats + if [ "$RET" ]; then + echo "report_stats: $RET" > $CONFIGFILE_REPORTSTATS + fi + + if ! getent passwd $USER >/dev/null; then + adduser --quiet --system --no-create-home --home /var/lib/matrix-synapse $USER + fi + + for DIR in /var/lib/matrix-synapse /var/log/matrix-synapse /etc/matrix-synapse; do + if ! dpkg-statoverride --list --quiet $DIR >/dev/null; then + dpkg-statoverride --force --quiet --update --add $USER nogroup 0755 $DIR + fi + done + + ;; +esac + +#DEBHELPER# + +exit 0 diff --git a/debian/matrix-synapse-py3.preinst b/debian/matrix-synapse-py3.preinst new file mode 100644 index 0000000000..4b5612f050 --- /dev/null +++ b/debian/matrix-synapse-py3.preinst @@ -0,0 +1,31 @@ +#!/bin/sh -e + +# Attempt to undo some of the braindamage caused by +# https://github.com/matrix-org/package-synapse-debian/issues/18. +# +# Due to reasons [1], the old python2 matrix-synapse package will not stop the +# service when the package is uninstalled. Our maintainer scripts will do the +# right thing in terms of ensuring the service is enabled and unmasked, but +# then do a `systemctl start matrix-synapse`, which of course does nothing - +# leaving the old (py2) service running. +# +# There should normally be no reason for the service to be running during our +# preinst, so we assume that if it *is* running, it's due to that situation, +# and stop it. +# +# [1] dh_systemd_start doesn't do anything because it sees that there is an +# init.d script with the same name, so leaves it to dh_installinit. +# +# dh_installinit doesn't do anything because somebody gave it a --no-start +# for unknown reasons. + +if [ -x /bin/systemctl ]; then + if /bin/systemctl --quiet is-active -- matrix-synapse; then + echo >&2 "stopping existing matrix-synapse service" + /bin/systemctl stop matrix-synapse || true + fi +fi + +#DEBHELPER# + +exit 0 diff --git a/debian/matrix-synapse-py3.triggers b/debian/matrix-synapse-py3.triggers new file mode 100644 index 0000000000..f8c1fdb021 --- /dev/null +++ b/debian/matrix-synapse-py3.triggers @@ -0,0 +1,9 @@ +# Register interest in Python interpreter changes and +# don't make the Python package dependent on the virtualenv package +# processing (noawait) +interest-noawait /usr/bin/python3.5 +interest-noawait /usr/bin/python3.6 +interest-noawait /usr/bin/python3.7 + +# Also provide a symbolic trigger for all dh-virtualenv packages +interest dh-virtualenv-interpreter-update diff --git a/debian/matrix-synapse.init b/debian/matrix-synapse.init deleted file mode 100755 index 7a9e8b3296..0000000000 --- a/debian/matrix-synapse.init +++ /dev/null @@ -1,184 +0,0 @@ -#!/bin/sh -### BEGIN INIT INFO -# Provides: matrix-synapse -# Required-Start: $local_fs $network $remote_fs $syslog -# Required-Stop: $local_fs $network $remote_fs $syslog -# Default-Start: 2 3 4 5 -# Default-Stop: 0 1 6 -# Short-Description: -# Description: -# <...> -# <...> -### END INIT INFO - -# Author: Paul "LeoNerd" Evans - -# Do NOT "set -e" - -# PATH should only include /usr/* if it runs after the mountnfs.sh script -PATH=/sbin:/usr/sbin:/bin:/usr/bin -DESC="matrix-synapse" -NAME=matrix-synapse -SCRIPTNAME=/etc/init.d/$NAME - -PYTHON="/usr/bin/python" -CONFIGS="--config-path /etc/matrix-synapse/homeserver.yaml --config-path /etc/matrix-synapse/conf.d/" -USER="matrix-synapse" -SHAREDIR=/var/lib/$NAME - -# Exit if the package is not installed -[ -f "/etc/matrix-synapse/homeserver.yaml" ] || exit 0 - -# Read configuration variable file if it is present -[ -r /etc/default/$NAME ] && . /etc/default/$NAME - -# Load the VERBOSE setting and other rcS variables -. /lib/init/vars.sh - -# Define LSB log_* functions. -# Depend on lsb-base (>= 3.2-14) to ensure that this file is present -# and status_of_proc is working. -. /lib/lsb/init-functions - -get_config_key() -{ - python -m synapse.config read "$1" $CONFIGS || return 2 -} - -# -# Function that starts the daemon/service -# -do_start() -{ - # Running --generate-config to create keys if any are absent. - # Doesn't matter if not - $PYTHON -m "synapse.app.homeserver" $CONFIGS --generate-keys || return 2 - - # Return - # 0 if daemon has been started - # 1 if daemon was already running - # 2 if daemon could not be started - PIDFILE=`get_config_key "pid_file"` - RETVAL=$? - if [ "$RETVAL" != 0 ]; then - return $RETVAL - fi - if [ -r "$PIDFILE" ]; then - kill -0 `cat $PIDFILE` && return 1 - fi - - export PYTHONPATH - - # Create the PID file so that synapse can write to it as nonroot - touch $PIDFILE - chown $USER:nogroup $PIDFILE - chown $USER:nogroup $SHAREDIR/media/ - chown $USER:nogroup $SHAREDIR/uploads/ - - start-stop-daemon --start --pidfile $PIDFILE --chuid $USER \ - --exec $PYTHON -- -m "synapse.app.homeserver" $CONFIGS --daemonize || return 2 - - return 0 -} - -# -# Function that stops the daemon/service -# -do_stop() -{ - # Return - # 0 if daemon has been stopped - # 1 if daemon was already stopped - # 2 if daemon could not be stopped - # other if a failure occurred - PIDFILE=`get_config_key "pid_file"` - RETVAL=$? - if [ "$RETVAL" != 0 ]; then - return $RETVAL - fi - - start-stop-daemon --stop --quiet --retry=TERM/30/KILL/5 --pidfile $PIDFILE --user $USER --exec $PYTHON - RETVAL="$?" - [ "$RETVAL" = 2 ] && return 2 - - # Many daemons don't delete their pidfiles when they exit. - rm -f $PIDFILE - return "$RETVAL" -} - -# -# Function that sends a SIGHUP to the daemon/service -# -do_reload() { - # - # If the daemon can reload its configuration without - # restarting (for example, when it is sent a SIGHUP), - # then implement that here. - # - return 1 -} - -case "$1" in - start) - [ "$VERBOSE" != no ] && log_daemon_msg "Starting $DESC" "$NAME" - do_start - case "$?" in - 0|1) [ "$VERBOSE" != no ] && log_end_msg 0 ;; - 2) [ "$VERBOSE" != no ] && log_end_msg 1 ;; - esac - ;; - stop) - [ "$VERBOSE" != no ] && log_daemon_msg "Stopping $DESC" "$NAME" - do_stop - case "$?" in - 0|1) [ "$VERBOSE" != no ] && log_end_msg 0 ;; - 2) [ "$VERBOSE" != no ] && log_end_msg 1 ;; - esac - ;; - status) - PIDFILE=`get_config_key "pid_file"` - RETVAL=$? - if [ "$RETVAL" != 0 ]; then - return $RETVAL - fi - status_of_proc -p "$PIDFILE" "$PYTHON" "$NAME" && exit 0 || exit $? - ;; - #reload|force-reload) - # - # If do_reload() is not implemented then leave this commented out - # and leave 'force-reload' as an alias for 'restart'. - # - #log_daemon_msg "Reloading $DESC" "$NAME" - #do_reload - #log_end_msg $? - #;; - restart|force-reload) - # - # If the "reload" option is implemented then remove the - # 'force-reload' alias - # - log_daemon_msg "Restarting $DESC" "$NAME" - do_stop - case "$?" in - 0|1) - do_start - case "$?" in - 0) log_end_msg 0 ;; - 1) log_end_msg 1 ;; # Old process is still running - *) log_end_msg 1 ;; # Failed to start - esac - ;; - *) - # Failed to stop - log_end_msg 1 - ;; - esac - ;; - *) - #echo "Usage: $SCRIPTNAME {start|stop|restart|reload|force-reload}" >&2 - echo "Usage: $SCRIPTNAME {start|stop|status|restart|force-reload}" >&2 - exit 3 - ;; -esac - -: diff --git a/debian/matrix-synapse.service b/debian/matrix-synapse.service index ab94e073a6..2e9cd83b5f 100644 --- a/debian/matrix-synapse.service +++ b/debian/matrix-synapse.service @@ -6,8 +6,8 @@ Type=simple User=matrix-synapse WorkingDirectory=/var/lib/matrix-synapse EnvironmentFile=/etc/default/matrix-synapse -ExecStartPre=/usr/bin/python -m synapse.app.homeserver --config-path=/etc/matrix-synapse/homeserver.yaml --config-path=/etc/matrix-synapse/conf.d/ --generate-keys -ExecStart=/usr/bin/python -m synapse.app.homeserver --config-path=/etc/matrix-synapse/homeserver.yaml --config-path=/etc/matrix-synapse/conf.d/ +ExecStartPre=/opt/venvs/matrix-synapse/bin/python -m synapse.app.homeserver --config-path=/etc/matrix-synapse/homeserver.yaml --config-path=/etc/matrix-synapse/conf.d/ --generate-keys +ExecStart=/opt/venvs/matrix-synapse/bin/python -m synapse.app.homeserver --config-path=/etc/matrix-synapse/homeserver.yaml --config-path=/etc/matrix-synapse/conf.d/ Restart=always RestartSec=3 diff --git a/debian/patches/0001-tox.patch b/debian/patches/0001-tox.patch deleted file mode 100644 index 2cf3ec0fe8..0000000000 --- a/debian/patches/0001-tox.patch +++ /dev/null @@ -1,19 +0,0 @@ -From: Erik Johnston -Date: Fri, 10 Jun 2016 10:57:07 +0100 -Subject: tox - ---- - tox.ini | 1 + - 1 file changed, 1 insertion(+) - -Index: package-synapse-debian/tox.ini -=================================================================== ---- package-synapse-debian.orig/tox.ini -+++ package-synapse-debian/tox.ini -@@ -1,5 +1,6 @@ - [tox] - envlist = packaging, py27, py36, pep8, check_isort -+sitepackages = True - - [base] - deps = diff --git a/debian/patches/0002-change_instructions.patch b/debian/patches/0002-change_instructions.patch deleted file mode 100644 index 933de3ab94..0000000000 --- a/debian/patches/0002-change_instructions.patch +++ /dev/null @@ -1,34 +0,0 @@ -From: Erik Johnston -Date: Fri, 10 Jun 2016 10:57:07 +0100 -Subject: change_instructions - ---- - synapse/config/_base.py | 10 ++++++++++ - 1 file changed, 10 insertions(+) - ---- a/synapse/config/_base.py -+++ b/synapse/config/_base.py -@@ -31,6 +31,11 @@ - MISSING_REPORT_STATS_CONFIG_INSTRUCTIONS = """\ - Please opt in or out of reporting anonymized homeserver usage statistics, by - setting the `report_stats` key in your config file to either True or False. -+ -+To set it run: -+ -+ dpkg-reconfigure matrix-synapse -+ - """ - - MISSING_REPORT_STATS_SPIEL = """\ -@@ -45,6 +50,11 @@ - - MISSING_SERVER_NAME = """\ - Missing mandatory `server_name` config option. -+ -+To set it run: -+ -+ dpkg-reconfigure matrix-synapse -+ - """ - - diff --git a/debian/patches/0004-webclient-instructions.patch b/debian/patches/0004-webclient-instructions.patch deleted file mode 100644 index e2e61a8446..0000000000 --- a/debian/patches/0004-webclient-instructions.patch +++ /dev/null @@ -1,27 +0,0 @@ -From: Erik Johnston -Date: Fri, 10 Jun 2016 10:57:07 +0100 -Subject: webclient-instructions - ---- - synapse/app/homeserver.py | 8 +++----- - 1 file changed, 3 insertions(+), 5 deletions(-) - -Index: package-synapse-debian/synapse/app/homeserver.py -=================================================================== ---- package-synapse-debian.orig/synapse/app/homeserver.py -+++ package-synapse-debian/synapse/app/homeserver.py -@@ -86,12 +86,11 @@ def build_resource_for_web_client(hs): - "Please either install the matrix-angular-sdk or configure\n" - "the location of the source to serve via the configuration\n" - "option `web_client_location`\n\n" -- "To install the `matrix-angular-sdk` via pip, run:\n\n" -- " pip install '%(dep)s'\n" -+ "To install the `matrix-angular-sdk` via apt-get, run:\n\n" -+ " apt-get install matrix-synapse-angular-client\n" - "\n" - "You can also disable hosting of the webclient via the\n" - "configuration option `web_client`\n" -- % {"dep": CONDITIONAL_REQUIREMENTS["web_client"].keys()[0]} - ) - syweb_path = os.path.dirname(syweb.__file__) - webclient_path = os.path.join(syweb_path, "webclient") diff --git a/debian/patches/0006-Don-t-require-strict-nacl-0.3.0-requirement.patch b/debian/patches/0006-Don-t-require-strict-nacl-0.3.0-requirement.patch deleted file mode 100644 index 8370c96166..0000000000 --- a/debian/patches/0006-Don-t-require-strict-nacl-0.3.0-requirement.patch +++ /dev/null @@ -1,21 +0,0 @@ -From: Erik Johnston -Date: Mon, 20 Jun 2016 13:20:37 +0100 -Subject: Don't require strict nacl==0.3.0 requirement - ---- - synapse/python_dependencies.py | 2 +- - 1 file changed, 1 insertion(+), 1 deletion(-) - -Index: package-synapse-debian/synapse/python_dependencies.py -=================================================================== ---- package-synapse-debian.orig/synapse/python_dependencies.py -+++ package-synapse-debian/synapse/python_dependencies.py -@@ -37,7 +37,7 @@ REQUIREMENTS = { - "unpaddedbase64>=1.1.0": ["unpaddedbase64>=1.1.0"], - "canonicaljson>=1.1.3": ["canonicaljson>=1.1.3"], - "signedjson>=1.0.0": ["signedjson>=1.0.0"], -- "pynacl>=1.2.1": ["nacl>=1.2.1", "nacl.bindings"], -+ "pynacl>=0.3.0": ["nacl>=0.3.0", "nacl.bindings"], - "service_identity>=16.0.0": ["service_identity>=16.0.0"], - "Twisted>=17.1.0": ["twisted>=17.1.0"], - "treq>=15.1": ["treq>=15.1"], diff --git a/debian/patches/bcrypt.patch b/debian/patches/bcrypt.patch deleted file mode 100644 index a962949920..0000000000 --- a/debian/patches/bcrypt.patch +++ /dev/null @@ -1,30 +0,0 @@ -Index: package-synapse-debian/synapse/handlers/auth.py -=================================================================== ---- package-synapse-debian.orig/synapse/handlers/auth.py -+++ package-synapse-debian/synapse/handlers/auth.py -@@ -921,10 +921,10 @@ class AuthHandler(BaseHandler): - # Normalise the Unicode in the password - pw = unicodedata.normalize("NFKC", password) - -- return bcrypt.checkpw( -+ return bcrypt.hashpw( - pw.encode('utf8') + self.hs.config.password_pepper.encode("utf8"), - stored_hash -- ) -+ ) == stored_hash - - if stored_hash: - if not isinstance(stored_hash, bytes): -Index: package-synapse-debian/synapse/python_dependencies.py -=================================================================== ---- package-synapse-debian.orig/synapse/python_dependencies.py -+++ package-synapse-debian/synapse/python_dependencies.py -@@ -49,7 +49,7 @@ REQUIREMENTS = { - "pyasn1>=0.1.9": ["pyasn1"], - "pyasn1-modules>=0.0.7": ["pyasn1_modules"], - "daemonize>=2.3.1": ["daemonize"], -- "bcrypt>=3.1.0": ["bcrypt>=3.1.0"], -+ "bcrypt": ["bcrypt"], - "pillow>=3.1.2": ["PIL"], - "sortedcontainers>=1.4.4": ["sortedcontainers"], - "psutil>=2.0.0": ["psutil>=2.0.0"], diff --git a/debian/patches/no_install_with_pip b/debian/patches/no_install_with_pip deleted file mode 100644 index 654656f596..0000000000 --- a/debian/patches/no_install_with_pip +++ /dev/null @@ -1,43 +0,0 @@ -Index: package-synapse-debian/synapse/app/__init__.py -=================================================================== ---- package-synapse-debian.orig/synapse/app/__init__.py -+++ package-synapse-debian/synapse/app/__init__.py -@@ -25,8 +25,8 @@ try: - except python_dependencies.MissingRequirementError as e: - message = "\n".join([ - "Missing Requirement: %s" % (str(e),), -- "To install run:", -- " pip install --upgrade --force \"%s\"" % (e.dependency,), -+ "To install, try:", -+ " sudo apt-get install python-%s" % (e.dependency,), - "", - ]) - sys.stderr.writelines(message) -Index: package-synapse-debian/synapse/config/jwt_config.py -=================================================================== ---- package-synapse-debian.orig/synapse/config/jwt_config.py -+++ package-synapse-debian/synapse/config/jwt_config.py -@@ -19,7 +19,7 @@ MISSING_JWT = ( - """Missing jwt library. This is required for jwt login. - - Install by running: -- pip install pyjwt -+ sudo apt-get install python-jwt - """ - ) - -Index: package-synapse-debian/synapse/config/repository.py -=================================================================== ---- package-synapse-debian.orig/synapse/config/repository.py -+++ package-synapse-debian/synapse/config/repository.py -@@ -27,9 +27,7 @@ MISSING_LXML = ( - """Missing lxml library. This is required for URL preview API. - - Install by running: -- pip install lxml -- -- Requires libxslt1-dev system package. -+ sudo apt-get install python-lxml - """ - ) - diff --git a/debian/patches/remove-webclient.patch b/debian/patches/remove-webclient.patch deleted file mode 100644 index bfd2fce576..0000000000 --- a/debian/patches/remove-webclient.patch +++ /dev/null @@ -1,31 +0,0 @@ -Index: package-synapse-debian/synapse/app/homeserver.py -=================================================================== ---- package-synapse-debian.orig/synapse/app/homeserver.py -+++ package-synapse-debian/synapse/app/homeserver.py -@@ -124,7 +124,7 @@ class SynapseHomeServer(HomeServer): - for res in listener_config["resources"]: - for name in res["names"]: - resources.update(self._configure_named_resource( -- name, res.get("compress", False), -+ config, name, res.get("compress", False), - )) - - additional_resources = listener_config.get("additional_resources", {}) -@@ -171,7 +171,7 @@ class SynapseHomeServer(HomeServer): - ) - logger.info("Synapse now listening on port %d", port) - -- def _configure_named_resource(self, name, compress=False): -+ def _configure_named_resource(self, config, name, compress=False): - """Build a resource map for a named resource - - Args: -@@ -235,7 +235,7 @@ class SynapseHomeServer(HomeServer): - if name in ["keys", "federation"]: - resources[SERVER_KEY_V2_PREFIX] = KeyApiV2Resource(self) - -- if name == "webclient": -+ if name == "webclient" and config.web_client: - resources[WEB_CLIENT_PREFIX] = build_resource_for_web_client(self) - - if name == "metrics" and self.get_config().enable_metrics: diff --git a/debian/patches/series b/debian/patches/series deleted file mode 100644 index ae46209e92..0000000000 --- a/debian/patches/series +++ /dev/null @@ -1,7 +0,0 @@ -0001-tox.patch -0002-change_instructions.patch -0004-webclient-instructions.patch -0006-Don-t-require-strict-nacl-0.3.0-requirement.patch -remove-webclient.patch -bcrypt.patch -no_install_with_pip diff --git a/debian/postinst b/debian/postinst deleted file mode 100755 index 0509acd0a4..0000000000 --- a/debian/postinst +++ /dev/null @@ -1,39 +0,0 @@ -#!/bin/sh -e - -. /usr/share/debconf/confmodule - -CONFIGFILE_SERVERNAME="/etc/matrix-synapse/conf.d/server_name.yaml" -CONFIGFILE_REPORTSTATS="/etc/matrix-synapse/conf.d/report_stats.yaml" -USER="matrix-synapse" - -case "$1" in - configure|reconfigure) - # Set server name in config file - mkdir -p "/etc/matrix-synapse/conf.d/" - db_get matrix-synapse/server-name - - if [ "$RET" ]; then - echo "server_name: $RET" > $CONFIGFILE_SERVERNAME - fi - - db_get matrix-synapse/report-stats - if [ "$RET" ]; then - echo "report_stats: $RET" > $CONFIGFILE_REPORTSTATS - fi - - if ! getent passwd $USER >/dev/null; then - adduser --quiet --system --no-create-home --home /var/lib/matrix-synapse $USER - fi - - for DIR in /var/lib/matrix-synapse /var/log/matrix-synapse /etc/matrix-synapse; do - if ! dpkg-statoverride --list --quiet $DIR >/dev/null; then - dpkg-statoverride --force --quiet --update --add $USER nogroup 0755 $DIR - fi - done - - ;; -esac - -#DEBHELPER# - -exit 0 diff --git a/debian/pydist-overrides b/debian/pydist-overrides deleted file mode 100644 index e2e34b7d30..0000000000 --- a/debian/pydist-overrides +++ /dev/null @@ -1,5 +0,0 @@ -matrix-angular-sdk -jinja2 -bleach -ldap3 -matrix-synapse-ldap3 diff --git a/debian/rules b/debian/rules index ad0e95c83d..05cbbdde08 100755 --- a/debian/rules +++ b/debian/rules @@ -1,18 +1,22 @@ #!/usr/bin/make -f +# +# Build Debian package using https://github.com/spotify/dh-virtualenv +# -# This file was automatically generated by stdeb 0.8.2 at -# Fri, 12 Jun 2015 14:32:03 +0100 -export PYBUILD_NAME=matrix-synapse -%: - dh $@ --with python2 --with systemd --buildsystem=pybuild --no-guessing-deps +override_dh_systemd_enable: + dh_systemd_enable --name=matrix-synapse -override_dh_auto_install: - python setup.py install --root=debian/matrix-synapse --install-layout=deb +override_dh_installinit: + dh_installinit --name=matrix-synapse -override_dh_auto_build: +override_dh_strip: -override_dh_installinit: - dh_installinit --no-start +override_dh_shlibdeps: -override_dh_auto_test: - PYTHONPATH=. trial tests +override_dh_virtualenv: + ./debian/build_virtualenv + +# We are restricted to compat level 9 (because xenial), so have to +# enable the systemd bits manually. +%: + dh $@ --with python-virtualenv --with systemd diff --git a/debian/source/format b/debian/source/format index 163aaf8d82..89ae9db8f8 100644 --- a/debian/source/format +++ b/debian/source/format @@ -1 +1 @@ -3.0 (quilt) +3.0 (native) diff --git a/debian/watch b/debian/watch deleted file mode 100644 index b1b10d4cd0..0000000000 --- a/debian/watch +++ /dev/null @@ -1,11 +0,0 @@ -# Example watch control file for uscan -# Rename this file to "watch" and then you can run the "uscan" command -# to check for upstream updates and more. -# See uscan(1) for format - -# Compulsory line, this is a version 3 file -version=3 - - -opts=filenamemangle=s/.+\/v?(\d\S*)\.tar\.gz/matrix-synapse-$1\.tar\.gz/,uversionmangle=s/-?rc/~rc/ \ - https://github.com/matrix-org/synapse/tags .*/v?(\d[^\s\-]*)\.tar\.gz debian uupdate diff --git a/docker/Dockerfile-dhvirtualenv b/docker/Dockerfile-dhvirtualenv new file mode 100644 index 0000000000..ea6b650af2 --- /dev/null +++ b/docker/Dockerfile-dhvirtualenv @@ -0,0 +1,35 @@ +# A dockerfile which builds a docker image for building a debian package for +# synapse. The distro to build for is passed as a docker build var. +# +# The default entrypoint expects the synapse source to be mounted as a +# (read-only) volume at /synapse/source, and an output directory at /debs. +# +# A pair of environment variables (TARGET_USERID and TARGET_GROUPID) can be +# passed to the docker container; if these are set, the build script will chown +# the build products accordingly, to avoid ending up with things owned by root +# in the host filesystem. + +# Get the distro we want to pull from as a dynamic build variable +ARG distro="" +FROM ${distro} + +# Install the build dependencies +RUN apt-get update -qq -o Acquire::Languages=none \ + && env DEBIAN_FRONTEND=noninteractive apt-get install \ + -yqq --no-install-recommends -o Dpkg::Options::=--force-unsafe-io \ + build-essential \ + debhelper \ + devscripts \ + dh-systemd \ + dh-virtualenv \ + equivs \ + lsb-release \ + python3-dev \ + python3-pip \ + python3-setuptools \ + python3-venv \ + sqlite3 \ + wget + +WORKDIR /synapse/source +ENTRYPOINT ["bash","/synapse/source/docker/build_debian.sh"] diff --git a/docker/build_debian.sh b/docker/build_debian.sh new file mode 100644 index 0000000000..cea5067fe9 --- /dev/null +++ b/docker/build_debian.sh @@ -0,0 +1,41 @@ +#!/bin/bash + +# The script to build the Debian package, as ran inside the Docker image. + +set -ex + +DIST=`lsb_release -c -s` + +# We need to build a newer dh_virtualenv on older OSes like Xenial. +if [ "$DIST" = 'xenial' ]; then + mkdir -p /tmp/dhvenv + cd /tmp/dhvenv + wget https://github.com/spotify/dh-virtualenv/archive/1.1.tar.gz + tar xvf 1.1.tar.gz + cd dh-virtualenv-1.1/ + env DEBIAN_FRONTEND=noninteractive mk-build-deps -ri -t "apt-get -yqq --no-install-recommends -o Dpkg::Options::=--force-unsafe-io" + dpkg-buildpackage -us -uc -b + cd /tmp/dhvenv + apt-get install -yqq ./dh-virtualenv_1.1-1_all.deb +fi + + +# we get a read-only copy of the source: make a writeable copy +cp -aT /synapse/source /synapse/build +cd /synapse/build + +# add an entry to the changelog for this distribution +dch -M -l "+$DIST" "build for $DIST" +dch -M -r "" --force-distribution --distribution "$DIST" + +dpkg-buildpackage -us -uc + +ls -l .. + +# copy the build results out, setting perms if necessary +shopt -s nullglob +for i in ../*.deb ../*.dsc ../*.tar.xz ../*.changes ../*.buildinfo; do + [ -z "$TARGET_USERID" ] || chown "$TARGET_USERID" "$i" + [ -z "$TARGET_GROUPID" ] || chgrp "$TARGET_GROUPID" "$i" + mv "$i" /debs +done diff --git a/docker/build_debian_packages.sh b/docker/build_debian_packages.sh new file mode 100755 index 0000000000..eafed4ac41 --- /dev/null +++ b/docker/build_debian_packages.sh @@ -0,0 +1,39 @@ +#!/bin/bash + +# Build the Debian packages using Docker images. +# +# This script builds the Docker images and then executes them sequentially, each +# one building a Debian package for the targeted operating system. It is +# designed to be a "single command" to produce all the images. +# +# By default, builds for all known distributions, but a list of distributions +# can be passed on the commandline for debugging. + +set -ex + +cd `dirname $0` + +if [ $# -lt 1 ]; then + DISTS=(debian:stretch debian:sid ubuntu:xenial ubuntu:bionic ubuntu:cosmic) +else + DISTS=("$@") +fi + +# Make the dir where the debs will live. +# +# Note that we deliberately put this outside the source tree, otherwise we tend +# to get source packages which are full of debs. (We could hack around that +# with more magic in the build_debian.sh script, but that doesn't solve the +# problem for natively-run dpkg-buildpakage). + +mkdir -p ../../debs + +# Build each OS image; +for i in "${DISTS[@]}"; do + TAG=$(echo ${i} | cut -d ":" -f 2) + docker build --tag dh-venv-builder:${TAG} --build-arg distro=${i} -f Dockerfile-dhvirtualenv . + docker run -it --rm --volume=$(pwd)/../\:/synapse/source:ro --volume=$(pwd)/../../debs:/debs \ + -e TARGET_USERID=$(id -u) \ + -e TARGET_GROUPID=$(id -g) \ + dh-venv-builder:${TAG} +done diff --git a/synapse/python_dependencies.py b/synapse/python_dependencies.py index 92422c6ffc..96cd154234 100644 --- a/synapse/python_dependencies.py +++ b/synapse/python_dependencies.py @@ -78,7 +78,7 @@ CONDITIONAL_REQUIREMENTS = { }, "postgres": { "psycopg2>=2.6": ["psycopg2"] - } + }, } diff --git a/synapse/storage/e2e_room_keys.py b/synapse/storage/e2e_room_keys.py index 16b7f005aa..45cebe61d1 100644 --- a/synapse/storage/e2e_room_keys.py +++ b/synapse/storage/e2e_room_keys.py @@ -182,7 +182,7 @@ class EndToEndRoomKeyStore(SQLBaseStore): keyvalues = { "user_id": user_id, - "version": version, + "version": int(version), } if room_id: keyvalues['room_id'] = room_id diff --git a/tox.ini b/tox.ini index 731094b5da..44371f211f 100644 --- a/tox.ini +++ b/tox.ini @@ -119,6 +119,7 @@ setenv = [testenv:packaging] +skip_install=True deps = check-manifest commands = -- cgit 1.4.1 From bdc44b99a2565f72012e189f2cf8364992448a18 Mon Sep 17 00:00:00 2001 From: Richard van der Hoff Date: Thu, 20 Dec 2018 16:11:21 +0000 Subject: buster is a thing --- docker/build_debian_packages.sh | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) (limited to 'docker') diff --git a/docker/build_debian_packages.sh b/docker/build_debian_packages.sh index eafed4ac41..08c68dd46a 100755 --- a/docker/build_debian_packages.sh +++ b/docker/build_debian_packages.sh @@ -14,7 +14,14 @@ set -ex cd `dirname $0` if [ $# -lt 1 ]; then - DISTS=(debian:stretch debian:sid ubuntu:xenial ubuntu:bionic ubuntu:cosmic) + DISTS=( + debian:stretch + debian:buster + debian:sid + ubuntu:xenial + ubuntu:bionic + ubuntu:cosmic + ) else DISTS=("$@") fi -- cgit 1.4.1 From 7134832c017fab1e4e6b90df9ad39dd618656427 Mon Sep 17 00:00:00 2001 From: Richard van der Hoff <1389908+richvdh@users.noreply.github.com> Date: Wed, 2 Jan 2019 07:17:39 +0000 Subject: Install the optional dependencies into the debian package (#4325) since #4298, the optional dependencies are no longer installed with a simple `pip install .`, which meant that they were not being included in the debian package. The easy fix to that is dh_virtualenv --extras, but that needs dh_virtualenv 1.1... --- changelog.d/4325.feature | 1 + debian/build_virtualenv | 3 ++- debian/control | 2 +- docker/Dockerfile-dhvirtualenv | 37 +++++++++++++++++++++++++++++++++---- docker/build_debian.sh | 14 -------------- 5 files changed, 37 insertions(+), 20 deletions(-) create mode 100644 changelog.d/4325.feature (limited to 'docker') diff --git a/changelog.d/4325.feature b/changelog.d/4325.feature new file mode 100644 index 0000000000..05ad70fe72 --- /dev/null +++ b/changelog.d/4325.feature @@ -0,0 +1 @@ +Synapse can now have its conditional/extra dependencies installed by pip. This functionality can be used by using `pip install matrix-synapse[feature]`, where feature is a comma separated list with the possible values "email.enable_notifs", "ldap3", "postgres", "saml2", "url_preview", and "test". If you want to install all optional dependencies, you can use "all" instead. diff --git a/debian/build_virtualenv b/debian/build_virtualenv index 61ffb13192..e9ef245476 100755 --- a/debian/build_virtualenv +++ b/debian/build_virtualenv @@ -33,7 +33,8 @@ dh_virtualenv \ --preinstall="lxml" \ --preinstall="mock" \ --extra-pip-arg="--no-cache-dir" \ - --extra-pip-arg="--compile" + --extra-pip-arg="--compile" \ + --extras="all" # we copy the tests to a temporary directory so that we can put them on the # PYTHONPATH without putting the uninstalled synapse on the pythonpath. diff --git a/debian/control b/debian/control index 552a81dcb0..9606a4c135 100644 --- a/debian/control +++ b/debian/control @@ -5,7 +5,7 @@ Maintainer: Synapse Packaging team Build-Depends: debhelper (>= 9), dh-systemd, - dh-virtualenv (>= 1.0), + dh-virtualenv (>= 1.1), lsb-release, python3-dev, python3, diff --git a/docker/Dockerfile-dhvirtualenv b/docker/Dockerfile-dhvirtualenv index ea6b650af2..ab28e49291 100644 --- a/docker/Dockerfile-dhvirtualenv +++ b/docker/Dockerfile-dhvirtualenv @@ -11,6 +11,35 @@ # Get the distro we want to pull from as a dynamic build variable ARG distro="" + +### +### Stage 0: build a dh-virtualenv +### +FROM ${distro} as builder + +RUN apt-get update -qq -o Acquire::Languages=none +RUN env DEBIAN_FRONTEND=noninteractive apt-get install \ + -yqq --no-install-recommends \ + build-essential \ + ca-certificates \ + devscripts \ + equivs \ + wget + +# fetch and unpack the package +RUN wget -q -O /dh-virtuenv-1.1.tar.gz https://github.com/spotify/dh-virtualenv/archive/1.1.tar.gz +RUN tar xvf /dh-virtuenv-1.1.tar.gz + +# install its build deps +RUN cd dh-virtualenv-1.1/ \ + && env DEBIAN_FRONTEND=noninteractive mk-build-deps -ri -t "apt-get -yqq --no-install-recommends" + +# build it +RUN cd dh-virtualenv-1.1 && dpkg-buildpackage -us -uc -b + +### +### Stage 1 +### FROM ${distro} # Install the build dependencies @@ -21,15 +50,15 @@ RUN apt-get update -qq -o Acquire::Languages=none \ debhelper \ devscripts \ dh-systemd \ - dh-virtualenv \ - equivs \ lsb-release \ python3-dev \ python3-pip \ python3-setuptools \ python3-venv \ - sqlite3 \ - wget + sqlite3 + +COPY --from=builder /dh-virtualenv_1.1-1_all.deb / +RUN apt-get install -yq /dh-virtualenv_1.1-1_all.deb WORKDIR /synapse/source ENTRYPOINT ["bash","/synapse/source/docker/build_debian.sh"] diff --git a/docker/build_debian.sh b/docker/build_debian.sh index cea5067fe9..6ed2b39898 100644 --- a/docker/build_debian.sh +++ b/docker/build_debian.sh @@ -6,20 +6,6 @@ set -ex DIST=`lsb_release -c -s` -# We need to build a newer dh_virtualenv on older OSes like Xenial. -if [ "$DIST" = 'xenial' ]; then - mkdir -p /tmp/dhvenv - cd /tmp/dhvenv - wget https://github.com/spotify/dh-virtualenv/archive/1.1.tar.gz - tar xvf 1.1.tar.gz - cd dh-virtualenv-1.1/ - env DEBIAN_FRONTEND=noninteractive mk-build-deps -ri -t "apt-get -yqq --no-install-recommends -o Dpkg::Options::=--force-unsafe-io" - dpkg-buildpackage -us -uc -b - cd /tmp/dhvenv - apt-get install -yqq ./dh-virtualenv_1.1-1_all.deb -fi - - # we get a read-only copy of the source: make a writeable copy cp -aT /synapse/source /synapse/build cd /synapse/build -- cgit 1.4.1 From 95fca1c7e95abe5c40e770c8834ee64bb62ad6d2 Mon Sep 17 00:00:00 2001 From: Richard van der Hoff Date: Wed, 9 Jan 2019 16:37:51 +0000 Subject: fix docker build to install optional deps --- docker/Dockerfile | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) (limited to 'docker') diff --git a/docker/Dockerfile b/docker/Dockerfile index db44c02a92..4b739e7d02 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -33,9 +33,7 @@ RUN pip install --prefix="/install" --no-warn-script-location \ COPY . /synapse RUN pip install --prefix="/install" --no-warn-script-location \ - lxml \ - psycopg2 \ - /synapse + /synapse[all] ### ### Stage 1: runtime -- cgit 1.4.1 From b5b868d41e143f4d7eb41aad69d13ac451605fc0 Mon Sep 17 00:00:00 2001 From: Richard van der Hoff Date: Sat, 12 Jan 2019 12:26:53 +0000 Subject: Rewrite build_debian_packages Rewrite this in python so that it can be run in parallel. --- docker/build_debian_packages.sh | 46 ------------ scripts-dev/build_debian_packages | 154 ++++++++++++++++++++++++++++++++++++++ 2 files changed, 154 insertions(+), 46 deletions(-) delete mode 100755 docker/build_debian_packages.sh create mode 100755 scripts-dev/build_debian_packages (limited to 'docker') diff --git a/docker/build_debian_packages.sh b/docker/build_debian_packages.sh deleted file mode 100755 index 08c68dd46a..0000000000 --- a/docker/build_debian_packages.sh +++ /dev/null @@ -1,46 +0,0 @@ -#!/bin/bash - -# Build the Debian packages using Docker images. -# -# This script builds the Docker images and then executes them sequentially, each -# one building a Debian package for the targeted operating system. It is -# designed to be a "single command" to produce all the images. -# -# By default, builds for all known distributions, but a list of distributions -# can be passed on the commandline for debugging. - -set -ex - -cd `dirname $0` - -if [ $# -lt 1 ]; then - DISTS=( - debian:stretch - debian:buster - debian:sid - ubuntu:xenial - ubuntu:bionic - ubuntu:cosmic - ) -else - DISTS=("$@") -fi - -# Make the dir where the debs will live. -# -# Note that we deliberately put this outside the source tree, otherwise we tend -# to get source packages which are full of debs. (We could hack around that -# with more magic in the build_debian.sh script, but that doesn't solve the -# problem for natively-run dpkg-buildpakage). - -mkdir -p ../../debs - -# Build each OS image; -for i in "${DISTS[@]}"; do - TAG=$(echo ${i} | cut -d ":" -f 2) - docker build --tag dh-venv-builder:${TAG} --build-arg distro=${i} -f Dockerfile-dhvirtualenv . - docker run -it --rm --volume=$(pwd)/../\:/synapse/source:ro --volume=$(pwd)/../../debs:/debs \ - -e TARGET_USERID=$(id -u) \ - -e TARGET_GROUPID=$(id -g) \ - dh-venv-builder:${TAG} -done diff --git a/scripts-dev/build_debian_packages b/scripts-dev/build_debian_packages new file mode 100755 index 0000000000..577d93e6f6 --- /dev/null +++ b/scripts-dev/build_debian_packages @@ -0,0 +1,154 @@ +#!/usr/bin/env python3 + +# Build the Debian packages using Docker images. +# +# This script builds the Docker images and then executes them sequentially, each +# one building a Debian package for the targeted operating system. It is +# designed to be a "single command" to produce all the images. +# +# By default, builds for all known distributions, but a list of distributions +# can be passed on the commandline for debugging. + +import argparse +from concurrent.futures import ThreadPoolExecutor +import os +import signal +import subprocess +import sys +import threading + +DISTS = ( + "debian:stretch", + "debian:buster", + "debian:sid", + "ubuntu:xenial", + "ubuntu:bionic", + "ubuntu:cosmic", +) + +DESC = '''\ +Builds .debs for synapse, using a Docker image for the build environment. + +By default, builds for all known distributions, but a list of distributions +can be passed on the commandline for debugging. +''' + + +class Builder(object): + def __init__(self, redirect_stdout=False): + self.redirect_stdout = redirect_stdout + self.active_containers = set() + self._lock = threading.Lock() + self._failed = False + + def run_build(self, dist): + """Build deb for a single distribution""" + + if self._failed: + print("not building %s due to earlier failure" % (dist, )) + raise Exception("failed") + + try: + self._inner_build(dist) + except Exception as e: + print("build of %s failed: %s" % (dist, e), file=sys.stderr) + self._failed = True + raise + + def _inner_build(self, dist): + projdir = os.path.dirname(os.path.dirname(os.path.realpath(__file__))) + os.chdir(projdir) + + tag = dist.split(":", 1)[1] + + # Make the dir where the debs will live. + # + # Note that we deliberately put this outside the source tree, otherwise + # we tend to get source packages which are full of debs. (We could hack + # around that with more magic in the build_debian.sh script, but that + # doesn't solve the problem for natively-run dpkg-buildpakage). + debsdir = os.path.join(projdir, '../debs') + os.makedirs(debsdir, exist_ok=True) + + if self.redirect_stdout: + logfile = os.path.join(debsdir, "%s.buildlog" % (tag, )) + print("building %s: directing output to %s" % (dist, logfile)) + stdout = open(logfile, "w") + else: + stdout = None + + # first build a docker image for the build environment + subprocess.check_call([ + "docker", "build", + "--tag", "dh-venv-builder:" + tag, + "--build-arg", "distro=" + dist, + "-f", "docker/Dockerfile-dhvirtualenv", + "docker", + ], stdout=stdout, stderr=subprocess.STDOUT) + + container_name = "synapse_build_" + tag + with self._lock: + self.active_containers.add(container_name) + + # then run the build itself + subprocess.check_call([ + "docker", "run", + "--rm", + "--name", container_name, + "--volume=" + projdir + ":/synapse/source:ro", + "--volume=" + debsdir + ":/debs", + "-e", "TARGET_USERID=%i" % (os.getuid(), ), + "-e", "TARGET_GROUPID=%i" % (os.getgid(), ), + "dh-venv-builder:" + tag, + ], stdout=stdout, stderr=subprocess.STDOUT) + + with self._lock: + self.active_containers.remove(container_name) + + if stdout is not None: + stdout.close() + print("Completed build of %s" % (dist, )) + + def kill_containers(self): + with self._lock: + active = list(self.active_containers) + + for c in active: + print("killing container %s" % (c,)) + subprocess.run([ + "docker", "kill", c, + ], stdout=subprocess.DEVNULL) + with self._lock: + self.active_containers.remove(c) + + +def run_builds(dists, jobs=1): + builder = Builder(redirect_stdout=(jobs > 1)) + + def sig(signum, _frame): + print("Caught SIGINT") + builder.kill_containers() + signal.signal(signal.SIGINT, sig) + + with ThreadPoolExecutor(max_workers=jobs) as e: + res = e.map(builder.run_build, dists) + + # make sure we consume the iterable so that exceptions are raised. + for r in res: + pass + + +if __name__ == '__main__': + parser = argparse.ArgumentParser( + description=DESC, + ) + parser.add_argument( + '-j', '--jobs', type=int, default=1, + help='specify the number of builds to run in parallel', + ) + parser.add_argument( + 'dist', nargs='*', default=DISTS, + help='a list of distributions to build for. Default: %(default)s', + ) + args = parser.parse_args() + run_builds(dists=args.dist, jobs=args.jobs) -- cgit 1.4.1 From 23b08135998e932d5d600941bd42389db0628a11 Mon Sep 17 00:00:00 2001 From: Amber Brown Date: Tue, 22 Jan 2019 21:58:50 +1100 Subject: Require ECDH key exchange & remove dh_params (#4429) * remove dh_params and set better cipher string --- README.rst | 2 +- changelog.d/4229.feature | 1 + debian/homeserver.yaml | 3 --- demo/demo.tls.dh | 9 --------- docker/conf/homeserver.yaml | 1 - synapse/config/tls.py | 40 --------------------------------------- synapse/crypto/context_factory.py | 6 ++++-- tests/config/test_generate.py | 1 - 8 files changed, 6 insertions(+), 57 deletions(-) create mode 100644 changelog.d/4229.feature delete mode 100644 demo/demo.tls.dh (limited to 'docker') diff --git a/README.rst b/README.rst index 8bff55e78e..05a3bb3751 100644 --- a/README.rst +++ b/README.rst @@ -220,7 +220,7 @@ is configured to use TLS with a self-signed certificate. If you would like to do initial test with a client without having to setup a reverse proxy, you can temporarly use another certificate. (Note that a self-signed certificate is fine for `Federation`_). You can do so by changing -``tls_certificate_path``, ``tls_private_key_path`` and ``tls_dh_params_path`` +``tls_certificate_path`` and ``tls_private_key_path`` in ``homeserver.yaml``; alternatively, you can use a reverse-proxy, but be sure to read `Using a reverse proxy with Synapse`_ when doing so. diff --git a/changelog.d/4229.feature b/changelog.d/4229.feature new file mode 100644 index 0000000000..0d1996c7e8 --- /dev/null +++ b/changelog.d/4229.feature @@ -0,0 +1 @@ +Synapse's cipher string has been updated to require ECDH key exchange. Configuring and generating dh_params is no longer required, and they will be ignored. diff --git a/debian/homeserver.yaml b/debian/homeserver.yaml index 188a2d5483..0bb2d22a95 100644 --- a/debian/homeserver.yaml +++ b/debian/homeserver.yaml @@ -9,9 +9,6 @@ tls_certificate_path: "/etc/matrix-synapse/homeserver.tls.crt" # PEM encoded private key for TLS tls_private_key_path: "/etc/matrix-synapse/homeserver.tls.key" -# PEM dh parameters for ephemeral keys -tls_dh_params_path: "/etc/matrix-synapse/homeserver.tls.dh" - # Don't bind to the https port no_tls: False diff --git a/demo/demo.tls.dh b/demo/demo.tls.dh deleted file mode 100644 index cbc58272a0..0000000000 --- a/demo/demo.tls.dh +++ /dev/null @@ -1,9 +0,0 @@ -2048-bit DH parameters taken from rfc3526 ------BEGIN DH PARAMETERS----- -MIIBCAKCAQEA///////////JD9qiIWjCNMTGYouA3BzRKQJOCIpnzHQCC76mOxOb -IlFKCHmONATd75UZs806QxswKwpt8l8UN0/hNW1tUcJF5IW1dmJefsb0TELppjft -awv/XLb0Brft7jhr+1qJn6WunyQRfEsf5kkoZlHs5Fs9wgB8uKFjvwWY2kg2HFXT -mmkWP6j9JM9fg2VdI9yjrZYcYvNWIIVSu57VKQdwlpZtZww1Tkq8mATxdGwIyhgh -fDKQXkYuNs474553LBgOhgObJ4Oi7Aeij7XFXfBvTFLJ3ivL9pVYFxg5lUl86pVq -5RXSJhiY+gUQFXKOWoqsqmj//////////wIBAg== ------END DH PARAMETERS----- diff --git a/docker/conf/homeserver.yaml b/docker/conf/homeserver.yaml index c2b8576a32..529118d184 100644 --- a/docker/conf/homeserver.yaml +++ b/docker/conf/homeserver.yaml @@ -4,7 +4,6 @@ tls_certificate_path: "/data/{{ SYNAPSE_SERVER_NAME }}.tls.crt" tls_private_key_path: "/data/{{ SYNAPSE_SERVER_NAME }}.tls.key" -tls_dh_params_path: "/data/{{ SYNAPSE_SERVER_NAME }}.tls.dh" no_tls: {{ "True" if SYNAPSE_NO_TLS else "False" }} tls_fingerprints: [] diff --git a/synapse/config/tls.py b/synapse/config/tls.py index fef1ea99cb..bb8952c672 100644 --- a/synapse/config/tls.py +++ b/synapse/config/tls.py @@ -14,7 +14,6 @@ # limitations under the License. import os -import subprocess from hashlib import sha256 from unpaddedbase64 import encode_base64 @@ -23,8 +22,6 @@ from OpenSSL import crypto from ._base import Config -GENERATE_DH_PARAMS = False - class TlsConfig(Config): def read_config(self, config): @@ -42,10 +39,6 @@ class TlsConfig(Config): config.get("tls_private_key_path") ) - self.tls_dh_params_path = self.check_file( - config.get("tls_dh_params_path"), "tls_dh_params" - ) - self.tls_fingerprints = config["tls_fingerprints"] # Check that our own certificate is included in the list of fingerprints @@ -72,7 +65,6 @@ class TlsConfig(Config): tls_certificate_path = base_key_name + ".tls.crt" tls_private_key_path = base_key_name + ".tls.key" - tls_dh_params_path = base_key_name + ".tls.dh" return """\ # PEM encoded X509 certificate for TLS. @@ -85,9 +77,6 @@ class TlsConfig(Config): # PEM encoded private key for TLS tls_private_key_path: "%(tls_private_key_path)s" - # PEM dh parameters for ephemeral keys - tls_dh_params_path: "%(tls_dh_params_path)s" - # Don't bind to the https port no_tls: False @@ -131,7 +120,6 @@ class TlsConfig(Config): def generate_files(self, config): tls_certificate_path = config["tls_certificate_path"] tls_private_key_path = config["tls_private_key_path"] - tls_dh_params_path = config["tls_dh_params_path"] if not self.path_exists(tls_private_key_path): with open(tls_private_key_path, "wb") as private_key_file: @@ -165,31 +153,3 @@ class TlsConfig(Config): cert_pem = crypto.dump_certificate(crypto.FILETYPE_PEM, cert) certificate_file.write(cert_pem) - - if not self.path_exists(tls_dh_params_path): - if GENERATE_DH_PARAMS: - subprocess.check_call([ - "openssl", "dhparam", - "-outform", "PEM", - "-out", tls_dh_params_path, - "2048" - ]) - else: - with open(tls_dh_params_path, "w") as dh_params_file: - dh_params_file.write( - "2048-bit DH parameters taken from rfc3526\n" - "-----BEGIN DH PARAMETERS-----\n" - "MIIBCAKCAQEA///////////JD9qiIWjC" - "NMTGYouA3BzRKQJOCIpnzHQCC76mOxOb\n" - "IlFKCHmONATd75UZs806QxswKwpt8l8U" - "N0/hNW1tUcJF5IW1dmJefsb0TELppjft\n" - "awv/XLb0Brft7jhr+1qJn6WunyQRfEsf" - "5kkoZlHs5Fs9wgB8uKFjvwWY2kg2HFXT\n" - "mmkWP6j9JM9fg2VdI9yjrZYcYvNWIIVS" - "u57VKQdwlpZtZww1Tkq8mATxdGwIyhgh\n" - "fDKQXkYuNs474553LBgOhgObJ4Oi7Aei" - "j7XFXfBvTFLJ3ivL9pVYFxg5lUl86pVq\n" - "5RXSJhiY+gUQFXKOWoqsqmj/////////" - "/wIBAg==\n" - "-----END DH PARAMETERS-----\n" - ) diff --git a/synapse/crypto/context_factory.py b/synapse/crypto/context_factory.py index 02b76dfcfb..6ba3eca7b2 100644 --- a/synapse/crypto/context_factory.py +++ b/synapse/crypto/context_factory.py @@ -46,8 +46,10 @@ class ServerContextFactory(ContextFactory): if not config.no_tls: context.use_privatekey(config.tls_private_key) - context.load_tmp_dh(config.tls_dh_params_path) - context.set_cipher_list("!ADH:HIGH+kEDH:!AECDH:HIGH+kEECDH") + # https://hynek.me/articles/hardening-your-web-servers-ssl-ciphers/ + context.set_cipher_list( + "ECDH+AESGCM:ECDH+CHACHA20:ECDH+AES256:ECDH+AES128:!aNULL:!SHA1" + ) def getContext(self): return self._context diff --git a/tests/config/test_generate.py b/tests/config/test_generate.py index 0c23068bcf..b5ad99348d 100644 --- a/tests/config/test_generate.py +++ b/tests/config/test_generate.py @@ -51,7 +51,6 @@ class ConfigGenerationTestCase(unittest.TestCase): "lemurs.win.log.config", "lemurs.win.signing.key", "lemurs.win.tls.crt", - "lemurs.win.tls.dh", "lemurs.win.tls.key", ] ), -- cgit 1.4.1 From 3ef71a6ea057c7ad8b87933f06861e9973660b51 Mon Sep 17 00:00:00 2001 From: Richard van der Hoff <1389908+richvdh@users.noreply.github.com> Date: Tue, 5 Feb 2019 11:44:40 +0000 Subject: Docker: only copy what we need to the build image (#4562) There are two reasons this is a good thing: * first, it means that you don't end up with stuff kicking around your working copy ending up in the build image by mistake (which can upset the pip install process) * second: it means that the docker image cache is more effective, and we can reuse docker images when iterating on the docker stuff. --- changelog.d/4562.misc | 1 + docker/Dockerfile | 5 ++++- 2 files changed, 5 insertions(+), 1 deletion(-) create mode 100644 changelog.d/4562.misc (limited to 'docker') diff --git a/changelog.d/4562.misc b/changelog.d/4562.misc new file mode 100644 index 0000000000..f7185fa768 --- /dev/null +++ b/changelog.d/4562.misc @@ -0,0 +1 @@ +Docker: only copy what we need to the build image diff --git a/docker/Dockerfile b/docker/Dockerfile index 4b739e7d02..d212334844 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -31,7 +31,10 @@ RUN pip install --prefix="/install" --no-warn-script-location \ # now install synapse and all of the python deps to /install. -COPY . /synapse +COPY synapse /synapse/synapse/ +COPY scripts /synapse/scripts/ +COPY MANIFEST.in README.rst setup.py synctl /synapse/ + RUN pip install --prefix="/install" --no-warn-script-location \ /synapse[all] -- cgit 1.4.1 From 40b35fb87516f461ae562b247ab13a80f57beede Mon Sep 17 00:00:00 2001 From: Richard van der Hoff <1389908+richvdh@users.noreply.github.com> Date: Tue, 5 Feb 2019 13:42:21 +0000 Subject: Enable ACME support in the docker image (#4566) Also: * Fix wrapping in docker readme * Clean up some docs on the docker image * a workaround for #4554 --- changelog.d/4566.feature | 1 + docker/Dockerfile | 15 +++++++- docker/README.md | 88 +++++++++++++++++++++++++++------------------ docker/conf/dummy.tls.crt | 17 +++++++++ docker/conf/homeserver.yaml | 18 ++++++++-- docker/start.py | 18 +++++++--- 6 files changed, 116 insertions(+), 41 deletions(-) create mode 100644 changelog.d/4566.feature create mode 100644 docker/conf/dummy.tls.crt (limited to 'docker') diff --git a/changelog.d/4566.feature b/changelog.d/4566.feature new file mode 100644 index 0000000000..11fc07476e --- /dev/null +++ b/changelog.d/4566.feature @@ -0,0 +1 @@ +enable ACME support in the docker image diff --git a/docker/Dockerfile b/docker/Dockerfile index d212334844..c35da67a2a 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -1,3 +1,16 @@ +# Dockerfile to build the matrixdotorg/synapse docker images. +# +# To build the image, run `docker build` command from the root of the +# synapse repository: +# +# docker build -f docker/Dockerfile . +# +# There is an optional PYTHON_VERSION build argument which sets the +# version of python to build against: for example: +# +# docker build -f docker/Dockerfile --build-arg PYTHON_VERSION=3.6 . +# + ARG PYTHON_VERSION=2 ### @@ -59,6 +72,6 @@ COPY ./docker/conf /conf VOLUME ["/data"] -EXPOSE 8008/tcp 8448/tcp +EXPOSE 8008/tcp 8009/tcp 8448/tcp ENTRYPOINT ["/start.py"] diff --git a/docker/README.md b/docker/README.md index 3c00d1e948..3faedf629f 100644 --- a/docker/README.md +++ b/docker/README.md @@ -1,22 +1,21 @@ # Synapse Docker -This Docker image will run Synapse as a single process. It does not provide a database -server or a TURN server, you should run these separately. +This Docker image will run Synapse as a single process. By default it uses a +sqlite database; for production use you should connect it to a separate +postgres database. -## Run +The image also does *not* provide a TURN server. -We do not currently offer a `latest` image, as this has somewhat undefined semantics. -We instead release only tagged versions so upgrading between releases is entirely -within your control. +## Run ### Using docker-compose (easier) -This image is designed to run either with an automatically generated configuration -file or with a custom configuration that requires manual editing. +This image is designed to run either with an automatically generated +configuration file or with a custom configuration that requires manual editing. An easy way to make use of this image is via docker-compose. See the -[contrib/docker](../contrib/docker) -section of the synapse project for examples. +[contrib/docker](../contrib/docker) section of the synapse project for +examples. ### Without Compose (harder) @@ -32,7 +31,7 @@ docker run \ -v ${DATA_PATH}:/data \ -e SYNAPSE_SERVER_NAME=my.matrix.host \ -e SYNAPSE_REPORT_STATS=yes \ - docker.io/matrixdotorg/synapse:latest + matrixdotorg/synapse:latest ``` ## Volumes @@ -53,6 +52,28 @@ In order to setup an application service, simply create an ``appservices`` directory in the data volume and write the application service Yaml configuration file there. Multiple application services are supported. +## TLS certificates + +Synapse requires a valid TLS certificate. You can do one of the following: + + * Provide your own certificate and key (as + `${DATA_PATH}/${SYNAPSE_SERVER_NAME}.crt` and + `${DATA_PATH}/${SYNAPSE_SERVER_NAME}.key`, or elsewhere by providing an + entire config as `${SYNAPSE_CONFIG_PATH}`). + + * Use a reverse proxy to terminate incoming TLS, and forward the plain http + traffic to port 8008 in the container. In this case you should set `-e + SYNAPSE_NO_TLS=1`. + + * Use the ACME (Let's Encrypt) support built into Synapse. This requires + `${SYNAPSE_SERVER_NAME}` port 80 to be forwarded to port 8009 in the + container, for example with `-p 80:8009`. To enable it in the docker + container, set `-e SYNAPSE_ACME=1`. + +If you don't do any of these, Synapse will fail to start with an error similar to: + + synapse.config._base.ConfigError: Error accessing file '/data/.tls.crt' (config for tls_certificate): No such file or directory + ## Environment Unless you specify a custom path for the configuration file, a very generic @@ -71,7 +92,7 @@ then customize it manually. No other environment variable is required. Otherwise, a dynamic configuration file will be used. The following environment variables are available for configuration: -* ``SYNAPSE_SERVER_NAME`` (mandatory), the current server public hostname. +* ``SYNAPSE_SERVER_NAME`` (mandatory), the server public hostname. * ``SYNAPSE_REPORT_STATS``, (mandatory, ``yes`` or ``no``), enable anonymous statistics reporting back to the Matrix project which helps us to get funding. * ``SYNAPSE_NO_TLS``, set this variable to disable TLS in Synapse (use this if @@ -80,7 +101,6 @@ variables are available for configuration: the Synapse instance. * ``SYNAPSE_ALLOW_GUEST``, set this variable to allow guest joining this server. * ``SYNAPSE_EVENT_CACHE_SIZE``, the event cache size [default `10K`]. -* ``SYNAPSE_CACHE_FACTOR``, the cache factor [default `0.5`]. * ``SYNAPSE_RECAPTCHA_PUBLIC_KEY``, set this variable to the recaptcha public key in order to enable recaptcha upon registration. * ``SYNAPSE_RECAPTCHA_PRIVATE_KEY``, set this variable to the recaptcha private @@ -88,7 +108,9 @@ variables are available for configuration: * ``SYNAPSE_TURN_URIS``, set this variable to the coma-separated list of TURN uris to enable TURN for this homeserver. * ``SYNAPSE_TURN_SECRET``, set this to the TURN shared secret if required. -* ``SYNAPSE_MAX_UPLOAD_SIZE``, set this variable to change the max upload size [default `10M`]. +* ``SYNAPSE_MAX_UPLOAD_SIZE``, set this variable to change the max upload size + [default `10M`]. +* ``SYNAPSE_ACME``: set this to enable the ACME certificate renewal support. Shared secrets, that will be initialized to random values if not set: @@ -99,27 +121,25 @@ Shared secrets, that will be initialized to random values if not set: Database specific values (will use SQLite if not set): -* `POSTGRES_DB` - The database name for the synapse postgres database. [default: `synapse`] -* `POSTGRES_HOST` - The host of the postgres database if you wish to use postgresql instead of sqlite3. [default: `db` which is useful when using a container on the same docker network in a compose file where the postgres service is called `db`] -* `POSTGRES_PASSWORD` - The password for the synapse postgres database. **If this is set then postgres will be used instead of sqlite3.** [default: none] **NOTE**: You are highly encouraged to use postgresql! Please use the compose file to make it easier to deploy. -* `POSTGRES_USER` - The user for the synapse postgres database. [default: `matrix`] +* `POSTGRES_DB` - The database name for the synapse postgres + database. [default: `synapse`] +* `POSTGRES_HOST` - The host of the postgres database if you wish to use + postgresql instead of sqlite3. [default: `db` which is useful when using a + container on the same docker network in a compose file where the postgres + service is called `db`] +* `POSTGRES_PASSWORD` - The password for the synapse postgres database. **If + this is set then postgres will be used instead of sqlite3.** [default: none] + **NOTE**: You are highly encouraged to use postgresql! Please use the compose + file to make it easier to deploy. +* `POSTGRES_USER` - The user for the synapse postgres database. [default: + `matrix`] Mail server specific values (will not send emails if not set): * ``SYNAPSE_SMTP_HOST``, hostname to the mail server. -* ``SYNAPSE_SMTP_PORT``, TCP port for accessing the mail server [default ``25``]. -* ``SYNAPSE_SMTP_USER``, username for authenticating against the mail server if any. -* ``SYNAPSE_SMTP_PASSWORD``, password for authenticating against the mail server if any. - -## Build - -Build the docker image with the `docker build` command from the root of the synapse repository. - -``` -docker build -t docker.io/matrixdotorg/synapse . -f docker/Dockerfile -``` - -The `-t` option sets the image tag. Official images are tagged `matrixdotorg/synapse:` where `` is the same as the release tag in the synapse git repository. - -You may have a local Python wheel cache available, in which case copy the relevant -packages in the ``cache/`` directory at the root of the project. +* ``SYNAPSE_SMTP_PORT``, TCP port for accessing the mail server [default + ``25``]. +* ``SYNAPSE_SMTP_USER``, username for authenticating against the mail server if + any. +* ``SYNAPSE_SMTP_PASSWORD``, password for authenticating against the mail + server if any. diff --git a/docker/conf/dummy.tls.crt b/docker/conf/dummy.tls.crt new file mode 100644 index 0000000000..8e3b1a9aaa --- /dev/null +++ b/docker/conf/dummy.tls.crt @@ -0,0 +1,17 @@ +-----BEGIN CERTIFICATE----- +MIICnTCCAYUCAgPoMA0GCSqGSIb3DQEBCwUAMBQxEjAQBgNVBAMMCWxvY2FsaG9z +dDAeFw0xOTAxMTUwMDQxNTBaFw0yOTAxMTIwMDQxNTBaMBQxEjAQBgNVBAMMCWxv +Y2FsaG9zdDCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEBAMKqm81/8j5d +R1s7VZ8ueg12gJrPVCCAOkp0UnuC/ZlXhN0HTvnhQ+B0IlSgB4CcQZyf4jnA6o4M +rwSc7VX0MPE9x/idoA0g/0WoC6tsxugOrvbzCw8Tv+fnXglm6uVc7aFPfx69wU3q +lUHGD/8jtEoHxmCG177Pt2lHAfiVLBAyMQGtETzxt/yAfkloaybe316qoljgK5WK +cokdAt9G84EEqxNeEnx5FG3Vc100bAqJS4GvQlFgtF9KFEqZKEyB1yKBpPMDfPIS +V9hIV0gswSmYI8dpyBlGf5lPElY68ZGABmOQgr0RI5qHK/h28OpFPE0q3v4AMHgZ +I36wii4NrAUCAwEAATANBgkqhkiG9w0BAQsFAAOCAQEAfD8kcpZ+dn08xh1qtKtp +X+/YNZaOBIeVdlCzfoZKNblSFAFD/jCfObNJYvZMUQ8NX2UtEJp1lTA6m7ltSsdY +gpC2k1VD8iN+ooXklJmL0kxc7UUqho8I0l9vn35h+lhLF0ihT6XfZVi/lDHWl+4G +rG+v9oxvCSCWrNWLearSlFPtQQ8xPtOE0nLwfXtOI/H/2kOuC38ihaIWM4jjbWXK +E/ksgUfuDv0mFiwf1YdBF5/M3/qOowqzU8HgMJ3WoT/9Po5Ya1pWc+3BcxxytUDf +XdMu0tWHKX84tZxLcR1nZHzluyvFFM8xNtLi9xV0Z7WbfT76V0C/ulEOybGInYsv +nQ== +-----END CERTIFICATE----- diff --git a/docker/conf/homeserver.yaml b/docker/conf/homeserver.yaml index 529118d184..f07d5c1001 100644 --- a/docker/conf/homeserver.yaml +++ b/docker/conf/homeserver.yaml @@ -2,10 +2,24 @@ ## TLS ## +{% if SYNAPSE_NO_TLS %} +no_tls: True + +# workaround for https://github.com/matrix-org/synapse/issues/4554 +tls_certificate_path: "/conf/dummy.tls.crt" + +{% else %} + tls_certificate_path: "/data/{{ SYNAPSE_SERVER_NAME }}.tls.crt" tls_private_key_path: "/data/{{ SYNAPSE_SERVER_NAME }}.tls.key" -no_tls: {{ "True" if SYNAPSE_NO_TLS else "False" }} -tls_fingerprints: [] + +{% if SYNAPSE_ACME %} +acme: + enabled: true + port: 8009 +{% endif %} + +{% endif %} ## Server ## diff --git a/docker/start.py b/docker/start.py index 346df8c87f..941d9996a8 100755 --- a/docker/start.py +++ b/docker/start.py @@ -47,9 +47,8 @@ if mode == "generate": # In normal mode, generate missing keys if any, then run synapse else: - # Parse the configuration file if "SYNAPSE_CONFIG_PATH" in environ: - args += ["--config-path", environ["SYNAPSE_CONFIG_PATH"]] + config_path = environ["SYNAPSE_CONFIG_PATH"] else: check_arguments(environ, ("SYNAPSE_SERVER_NAME", "SYNAPSE_REPORT_STATS")) generate_secrets(environ, { @@ -58,10 +57,21 @@ else: }) environ["SYNAPSE_APPSERVICES"] = glob.glob("/data/appservices/*.yaml") if not os.path.exists("/compiled"): os.mkdir("/compiled") - convert("/conf/homeserver.yaml", "/compiled/homeserver.yaml", environ) + + config_path = "/compiled/homeserver.yaml" + + convert("/conf/homeserver.yaml", config_path, environ) convert("/conf/log.config", "/compiled/log.config", environ) subprocess.check_output(["chown", "-R", ownership, "/data"]) - args += ["--config-path", "/compiled/homeserver.yaml"] + + + args += [ + "--config-path", config_path, + + # tell synapse to put any generated keys in /data rather than /compiled + "--keys-directory", "/data", + ] + # Generate missing keys and start synapse subprocess.check_output(args + ["--generate-keys"]) os.execv("/sbin/su-exec", ["su-exec", ownership] + args) -- cgit 1.4.1 From 91f8cd3307fee502c6bfb064f825952b05c4617b Mon Sep 17 00:00:00 2001 From: Richard van der Hoff Date: Mon, 11 Feb 2019 22:15:08 +0000 Subject: Remove redundant entries from docker config * no_tls is now redundant (#4613) * we don't need a dummy cert any more (#4618) --- changelog.d/4619.misc | 1 + docker/conf/dummy.tls.crt | 17 ----------------- docker/conf/homeserver.yaml | 8 +------- 3 files changed, 2 insertions(+), 24 deletions(-) create mode 100644 changelog.d/4619.misc delete mode 100644 docker/conf/dummy.tls.crt (limited to 'docker') diff --git a/changelog.d/4619.misc b/changelog.d/4619.misc new file mode 100644 index 0000000000..886fedf198 --- /dev/null +++ b/changelog.d/4619.misc @@ -0,0 +1 @@ +Remove redundant entries from docker config diff --git a/docker/conf/dummy.tls.crt b/docker/conf/dummy.tls.crt deleted file mode 100644 index 8e3b1a9aaa..0000000000 --- a/docker/conf/dummy.tls.crt +++ /dev/null @@ -1,17 +0,0 @@ ------BEGIN CERTIFICATE----- -MIICnTCCAYUCAgPoMA0GCSqGSIb3DQEBCwUAMBQxEjAQBgNVBAMMCWxvY2FsaG9z -dDAeFw0xOTAxMTUwMDQxNTBaFw0yOTAxMTIwMDQxNTBaMBQxEjAQBgNVBAMMCWxv -Y2FsaG9zdDCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEBAMKqm81/8j5d -R1s7VZ8ueg12gJrPVCCAOkp0UnuC/ZlXhN0HTvnhQ+B0IlSgB4CcQZyf4jnA6o4M -rwSc7VX0MPE9x/idoA0g/0WoC6tsxugOrvbzCw8Tv+fnXglm6uVc7aFPfx69wU3q -lUHGD/8jtEoHxmCG177Pt2lHAfiVLBAyMQGtETzxt/yAfkloaybe316qoljgK5WK -cokdAt9G84EEqxNeEnx5FG3Vc100bAqJS4GvQlFgtF9KFEqZKEyB1yKBpPMDfPIS -V9hIV0gswSmYI8dpyBlGf5lPElY68ZGABmOQgr0RI5qHK/h28OpFPE0q3v4AMHgZ -I36wii4NrAUCAwEAATANBgkqhkiG9w0BAQsFAAOCAQEAfD8kcpZ+dn08xh1qtKtp -X+/YNZaOBIeVdlCzfoZKNblSFAFD/jCfObNJYvZMUQ8NX2UtEJp1lTA6m7ltSsdY -gpC2k1VD8iN+ooXklJmL0kxc7UUqho8I0l9vn35h+lhLF0ihT6XfZVi/lDHWl+4G -rG+v9oxvCSCWrNWLearSlFPtQQ8xPtOE0nLwfXtOI/H/2kOuC38ihaIWM4jjbWXK -E/ksgUfuDv0mFiwf1YdBF5/M3/qOowqzU8HgMJ3WoT/9Po5Ya1pWc+3BcxxytUDf -XdMu0tWHKX84tZxLcR1nZHzluyvFFM8xNtLi9xV0Z7WbfT76V0C/ulEOybGInYsv -nQ== ------END CERTIFICATE----- diff --git a/docker/conf/homeserver.yaml b/docker/conf/homeserver.yaml index f07d5c1001..babd5bef9e 100644 --- a/docker/conf/homeserver.yaml +++ b/docker/conf/homeserver.yaml @@ -2,13 +2,7 @@ ## TLS ## -{% if SYNAPSE_NO_TLS %} -no_tls: True - -# workaround for https://github.com/matrix-org/synapse/issues/4554 -tls_certificate_path: "/conf/dummy.tls.crt" - -{% else %} +{% if not SYNAPSE_NO_TLS %} tls_certificate_path: "/data/{{ SYNAPSE_SERVER_NAME }}.tls.crt" tls_private_key_path: "/data/{{ SYNAPSE_SERVER_NAME }}.tls.key" -- cgit 1.4.1 From 649fe1c2be69676ffd73c9ad1807ea0bc747b003 Mon Sep 17 00:00:00 2001 From: Richard van der Hoff Date: Thu, 14 Feb 2019 17:29:40 +0000 Subject: Fix debian build dockerfile Make sure it refreshes the apt cache before trying to install stuff --- docker/Dockerfile-dhvirtualenv | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) (limited to 'docker') diff --git a/docker/Dockerfile-dhvirtualenv b/docker/Dockerfile-dhvirtualenv index ab28e49291..224c92352d 100644 --- a/docker/Dockerfile-dhvirtualenv +++ b/docker/Dockerfile-dhvirtualenv @@ -58,7 +58,11 @@ RUN apt-get update -qq -o Acquire::Languages=none \ sqlite3 COPY --from=builder /dh-virtualenv_1.1-1_all.deb / -RUN apt-get install -yq /dh-virtualenv_1.1-1_all.deb + +# install dhvirtualenv. Update the apt cache again first, in case we got a +# cached cache from docker the first time. +RUN apt-get update -qq -o Acquire::Languages=none \ + && apt-get install -yq /dh-virtualenv_1.1-1_all.deb WORKDIR /synapse/source ENTRYPOINT ["bash","/synapse/source/docker/build_debian.sh"] -- cgit 1.4.1