From a137f4eac028fefda0787a168dd8853653155bc3 Mon Sep 17 00:00:00 2001 From: Silke Hofstra Date: Tue, 16 Apr 2019 12:41:17 +0200 Subject: Add systemd-python to optional dependencies (#4339) Using systemd-python allows for logging to the systemd journal, as is documented in: `synapse/contrib/systemd/log_config.yaml`. Signed-off-by: Silke Hofstra --- docker/Dockerfile-dhvirtualenv | 2 ++ 1 file changed, 2 insertions(+) (limited to 'docker') diff --git a/docker/Dockerfile-dhvirtualenv b/docker/Dockerfile-dhvirtualenv index 224c92352d..9c4c9a5d80 100644 --- a/docker/Dockerfile-dhvirtualenv +++ b/docker/Dockerfile-dhvirtualenv @@ -50,7 +50,9 @@ RUN apt-get update -qq -o Acquire::Languages=none \ debhelper \ devscripts \ dh-systemd \ + libsystemd-dev \ lsb-release \ + pkg-config \ python3-dev \ python3-pip \ python3-setuptools \ -- cgit 1.5.1 From 4a9a118a94748ce287741f207f2077c6595ff178 Mon Sep 17 00:00:00 2001 From: *=0=1=4=* <21157384+f35f0ef9d0e827dae86552d3899f78fc@users.noreply.github.com> Date: Thu, 25 Apr 2019 08:47:22 -0500 Subject: Fix handling of SYNAPSE_NO_TLS in docker image (#5005) --- changelog.d/5005.misc | 1 + docker/README.md | 5 +++-- docker/start.py | 12 ++++++++++++ 3 files changed, 16 insertions(+), 2 deletions(-) create mode 100644 changelog.d/5005.misc (limited to 'docker') diff --git a/changelog.d/5005.misc b/changelog.d/5005.misc new file mode 100644 index 0000000000..f74147b352 --- /dev/null +++ b/changelog.d/5005.misc @@ -0,0 +1 @@ +Convert SYNAPSE_NO_TLS Docker variable to boolean for user friendliness. Contributed by Gabriel Eckerson. diff --git a/docker/README.md b/docker/README.md index b48d74e09c..b27a692d5b 100644 --- a/docker/README.md +++ b/docker/README.md @@ -102,8 +102,9 @@ when ``SYNAPSE_CONFIG_PATH`` is not set. * ``SYNAPSE_SERVER_NAME`` (mandatory), the server public hostname. * ``SYNAPSE_REPORT_STATS``, (mandatory, ``yes`` or ``no``), enable anonymous statistics reporting back to the Matrix project which helps us to get funding. -* ``SYNAPSE_NO_TLS``, set this variable to disable TLS in Synapse (use this if - you run your own TLS-capable reverse proxy). +* `SYNAPSE_NO_TLS`, (accepts `true`, `false`, `on`, `off`, `1`, `0`, `yes`, `no`]): disable + TLS in Synapse (use this if you run your own TLS-capable reverse proxy). Defaults + to `false` (ie, TLS is enabled by default). * ``SYNAPSE_ENABLE_REGISTRATION``, set this variable to enable registration on the Synapse instance. * ``SYNAPSE_ALLOW_GUEST``, set this variable to allow guest joining this server. diff --git a/docker/start.py b/docker/start.py index 941d9996a8..2da555272a 100755 --- a/docker/start.py +++ b/docker/start.py @@ -59,6 +59,18 @@ else: if not os.path.exists("/compiled"): os.mkdir("/compiled") config_path = "/compiled/homeserver.yaml" + + # Convert SYNAPSE_NO_TLS to boolean if exists + if "SYNAPSE_NO_TLS" in environ: + tlsanswerstring = str.lower(environ["SYNAPSE_NO_TLS"]) + if tlsanswerstring in ("true", "on", "1", "yes"): + environ["SYNAPSE_NO_TLS"] = True + else: + if tlsanswerstring in ("false", "off", "0", "no"): + environ["SYNAPSE_NO_TLS"] = False + else: + print("Environment variable \"SYNAPSE_NO_TLS\" found but value \"" + tlsanswerstring + "\" unrecognized; exiting.") + sys.exit(2) convert("/conf/homeserver.yaml", config_path, environ) convert("/conf/log.config", "/compiled/log.config", environ) -- cgit 1.5.1 From ecc09673156c340e390c9c8a19af80133622fd4f Mon Sep 17 00:00:00 2001 From: Richard van der Hoff Date: Fri, 3 May 2019 18:30:35 +0100 Subject: Debian: we now need libpq-dev. psycopg 2.8 is now out, which means that the C library gets built from source, so we now need libpq-dev when building. Turns out the need for this package is already documented in docs/postgres.rst. --- docker/Dockerfile-dhvirtualenv | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) (limited to 'docker') diff --git a/docker/Dockerfile-dhvirtualenv b/docker/Dockerfile-dhvirtualenv index 224c92352d..3de032cf8c 100644 --- a/docker/Dockerfile-dhvirtualenv +++ b/docker/Dockerfile-dhvirtualenv @@ -55,7 +55,8 @@ RUN apt-get update -qq -o Acquire::Languages=none \ python3-pip \ python3-setuptools \ python3-venv \ - sqlite3 + sqlite3 \ + libpq-dev COPY --from=builder /dh-virtualenv_1.1-1_all.deb / -- cgit 1.5.1 From 2dfbeea66f65f23582e045821ffd0dfda1dbab94 Mon Sep 17 00:00:00 2001 From: Steffen <33749463+YellowGarbageBag@users.noreply.github.com> Date: Wed, 22 May 2019 13:53:16 +0200 Subject: Update README.md (#5222) Add missing backslash --- docker/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'docker') diff --git a/docker/README.md b/docker/README.md index b27a692d5b..df5d0151e2 100644 --- a/docker/README.md +++ b/docker/README.md @@ -161,7 +161,7 @@ specify values for `SYNAPSE_CONFIG_PATH`, `SYNAPSE_SERVER_NAME` and example: ``` -docker run -it --rm +docker run -it --rm \ --mount type=volume,src=synapse-data,dst=/data \ -e SYNAPSE_CONFIG_PATH=/data/homeserver.yaml \ -e SYNAPSE_SERVER_NAME=my.matrix.host \ -- cgit 1.5.1 From 4f68188d0bbdb1966250375d34125572eb82a117 Mon Sep 17 00:00:00 2001 From: Neil Johnson Date: Thu, 13 Jun 2019 16:42:36 +0100 Subject: Change to absolute path for contrib/docker because this file is reproduced on dockerhub and relative paths don't work --- docker/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'docker') diff --git a/docker/README.md b/docker/README.md index df5d0151e2..5a596eecb9 100644 --- a/docker/README.md +++ b/docker/README.md @@ -14,7 +14,7 @@ This image is designed to run either with an automatically generated configuration file or with a custom configuration that requires manual editing. An easy way to make use of this image is via docker-compose. See the -[contrib/docker](../contrib/docker) section of the synapse project for +[contrib/docker](https://github.com/matrix-org/synapse/tree/master/contrib/docker) section of the synapse project for examples. ### Without Compose (harder) -- cgit 1.5.1 From eba7caf09fe9bb5f5a0d4b17c5dde1413343cadc Mon Sep 17 00:00:00 2001 From: Amber Brown Date: Tue, 18 Jun 2019 00:59:00 +1000 Subject: Remove Postgres 9.4 support (#5448) --- .buildkite/docker-compose.py35.pg94.yaml | 21 --------------------- .buildkite/pipeline.yml | 17 ----------------- CONTRIBUTING.rst | 19 +++++++++---------- UPGRADE.rst | 31 +++++++++++++++++++++++++++++-- changelog.d/5448.removal | 1 + docker/Dockerfile-pgtests | 4 ++-- docker/run_pg_tests.sh | 2 +- docs/postgres.rst | 4 ++-- synapse/storage/engines/postgres.py | 8 ++++++-- synapse/storage/search.py | 22 ---------------------- 10 files changed, 50 insertions(+), 79 deletions(-) delete mode 100644 .buildkite/docker-compose.py35.pg94.yaml create mode 100644 changelog.d/5448.removal (limited to 'docker') diff --git a/.buildkite/docker-compose.py35.pg94.yaml b/.buildkite/docker-compose.py35.pg94.yaml deleted file mode 100644 index 978aedd115..0000000000 --- a/.buildkite/docker-compose.py35.pg94.yaml +++ /dev/null @@ -1,21 +0,0 @@ -version: '3.1' - -services: - - postgres: - image: postgres:9.4 - environment: - POSTGRES_PASSWORD: postgres - - testenv: - image: python:3.5 - depends_on: - - postgres - env_file: .env - environment: - SYNAPSE_POSTGRES_HOST: postgres - SYNAPSE_POSTGRES_USER: postgres - SYNAPSE_POSTGRES_PASSWORD: postgres - working_dir: /app - volumes: - - ..:/app diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml index 6c6229a205..20c7aab5a7 100644 --- a/.buildkite/pipeline.yml +++ b/.buildkite/pipeline.yml @@ -116,23 +116,6 @@ steps: - exit_status: 2 limit: 2 - - label: ":python: 3.5 / :postgres: 9.4" - env: - TRIAL_FLAGS: "-j 4" - command: - - "bash -c 'python -m pip install tox && python -m tox -e py35-postgres,codecov'" - plugins: - - docker-compose#v2.1.0: - run: testenv - config: - - .buildkite/docker-compose.py35.pg94.yaml - retry: - automatic: - - exit_status: -1 - limit: 2 - - exit_status: 2 - limit: 2 - - label: ":python: 3.5 / :postgres: 9.5" env: TRIAL_FLAGS: "-j 4" diff --git a/CONTRIBUTING.rst b/CONTRIBUTING.rst index 9a283ced6e..2c44422a0e 100644 --- a/CONTRIBUTING.rst +++ b/CONTRIBUTING.rst @@ -30,21 +30,20 @@ use github's pull request workflow to review the contribution, and either ask you to make any refinements needed or merge it and make them ourselves. The changes will then land on master when we next do a release. -We use `CircleCI `_ and `Travis CI -`_ for continuous integration. All -pull requests to synapse get automatically tested by Travis and CircleCI. -If your change breaks the build, this will be shown in GitHub, so please -keep an eye on the pull request for feedback. +We use `CircleCI `_ and `Buildkite +`_ for continuous integration. +Buildkite builds need to be authorised by a maintainer. If your change breaks +the build, this will be shown in GitHub, so please keep an eye on the pull +request for feedback. To run unit tests in a local development environment, you can use: -- ``tox -e py27`` (requires tox to be installed by ``pip install tox``) for - SQLite-backed Synapse on Python 2.7. -- ``tox -e py35`` for SQLite-backed Synapse on Python 3.5. +- ``tox -e py35`` (requires tox to be installed by ``pip install tox``) + for SQLite-backed Synapse on Python 3.5. - ``tox -e py36`` for SQLite-backed Synapse on Python 3.6. -- ``tox -e py27-postgres`` for PostgreSQL-backed Synapse on Python 2.7 +- ``tox -e py36-postgres`` for PostgreSQL-backed Synapse on Python 3.6 (requires a running local PostgreSQL with access to create databases). -- ``./test_postgresql.sh`` for PostgreSQL-backed Synapse on Python 2.7 +- ``./test_postgresql.sh`` for PostgreSQL-backed Synapse on Python 3.5 (requires Docker). Entirely self-contained, recommended if you don't want to set up PostgreSQL yourself. diff --git a/UPGRADE.rst b/UPGRADE.rst index 6032a505c9..1fb109a218 100644 --- a/UPGRADE.rst +++ b/UPGRADE.rst @@ -49,6 +49,33 @@ returned by the Client-Server API: # configured on port 443. curl -kv https:///_matrix/client/versions 2>&1 | grep "Server:" +Upgrading to v1.1 +================= + +Synapse 1.1 removes support for older Python and PostgreSQL versions, as +outlined in `our deprecation notice `_. + +Minimum Python Version +---------------------- + +Synapse v1.1 has a minimum Python requirement of Python 3.5. Python 3.6 or +Python 3.7 are recommended as they have improved internal string handling, +significantly reducing memory usage. + +If you use current versions of the Matrix.org-distributed Debian packages or +Docker images, action is not required. + +If you install Synapse in a Python virtual environment, please see "Upgrading to +v0.34.0" for notes on setting up a new virtualenv under Python 3. + +Minimum PostgreSQL Version +-------------------------- + +If using PostgreSQL under Synapse, you will need to use PostgreSQL 9.5 or above. +Please see the +`PostgreSQL documentation `_ +for more details on upgrading your database. + Upgrading to v1.0 ================= @@ -71,11 +98,11 @@ server in a closed federation. This can be done in one of two ways:- * Configure a whitelist of server domains to trust via ``federation_certificate_verification_whitelist``. See the `sample configuration file `_ -for more details on these settings. +for more details on these settings. Email ----- -When a user requests a password reset, Synapse will send an email to the +When a user requests a password reset, Synapse will send an email to the user to confirm the request. Previous versions of Synapse delegated the job of sending this email to an diff --git a/changelog.d/5448.removal b/changelog.d/5448.removal new file mode 100644 index 0000000000..33b9859dae --- /dev/null +++ b/changelog.d/5448.removal @@ -0,0 +1 @@ +PostgreSQL 9.4 is no longer supported. Synapse requires Postgres 9.5+ or above for Postgres support. diff --git a/docker/Dockerfile-pgtests b/docker/Dockerfile-pgtests index 7da8eeb9eb..3bfee845c6 100644 --- a/docker/Dockerfile-pgtests +++ b/docker/Dockerfile-pgtests @@ -3,10 +3,10 @@ FROM matrixdotorg/sytest:latest # The Sytest image doesn't come with python, so install that -RUN apt-get -qq install -y python python-dev python-pip +RUN apt-get update && apt-get -qq install -y python3 python3-dev python3-pip # We need tox to run the tests in run_pg_tests.sh -RUN pip install tox +RUN python3 -m pip install tox ADD run_pg_tests.sh /pg_tests.sh ENTRYPOINT /pg_tests.sh diff --git a/docker/run_pg_tests.sh b/docker/run_pg_tests.sh index e77424c41a..d18d1e4c8e 100755 --- a/docker/run_pg_tests.sh +++ b/docker/run_pg_tests.sh @@ -17,4 +17,4 @@ su -c '/usr/lib/postgresql/9.6/bin/pg_ctl -w -D /var/lib/postgresql/data start' # Run the tests cd /src export TRIAL_FLAGS="-j 4" -tox --workdir=/tmp -e py27-postgres +tox --workdir=/tmp -e py35-postgres diff --git a/docs/postgres.rst b/docs/postgres.rst index e81e10403f..33f58e3ace 100644 --- a/docs/postgres.rst +++ b/docs/postgres.rst @@ -1,7 +1,7 @@ Using Postgres -------------- -Postgres version 9.4 or later is known to work. +Postgres version 9.5 or later is known to work. Install postgres client libraries ================================= @@ -16,7 +16,7 @@ a postgres database. * For other pre-built packages, please consult the documentation from the relevant package. -* If you installed synapse `in a virtualenv +* If you installed synapse `in a virtualenv <../INSTALL.md#installing-from-source>`_, you can install the library with:: ~/synapse/env/bin/pip install matrix-synapse[postgres] diff --git a/synapse/storage/engines/postgres.py b/synapse/storage/engines/postgres.py index 1b97ee74e3..289b6bc281 100644 --- a/synapse/storage/engines/postgres.py +++ b/synapse/storage/engines/postgres.py @@ -45,6 +45,10 @@ class PostgresEngine(object): # together. For example, version 8.1.5 will be returned as 80105 self._version = db_conn.server_version + # Are we on a supported PostgreSQL version? + if self._version < 90500: + raise RuntimeError("Synapse requires PostgreSQL 9.5+ or above.") + db_conn.set_isolation_level( self.module.extensions.ISOLATION_LEVEL_REPEATABLE_READ ) @@ -64,9 +68,9 @@ class PostgresEngine(object): @property def can_native_upsert(self): """ - Can we use native UPSERTs? This requires PostgreSQL 9.5+. + Can we use native UPSERTs? """ - return self._version >= 90500 + return True def is_deadlock(self, error): if isinstance(error, self.module.DatabaseError): diff --git a/synapse/storage/search.py b/synapse/storage/search.py index ff49eaae02..10a27c207a 100644 --- a/synapse/storage/search.py +++ b/synapse/storage/search.py @@ -341,29 +341,7 @@ class SearchStore(BackgroundUpdateStore): for entry in entries ) - # inserts to a GIN index are normally batched up into a pending - # list, and then all committed together once the list gets to a - # certain size. The trouble with that is that postgres (pre-9.5) - # uses work_mem to determine the length of the list, and work_mem - # is typically very large. - # - # We therefore reduce work_mem while we do the insert. - # - # (postgres 9.5 uses the separate gin_pending_list_limit setting, - # so doesn't suffer the same problem, but changing work_mem will - # be harmless) - # - # Note that we don't need to worry about restoring it on - # exception, because exceptions will cause the transaction to be - # rolled back, including the effects of the SET command. - # - # Also: we use SET rather than SET LOCAL because there's lots of - # other stuff going on in this transaction, which want to have the - # normal work_mem setting. - - txn.execute("SET work_mem='256kB'") txn.executemany(sql, args) - txn.execute("RESET work_mem") elif isinstance(self.database_engine, Sqlite3Engine): sql = ( -- cgit 1.5.1