From 6482075c95957ad980d9c1323f9f982e6f7aaff4 Mon Sep 17 00:00:00 2001 From: Richard van der Hoff <1389908+richvdh@users.noreply.github.com> Date: Fri, 14 May 2021 11:46:35 +0100 Subject: Run `black` on the scripts (#9981) Turns out these scripts weren't getting linted. --- scripts-dev/build_debian_packages | 105 ++++++++++++++++++++++++-------------- scripts-dev/lint.sh | 18 ++++++- 2 files changed, 83 insertions(+), 40 deletions(-) (limited to 'scripts-dev') diff --git a/scripts-dev/build_debian_packages b/scripts-dev/build_debian_packages index 07d018db99..546724f89f 100755 --- a/scripts-dev/build_debian_packages +++ b/scripts-dev/build_debian_packages @@ -21,18 +21,18 @@ DISTS = ( "debian:buster", "debian:bullseye", "debian:sid", - "ubuntu:bionic", # 18.04 LTS (our EOL forced by Py36 on 2021-12-23) - "ubuntu:focal", # 20.04 LTS (our EOL forced by Py38 on 2024-10-14) - "ubuntu:groovy", # 20.10 (EOL 2021-07-07) + "ubuntu:bionic", # 18.04 LTS (our EOL forced by Py36 on 2021-12-23) + "ubuntu:focal", # 20.04 LTS (our EOL forced by Py38 on 2024-10-14) + "ubuntu:groovy", # 20.10 (EOL 2021-07-07) "ubuntu:hirsute", # 21.04 (EOL 2022-01-05) ) -DESC = '''\ +DESC = """\ Builds .debs for synapse, using a Docker image for the build environment. By default, builds for all known distributions, but a list of distributions can be passed on the commandline for debugging. -''' +""" class Builder(object): @@ -46,7 +46,7 @@ class Builder(object): """Build deb for a single distribution""" if self._failed: - print("not building %s due to earlier failure" % (dist, )) + print("not building %s due to earlier failure" % (dist,)) raise Exception("failed") try: @@ -68,48 +68,65 @@ class Builder(object): # we tend to get source packages which are full of debs. (We could hack # around that with more magic in the build_debian.sh script, but that # doesn't solve the problem for natively-run dpkg-buildpakage). - debsdir = os.path.join(projdir, '../debs') + debsdir = os.path.join(projdir, "../debs") os.makedirs(debsdir, exist_ok=True) if self.redirect_stdout: - logfile = os.path.join(debsdir, "%s.buildlog" % (tag, )) + logfile = os.path.join(debsdir, "%s.buildlog" % (tag,)) print("building %s: directing output to %s" % (dist, logfile)) stdout = open(logfile, "w") else: stdout = None # first build a docker image for the build environment - subprocess.check_call([ - "docker", "build", - "--tag", "dh-venv-builder:" + tag, - "--build-arg", "distro=" + dist, - "-f", "docker/Dockerfile-dhvirtualenv", - "docker", - ], stdout=stdout, stderr=subprocess.STDOUT) + subprocess.check_call( + [ + "docker", + "build", + "--tag", + "dh-venv-builder:" + tag, + "--build-arg", + "distro=" + dist, + "-f", + "docker/Dockerfile-dhvirtualenv", + "docker", + ], + stdout=stdout, + stderr=subprocess.STDOUT, + ) container_name = "synapse_build_" + tag with self._lock: self.active_containers.add(container_name) # then run the build itself - subprocess.check_call([ - "docker", "run", - "--rm", - "--name", container_name, - "--volume=" + projdir + ":/synapse/source:ro", - "--volume=" + debsdir + ":/debs", - "-e", "TARGET_USERID=%i" % (os.getuid(), ), - "-e", "TARGET_GROUPID=%i" % (os.getgid(), ), - "-e", "DEB_BUILD_OPTIONS=%s" % ("nocheck" if skip_tests else ""), - "dh-venv-builder:" + tag, - ], stdout=stdout, stderr=subprocess.STDOUT) + subprocess.check_call( + [ + "docker", + "run", + "--rm", + "--name", + container_name, + "--volume=" + projdir + ":/synapse/source:ro", + "--volume=" + debsdir + ":/debs", + "-e", + "TARGET_USERID=%i" % (os.getuid(),), + "-e", + "TARGET_GROUPID=%i" % (os.getgid(),), + "-e", + "DEB_BUILD_OPTIONS=%s" % ("nocheck" if skip_tests else ""), + "dh-venv-builder:" + tag, + ], + stdout=stdout, + stderr=subprocess.STDOUT, + ) with self._lock: self.active_containers.remove(container_name) if stdout is not None: stdout.close() - print("Completed build of %s" % (dist, )) + print("Completed build of %s" % (dist,)) def kill_containers(self): with self._lock: @@ -117,9 +134,14 @@ class Builder(object): for c in active: print("killing container %s" % (c,)) - subprocess.run([ - "docker", "kill", c, - ], stdout=subprocess.DEVNULL) + subprocess.run( + [ + "docker", + "kill", + c, + ], + stdout=subprocess.DEVNULL, + ) with self._lock: self.active_containers.remove(c) @@ -130,31 +152,38 @@ def run_builds(dists, jobs=1, skip_tests=False): def sig(signum, _frame): print("Caught SIGINT") builder.kill_containers() + signal.signal(signal.SIGINT, sig) with ThreadPoolExecutor(max_workers=jobs) as e: res = e.map(lambda dist: builder.run_build(dist, skip_tests), dists) # make sure we consume the iterable so that exceptions are raised. - for r in res: + for _ in res: pass -if __name__ == '__main__': +if __name__ == "__main__": parser = argparse.ArgumentParser( description=DESC, ) parser.add_argument( - '-j', '--jobs', type=int, default=1, - help='specify the number of builds to run in parallel', + "-j", + "--jobs", + type=int, + default=1, + help="specify the number of builds to run in parallel", ) parser.add_argument( - '--no-check', action='store_true', - help='skip running tests after building', + "--no-check", + action="store_true", + help="skip running tests after building", ) parser.add_argument( - 'dist', nargs='*', default=DISTS, - help='a list of distributions to build for. Default: %(default)s', + "dist", + nargs="*", + default=DISTS, + help="a list of distributions to build for. Default: %(default)s", ) args = parser.parse_args() run_builds(dists=args.dist, jobs=args.jobs, skip_tests=args.no_check) diff --git a/scripts-dev/lint.sh b/scripts-dev/lint.sh index 9761e97594..869eb2372d 100755 --- a/scripts-dev/lint.sh +++ b/scripts-dev/lint.sh @@ -80,8 +80,22 @@ else # then lint everything! if [[ -z ${files+x} ]]; then # Lint all source code files and directories - # Note: this list aims the mirror the one in tox.ini - files=("synapse" "docker" "tests" "scripts-dev" "scripts" "contrib" "synctl" "setup.py" "synmark" "stubs" ".buildkite") + # Note: this list aims to mirror the one in tox.ini + files=( + "synapse" "docker" "tests" + # annoyingly, black doesn't find these so we have to list them + "scripts/export_signing_key" + "scripts/generate_config" + "scripts/generate_log_config" + "scripts/hash_password" + "scripts/register_new_matrix_user" + "scripts/synapse_port_db" + "scripts-dev" + "scripts-dev/build_debian_packages" + "scripts-dev/sign_json" + "scripts-dev/update_database" + "contrib" "synctl" "setup.py" "synmark" "stubs" ".buildkite" + ) fi fi -- cgit 1.5.1 From 057ce7b75406dc97be8ff2c890c47fd9357b0773 Mon Sep 17 00:00:00 2001 From: Jerin J Titus <72017981+jerinjtitus@users.noreply.github.com> Date: Mon, 24 May 2021 22:13:30 +0530 Subject: Remove tls_fingerprints option (#9280) Signed-off-by: Jerin J Titus <72017981+jerinjtitus@users.noreply.github.com> --- changelog.d/9280.removal | 1 + docs/sample_config.yaml | 27 ---------------- scripts-dev/convert_server_keys.py | 7 ----- synapse/config/tls.py | 50 ------------------------------ synapse/rest/key/v2/local_key_resource.py | 8 ----- synapse/rest/key/v2/remote_key_resource.py | 3 -- 6 files changed, 1 insertion(+), 95 deletions(-) create mode 100644 changelog.d/9280.removal (limited to 'scripts-dev') diff --git a/changelog.d/9280.removal b/changelog.d/9280.removal new file mode 100644 index 0000000000..c2ed3d308d --- /dev/null +++ b/changelog.d/9280.removal @@ -0,0 +1 @@ +Removed support for the deprecated `tls_fingerprints` configuration setting. Contributed by Jerin J Titus. \ No newline at end of file diff --git a/docs/sample_config.yaml b/docs/sample_config.yaml index f0f9f06a6e..6576b153d0 100644 --- a/docs/sample_config.yaml +++ b/docs/sample_config.yaml @@ -683,33 +683,6 @@ acme: # account_key_file: DATADIR/acme_account.key -# List of allowed TLS fingerprints for this server to publish along -# with the signing keys for this server. Other matrix servers that -# make HTTPS requests to this server will check that the TLS -# certificates returned by this server match one of the fingerprints. -# -# Synapse automatically adds the fingerprint of its own certificate -# to the list. So if federation traffic is handled directly by synapse -# then no modification to the list is required. -# -# If synapse is run behind a load balancer that handles the TLS then it -# will be necessary to add the fingerprints of the certificates used by -# the loadbalancers to this list if they are different to the one -# synapse is using. -# -# Homeservers are permitted to cache the list of TLS fingerprints -# returned in the key responses up to the "valid_until_ts" returned in -# key. It may be necessary to publish the fingerprints of a new -# certificate and wait until the "valid_until_ts" of the previous key -# responses have passed before deploying it. -# -# You can calculate a fingerprint from a given TLS listener via: -# openssl s_client -connect $host:$port < /dev/null 2> /dev/null | -# openssl x509 -outform DER | openssl sha256 -binary | base64 | tr -d '=' -# or by checking matrix.org/federationtester/api/report?server_name=$host -# -#tls_fingerprints: [{"sha256": ""}] - ## Federation ## diff --git a/scripts-dev/convert_server_keys.py b/scripts-dev/convert_server_keys.py index 961dc59f11..d4314a054c 100644 --- a/scripts-dev/convert_server_keys.py +++ b/scripts-dev/convert_server_keys.py @@ -1,4 +1,3 @@ -import hashlib import json import sys import time @@ -54,15 +53,9 @@ def convert_v1_to_v2(server_name, valid_until, keys, certificate): "server_name": server_name, "verify_keys": {key_id: {"key": key} for key_id, key in keys.items()}, "valid_until_ts": valid_until, - "tls_fingerprints": [fingerprint(certificate)], } -def fingerprint(certificate): - finger = hashlib.sha256(certificate) - return {"sha256": encode_base64(finger.digest())} - - def rows_v2(server, json): valid_until = json["valid_until_ts"] key_json = encode_canonical_json(json) diff --git a/synapse/config/tls.py b/synapse/config/tls.py index 7df4e4c3e6..26f1150ca5 100644 --- a/synapse/config/tls.py +++ b/synapse/config/tls.py @@ -16,11 +16,8 @@ import logging import os import warnings from datetime import datetime -from hashlib import sha256 from typing import List, Optional, Pattern -from unpaddedbase64 import encode_base64 - from OpenSSL import SSL, crypto from twisted.internet._sslverify import Certificate, trustRootFromCertificates @@ -83,13 +80,6 @@ class TlsConfig(Config): "configured." ) - self._original_tls_fingerprints = config.get("tls_fingerprints", []) - - if self._original_tls_fingerprints is None: - self._original_tls_fingerprints = [] - - self.tls_fingerprints = list(self._original_tls_fingerprints) - # Whether to verify certificates on outbound federation traffic self.federation_verify_certificates = config.get( "federation_verify_certificates", True @@ -248,19 +238,6 @@ class TlsConfig(Config): e, ) - self.tls_fingerprints = list(self._original_tls_fingerprints) - - if self.tls_certificate: - # Check that our own certificate is included in the list of fingerprints - # and include it if it is not. - x509_certificate_bytes = crypto.dump_certificate( - crypto.FILETYPE_ASN1, self.tls_certificate - ) - sha256_fingerprint = encode_base64(sha256(x509_certificate_bytes).digest()) - sha256_fingerprints = {f["sha256"] for f in self.tls_fingerprints} - if sha256_fingerprint not in sha256_fingerprints: - self.tls_fingerprints.append({"sha256": sha256_fingerprint}) - def generate_config_section( self, config_dir_path, @@ -443,33 +420,6 @@ class TlsConfig(Config): # If unspecified, we will use CONFDIR/client.key. # account_key_file: %(default_acme_account_file)s - - # List of allowed TLS fingerprints for this server to publish along - # with the signing keys for this server. Other matrix servers that - # make HTTPS requests to this server will check that the TLS - # certificates returned by this server match one of the fingerprints. - # - # Synapse automatically adds the fingerprint of its own certificate - # to the list. So if federation traffic is handled directly by synapse - # then no modification to the list is required. - # - # If synapse is run behind a load balancer that handles the TLS then it - # will be necessary to add the fingerprints of the certificates used by - # the loadbalancers to this list if they are different to the one - # synapse is using. - # - # Homeservers are permitted to cache the list of TLS fingerprints - # returned in the key responses up to the "valid_until_ts" returned in - # key. It may be necessary to publish the fingerprints of a new - # certificate and wait until the "valid_until_ts" of the previous key - # responses have passed before deploying it. - # - # You can calculate a fingerprint from a given TLS listener via: - # openssl s_client -connect $host:$port < /dev/null 2> /dev/null | - # openssl x509 -outform DER | openssl sha256 -binary | base64 | tr -d '=' - # or by checking matrix.org/federationtester/api/report?server_name=$host - # - #tls_fingerprints: [{"sha256": ""}] """ # Lowercase the string representation of boolean values % { diff --git a/synapse/rest/key/v2/local_key_resource.py b/synapse/rest/key/v2/local_key_resource.py index e8dbe240d8..a5fcd15e3a 100644 --- a/synapse/rest/key/v2/local_key_resource.py +++ b/synapse/rest/key/v2/local_key_resource.py @@ -48,11 +48,6 @@ class LocalKey(Resource): "key": # base64 encoded NACL verification key. } }, - "tls_fingerprints": [ # Fingerprints of the TLS certs this server uses. - { - "sha256": # base64 encoded sha256 fingerprint of the X509 cert - }, - ], "signatures": { "this.server.example.com": { "algorithm:version": # NACL signature for this server @@ -89,14 +84,11 @@ class LocalKey(Resource): "expired_ts": key.expired_ts, } - tls_fingerprints = self.config.tls_fingerprints - json_object = { "valid_until_ts": self.valid_until_ts, "server_name": self.config.server_name, "verify_keys": verify_keys, "old_verify_keys": old_verify_keys, - "tls_fingerprints": tls_fingerprints, } for key in self.config.signing_key: json_object = sign_json(json_object, self.config.server_name, key) diff --git a/synapse/rest/key/v2/remote_key_resource.py b/synapse/rest/key/v2/remote_key_resource.py index f648678b09..aba1734a55 100644 --- a/synapse/rest/key/v2/remote_key_resource.py +++ b/synapse/rest/key/v2/remote_key_resource.py @@ -73,9 +73,6 @@ class RemoteKey(DirectServeJsonResource): "expired_ts": 0, # when the key stop being used. } } - "tls_fingerprints": [ - { "sha256": # fingerprint } - ] "signatures": { "remote.server.example.com": {...} "this.server.example.com": {...} -- cgit 1.5.1 From 7d90d6ce9b6733bdebd4c5aca0c785e28e265e13 Mon Sep 17 00:00:00 2001 From: Patrick Cloke Date: Mon, 24 May 2021 15:32:45 -0400 Subject: Run complement with Synapse workers manually. (#10039) Adds an option to complement.sh to run Synapse in worker mode (instead of the default monolith mode). --- changelog.d/10039.misc | 1 + docker/configure_workers_and_start.py | 8 ++++---- scripts-dev/complement.sh | 25 ++++++++++++++++++++++--- 3 files changed, 27 insertions(+), 7 deletions(-) create mode 100644 changelog.d/10039.misc (limited to 'scripts-dev') diff --git a/changelog.d/10039.misc b/changelog.d/10039.misc new file mode 100644 index 0000000000..8855f141d9 --- /dev/null +++ b/changelog.d/10039.misc @@ -0,0 +1 @@ +Fix running complement tests with Synapse workers. diff --git a/docker/configure_workers_and_start.py b/docker/configure_workers_and_start.py index 4be6afc65d..1d22a4d571 100755 --- a/docker/configure_workers_and_start.py +++ b/docker/configure_workers_and_start.py @@ -184,18 +184,18 @@ stderr_logfile_maxbytes=0 """ NGINX_LOCATION_CONFIG_BLOCK = """ - location ~* {endpoint} { + location ~* {endpoint} {{ proxy_pass {upstream}; proxy_set_header X-Forwarded-For $remote_addr; proxy_set_header X-Forwarded-Proto $scheme; proxy_set_header Host $host; - } + }} """ NGINX_UPSTREAM_CONFIG_BLOCK = """ -upstream {upstream_worker_type} { +upstream {upstream_worker_type} {{ {body} -} +}} """ diff --git a/scripts-dev/complement.sh b/scripts-dev/complement.sh index 1612ab522c..0043964673 100755 --- a/scripts-dev/complement.sh +++ b/scripts-dev/complement.sh @@ -10,6 +10,9 @@ # checkout by setting the COMPLEMENT_DIR environment variable to the # filepath of a local Complement checkout. # +# By default Synapse is run in monolith mode. This can be overridden by +# setting the WORKERS environment variable. +# # A regular expression of test method names can be supplied as the first # argument to the script. Complement will then only run those tests. If # no regex is supplied, all tests are run. For example; @@ -32,10 +35,26 @@ if [[ -z "$COMPLEMENT_DIR" ]]; then echo "Checkout available at 'complement-master'" fi +# If we're using workers, modify the docker files slightly. +if [[ -n "$WORKERS" ]]; then + BASE_IMAGE=matrixdotorg/synapse-workers + BASE_DOCKERFILE=docker/Dockerfile-workers + export COMPLEMENT_BASE_IMAGE=complement-synapse-workers + COMPLEMENT_DOCKERFILE=SynapseWorkers.Dockerfile + # And provide some more configuration to complement. + export COMPLEMENT_CA=true + export COMPLEMENT_VERSION_CHECK_ITERATIONS=500 +else + BASE_IMAGE=matrixdotorg/synapse + BASE_DOCKERFILE=docker/Dockerfile + export COMPLEMENT_BASE_IMAGE=complement-synapse + COMPLEMENT_DOCKERFILE=Synapse.Dockerfile +fi + # Build the base Synapse image from the local checkout -docker build -t matrixdotorg/synapse -f docker/Dockerfile . +docker build -t $BASE_IMAGE -f "$BASE_DOCKERFILE" . # Build the Synapse monolith image from Complement, based on the above image we just built -docker build -t complement-synapse -f "$COMPLEMENT_DIR/dockerfiles/Synapse.Dockerfile" "$COMPLEMENT_DIR/dockerfiles" +docker build -t $COMPLEMENT_BASE_IMAGE -f "$COMPLEMENT_DIR/dockerfiles/$COMPLEMENT_DOCKERFILE" "$COMPLEMENT_DIR/dockerfiles" cd "$COMPLEMENT_DIR" @@ -46,4 +65,4 @@ if [[ -n "$1" ]]; then fi # Run the tests! -COMPLEMENT_BASE_IMAGE=complement-synapse go test -v -tags synapse_blacklist,msc2946,msc3083 -count=1 $EXTRA_COMPLEMENT_ARGS ./tests +go test -v -tags synapse_blacklist,msc2946,msc3083 -count=1 $EXTRA_COMPLEMENT_ARGS ./tests -- cgit 1.5.1