From 1fbe0316a991e77289d4577b16ff3fcd27c26dc8 Mon Sep 17 00:00:00 2001 From: David Robertson Date: Wed, 2 Mar 2022 18:00:26 +0000 Subject: Add suffices to scripts in scripts-dev (#12137) * Rename scripts-dev to have suffices * Update references to `scripts-dev` * Changelog * These scripts don't pass mypy --- .github/workflows/release-artifacts.yml | 4 +- .github/workflows/tests.yml | 4 +- changelog.d/12137.misc | 1 + docs/code_style.md | 2 +- mypy.ini | 12 +- scripts-dev/build_debian_packages | 217 -------------------------------- scripts-dev/build_debian_packages.py | 217 ++++++++++++++++++++++++++++++++ scripts-dev/check-newsfragment | 62 --------- scripts-dev/check-newsfragment.sh | 62 +++++++++ scripts-dev/generate_sample_config | 28 ----- scripts-dev/generate_sample_config.sh | 28 +++++ scripts-dev/lint.sh | 2 - scripts-dev/sign_json | 166 ------------------------ scripts-dev/sign_json.py | 166 ++++++++++++++++++++++++ tox.ini | 2 - 15 files changed, 490 insertions(+), 483 deletions(-) create mode 100644 changelog.d/12137.misc delete mode 100755 scripts-dev/build_debian_packages create mode 100755 scripts-dev/build_debian_packages.py delete mode 100755 scripts-dev/check-newsfragment create mode 100755 scripts-dev/check-newsfragment.sh delete mode 100755 scripts-dev/generate_sample_config create mode 100755 scripts-dev/generate_sample_config.sh delete mode 100755 scripts-dev/sign_json create mode 100755 scripts-dev/sign_json.py diff --git a/.github/workflows/release-artifacts.yml b/.github/workflows/release-artifacts.yml index eee3633d50..65ea761ad7 100644 --- a/.github/workflows/release-artifacts.yml +++ b/.github/workflows/release-artifacts.yml @@ -31,7 +31,7 @@ jobs: # if we're running from a tag, get the full list of distros; otherwise just use debian:sid dists='["debian:sid"]' if [[ $GITHUB_REF == refs/tags/* ]]; then - dists=$(scripts-dev/build_debian_packages --show-dists-json) + dists=$(scripts-dev/build_debian_packages.py --show-dists-json) fi echo "::set-output name=distros::$dists" # map the step outputs to job outputs @@ -74,7 +74,7 @@ jobs: # see https://github.com/docker/build-push-action/issues/252 # for the cache magic here run: | - ./src/scripts-dev/build_debian_packages \ + ./src/scripts-dev/build_debian_packages.py \ --docker-build-arg=--cache-from=type=local,src=/tmp/.buildx-cache \ --docker-build-arg=--cache-to=type=local,mode=max,dest=/tmp/.buildx-cache-new \ --docker-build-arg=--progress=plain \ diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index e9e4277322..3f4e44ca59 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -16,7 +16,7 @@ jobs: - uses: actions/checkout@v2 - uses: actions/setup-python@v2 - run: pip install -e . - - run: scripts-dev/generate_sample_config --check + - run: scripts-dev/generate_sample_config.sh --check lint: runs-on: ubuntu-latest @@ -51,7 +51,7 @@ jobs: fetch-depth: 0 - uses: actions/setup-python@v2 - run: "pip install 'towncrier>=18.6.0rc1'" - - run: scripts-dev/check-newsfragment + - run: scripts-dev/check-newsfragment.sh env: PULL_REQUEST_NUMBER: ${{ github.event.number }} diff --git a/changelog.d/12137.misc b/changelog.d/12137.misc new file mode 100644 index 0000000000..118ff77a91 --- /dev/null +++ b/changelog.d/12137.misc @@ -0,0 +1 @@ +Give `scripts-dev` scripts suffixes for neater CI config. \ No newline at end of file diff --git a/docs/code_style.md b/docs/code_style.md index 4d8e7c973d..e7c9cd1a5e 100644 --- a/docs/code_style.md +++ b/docs/code_style.md @@ -172,6 +172,6 @@ frobber: ``` Note that the sample configuration is generated from the synapse code -and is maintained by a script, `scripts-dev/generate_sample_config`. +and is maintained by a script, `scripts-dev/generate_sample_config.sh`. Making sure that the output from this script matches the desired format is left as an exercise for the reader! diff --git a/mypy.ini b/mypy.ini index 23ca4eaa5a..10971b7225 100644 --- a/mypy.ini +++ b/mypy.ini @@ -11,7 +11,7 @@ local_partial_types = True no_implicit_optional = True files = - scripts-dev/sign_json, + scripts-dev/, setup.py, synapse/, tests/ @@ -23,10 +23,20 @@ files = # https://docs.python.org/3/library/re.html#re.X exclude = (?x) ^( + |scripts-dev/build_debian_packages.py + |scripts-dev/check_signature.py + |scripts-dev/definitions.py + |scripts-dev/federation_client.py + |scripts-dev/hash_history.py + |scripts-dev/list_url_patterns.py + |scripts-dev/release.py + |scripts-dev/tail-synapse.py + |synapse/_scripts/export_signing_key.py |synapse/_scripts/move_remote_media_to_new_store.py |synapse/_scripts/synapse_port_db.py |synapse/_scripts/update_synapse_database.py + |synapse/storage/databases/__init__.py |synapse/storage/databases/main/__init__.py |synapse/storage/databases/main/cache.py diff --git a/scripts-dev/build_debian_packages b/scripts-dev/build_debian_packages deleted file mode 100755 index 7ff96a1ee6..0000000000 --- a/scripts-dev/build_debian_packages +++ /dev/null @@ -1,217 +0,0 @@ -#!/usr/bin/env python3 - -# Build the Debian packages using Docker images. -# -# This script builds the Docker images and then executes them sequentially, each -# one building a Debian package for the targeted operating system. It is -# designed to be a "single command" to produce all the images. -# -# By default, builds for all known distributions, but a list of distributions -# can be passed on the commandline for debugging. - -import argparse -import json -import os -import signal -import subprocess -import sys -import threading -from concurrent.futures import ThreadPoolExecutor -from typing import Optional, Sequence - -DISTS = ( - "debian:buster", # oldstable: EOL 2022-08 - "debian:bullseye", - "debian:bookworm", - "debian:sid", - "ubuntu:focal", # 20.04 LTS (our EOL forced by Py38 on 2024-10-14) - "ubuntu:impish", # 21.10 (EOL 2022-07) -) - -DESC = """\ -Builds .debs for synapse, using a Docker image for the build environment. - -By default, builds for all known distributions, but a list of distributions -can be passed on the commandline for debugging. -""" - -projdir = os.path.dirname(os.path.dirname(os.path.realpath(__file__))) - - -class Builder(object): - def __init__( - self, redirect_stdout=False, docker_build_args: Optional[Sequence[str]] = None - ): - self.redirect_stdout = redirect_stdout - self._docker_build_args = tuple(docker_build_args or ()) - self.active_containers = set() - self._lock = threading.Lock() - self._failed = False - - def run_build(self, dist, skip_tests=False): - """Build deb for a single distribution""" - - if self._failed: - print("not building %s due to earlier failure" % (dist,)) - raise Exception("failed") - - try: - self._inner_build(dist, skip_tests) - except Exception as e: - print("build of %s failed: %s" % (dist, e), file=sys.stderr) - self._failed = True - raise - - def _inner_build(self, dist, skip_tests=False): - tag = dist.split(":", 1)[1] - - # Make the dir where the debs will live. - # - # Note that we deliberately put this outside the source tree, otherwise - # we tend to get source packages which are full of debs. (We could hack - # around that with more magic in the build_debian.sh script, but that - # doesn't solve the problem for natively-run dpkg-buildpakage). - debsdir = os.path.join(projdir, "../debs") - os.makedirs(debsdir, exist_ok=True) - - if self.redirect_stdout: - logfile = os.path.join(debsdir, "%s.buildlog" % (tag,)) - print("building %s: directing output to %s" % (dist, logfile)) - stdout = open(logfile, "w") - else: - stdout = None - - # first build a docker image for the build environment - build_args = ( - ( - "docker", - "build", - "--tag", - "dh-venv-builder:" + tag, - "--build-arg", - "distro=" + dist, - "-f", - "docker/Dockerfile-dhvirtualenv", - ) - + self._docker_build_args - + ("docker",) - ) - - subprocess.check_call( - build_args, - stdout=stdout, - stderr=subprocess.STDOUT, - cwd=projdir, - ) - - container_name = "synapse_build_" + tag - with self._lock: - self.active_containers.add(container_name) - - # then run the build itself - subprocess.check_call( - [ - "docker", - "run", - "--rm", - "--name", - container_name, - "--volume=" + projdir + ":/synapse/source:ro", - "--volume=" + debsdir + ":/debs", - "-e", - "TARGET_USERID=%i" % (os.getuid(),), - "-e", - "TARGET_GROUPID=%i" % (os.getgid(),), - "-e", - "DEB_BUILD_OPTIONS=%s" % ("nocheck" if skip_tests else ""), - "dh-venv-builder:" + tag, - ], - stdout=stdout, - stderr=subprocess.STDOUT, - ) - - with self._lock: - self.active_containers.remove(container_name) - - if stdout is not None: - stdout.close() - print("Completed build of %s" % (dist,)) - - def kill_containers(self): - with self._lock: - active = list(self.active_containers) - - for c in active: - print("killing container %s" % (c,)) - subprocess.run( - [ - "docker", - "kill", - c, - ], - stdout=subprocess.DEVNULL, - ) - with self._lock: - self.active_containers.remove(c) - - -def run_builds(builder, dists, jobs=1, skip_tests=False): - def sig(signum, _frame): - print("Caught SIGINT") - builder.kill_containers() - - signal.signal(signal.SIGINT, sig) - - with ThreadPoolExecutor(max_workers=jobs) as e: - res = e.map(lambda dist: builder.run_build(dist, skip_tests), dists) - - # make sure we consume the iterable so that exceptions are raised. - for _ in res: - pass - - -if __name__ == "__main__": - parser = argparse.ArgumentParser( - description=DESC, - ) - parser.add_argument( - "-j", - "--jobs", - type=int, - default=1, - help="specify the number of builds to run in parallel", - ) - parser.add_argument( - "--no-check", - action="store_true", - help="skip running tests after building", - ) - parser.add_argument( - "--docker-build-arg", - action="append", - help="specify an argument to pass to docker build", - ) - parser.add_argument( - "--show-dists-json", - action="store_true", - help="instead of building the packages, just list the dists to build for, as a json array", - ) - parser.add_argument( - "dist", - nargs="*", - default=DISTS, - help="a list of distributions to build for. Default: %(default)s", - ) - args = parser.parse_args() - if args.show_dists_json: - print(json.dumps(DISTS)) - else: - builder = Builder( - redirect_stdout=(args.jobs > 1), docker_build_args=args.docker_build_arg - ) - run_builds( - builder, - dists=args.dist, - jobs=args.jobs, - skip_tests=args.no_check, - ) diff --git a/scripts-dev/build_debian_packages.py b/scripts-dev/build_debian_packages.py new file mode 100755 index 0000000000..7ff96a1ee6 --- /dev/null +++ b/scripts-dev/build_debian_packages.py @@ -0,0 +1,217 @@ +#!/usr/bin/env python3 + +# Build the Debian packages using Docker images. +# +# This script builds the Docker images and then executes them sequentially, each +# one building a Debian package for the targeted operating system. It is +# designed to be a "single command" to produce all the images. +# +# By default, builds for all known distributions, but a list of distributions +# can be passed on the commandline for debugging. + +import argparse +import json +import os +import signal +import subprocess +import sys +import threading +from concurrent.futures import ThreadPoolExecutor +from typing import Optional, Sequence + +DISTS = ( + "debian:buster", # oldstable: EOL 2022-08 + "debian:bullseye", + "debian:bookworm", + "debian:sid", + "ubuntu:focal", # 20.04 LTS (our EOL forced by Py38 on 2024-10-14) + "ubuntu:impish", # 21.10 (EOL 2022-07) +) + +DESC = """\ +Builds .debs for synapse, using a Docker image for the build environment. + +By default, builds for all known distributions, but a list of distributions +can be passed on the commandline for debugging. +""" + +projdir = os.path.dirname(os.path.dirname(os.path.realpath(__file__))) + + +class Builder(object): + def __init__( + self, redirect_stdout=False, docker_build_args: Optional[Sequence[str]] = None + ): + self.redirect_stdout = redirect_stdout + self._docker_build_args = tuple(docker_build_args or ()) + self.active_containers = set() + self._lock = threading.Lock() + self._failed = False + + def run_build(self, dist, skip_tests=False): + """Build deb for a single distribution""" + + if self._failed: + print("not building %s due to earlier failure" % (dist,)) + raise Exception("failed") + + try: + self._inner_build(dist, skip_tests) + except Exception as e: + print("build of %s failed: %s" % (dist, e), file=sys.stderr) + self._failed = True + raise + + def _inner_build(self, dist, skip_tests=False): + tag = dist.split(":", 1)[1] + + # Make the dir where the debs will live. + # + # Note that we deliberately put this outside the source tree, otherwise + # we tend to get source packages which are full of debs. (We could hack + # around that with more magic in the build_debian.sh script, but that + # doesn't solve the problem for natively-run dpkg-buildpakage). + debsdir = os.path.join(projdir, "../debs") + os.makedirs(debsdir, exist_ok=True) + + if self.redirect_stdout: + logfile = os.path.join(debsdir, "%s.buildlog" % (tag,)) + print("building %s: directing output to %s" % (dist, logfile)) + stdout = open(logfile, "w") + else: + stdout = None + + # first build a docker image for the build environment + build_args = ( + ( + "docker", + "build", + "--tag", + "dh-venv-builder:" + tag, + "--build-arg", + "distro=" + dist, + "-f", + "docker/Dockerfile-dhvirtualenv", + ) + + self._docker_build_args + + ("docker",) + ) + + subprocess.check_call( + build_args, + stdout=stdout, + stderr=subprocess.STDOUT, + cwd=projdir, + ) + + container_name = "synapse_build_" + tag + with self._lock: + self.active_containers.add(container_name) + + # then run the build itself + subprocess.check_call( + [ + "docker", + "run", + "--rm", + "--name", + container_name, + "--volume=" + projdir + ":/synapse/source:ro", + "--volume=" + debsdir + ":/debs", + "-e", + "TARGET_USERID=%i" % (os.getuid(),), + "-e", + "TARGET_GROUPID=%i" % (os.getgid(),), + "-e", + "DEB_BUILD_OPTIONS=%s" % ("nocheck" if skip_tests else ""), + "dh-venv-builder:" + tag, + ], + stdout=stdout, + stderr=subprocess.STDOUT, + ) + + with self._lock: + self.active_containers.remove(container_name) + + if stdout is not None: + stdout.close() + print("Completed build of %s" % (dist,)) + + def kill_containers(self): + with self._lock: + active = list(self.active_containers) + + for c in active: + print("killing container %s" % (c,)) + subprocess.run( + [ + "docker", + "kill", + c, + ], + stdout=subprocess.DEVNULL, + ) + with self._lock: + self.active_containers.remove(c) + + +def run_builds(builder, dists, jobs=1, skip_tests=False): + def sig(signum, _frame): + print("Caught SIGINT") + builder.kill_containers() + + signal.signal(signal.SIGINT, sig) + + with ThreadPoolExecutor(max_workers=jobs) as e: + res = e.map(lambda dist: builder.run_build(dist, skip_tests), dists) + + # make sure we consume the iterable so that exceptions are raised. + for _ in res: + pass + + +if __name__ == "__main__": + parser = argparse.ArgumentParser( + description=DESC, + ) + parser.add_argument( + "-j", + "--jobs", + type=int, + default=1, + help="specify the number of builds to run in parallel", + ) + parser.add_argument( + "--no-check", + action="store_true", + help="skip running tests after building", + ) + parser.add_argument( + "--docker-build-arg", + action="append", + help="specify an argument to pass to docker build", + ) + parser.add_argument( + "--show-dists-json", + action="store_true", + help="instead of building the packages, just list the dists to build for, as a json array", + ) + parser.add_argument( + "dist", + nargs="*", + default=DISTS, + help="a list of distributions to build for. Default: %(default)s", + ) + args = parser.parse_args() + if args.show_dists_json: + print(json.dumps(DISTS)) + else: + builder = Builder( + redirect_stdout=(args.jobs > 1), docker_build_args=args.docker_build_arg + ) + run_builds( + builder, + dists=args.dist, + jobs=args.jobs, + skip_tests=args.no_check, + ) diff --git a/scripts-dev/check-newsfragment b/scripts-dev/check-newsfragment deleted file mode 100755 index 493558ad65..0000000000 --- a/scripts-dev/check-newsfragment +++ /dev/null @@ -1,62 +0,0 @@ -#!/usr/bin/env bash -# -# A script which checks that an appropriate news file has been added on this -# branch. - -echo -e "+++ \033[32mChecking newsfragment\033[m" - -set -e - -# make sure that origin/develop is up to date -git remote set-branches --add origin develop -git fetch -q origin develop - -pr="$PULL_REQUEST_NUMBER" - -# if there are changes in the debian directory, check that the debian changelog -# has been updated -if ! git diff --quiet FETCH_HEAD... -- debian; then - if git diff --quiet FETCH_HEAD... -- debian/changelog; then - echo "Updates to debian directory, but no update to the changelog." >&2 - echo "!! Please see the contributing guide for help writing your changelog entry:" >&2 - echo "https://github.com/matrix-org/synapse/blob/develop/CONTRIBUTING.md#debian-changelog" >&2 - exit 1 - fi -fi - -# if there are changes *outside* the debian directory, check that the -# newsfragments have been updated. -if ! git diff --name-only FETCH_HEAD... | grep -qv '^debian/'; then - exit 0 -fi - -# Print a link to the contributing guide if the user makes a mistake -CONTRIBUTING_GUIDE_TEXT="!! Please see the contributing guide for help writing your changelog entry: -https://github.com/matrix-org/synapse/blob/develop/CONTRIBUTING.md#changelog" - -# If check-newsfragment returns a non-zero exit code, print the contributing guide and exit -python -m towncrier.check --compare-with=origin/develop || (echo -e "$CONTRIBUTING_GUIDE_TEXT" >&2 && exit 1) - -echo -echo "--------------------------" -echo - -matched=0 -for f in $(git diff --diff-filter=d --name-only FETCH_HEAD... -- changelog.d); do - # check that any added newsfiles on this branch end with a full stop. - lastchar=$(tr -d '\n' < "$f" | tail -c 1) - if [ "$lastchar" != '.' ] && [ "$lastchar" != '!' ]; then - echo -e "\e[31mERROR: newsfragment $f does not end with a '.' or '!'\e[39m" >&2 - echo -e "$CONTRIBUTING_GUIDE_TEXT" >&2 - exit 1 - fi - - # see if this newsfile corresponds to the right PR - [[ -n "$pr" && "$f" == changelog.d/"$pr".* ]] && matched=1 -done - -if [[ -n "$pr" && "$matched" -eq 0 ]]; then - echo -e "\e[31mERROR: Did not find a news fragment with the right number: expected changelog.d/$pr.*.\e[39m" >&2 - echo -e "$CONTRIBUTING_GUIDE_TEXT" >&2 - exit 1 -fi diff --git a/scripts-dev/check-newsfragment.sh b/scripts-dev/check-newsfragment.sh new file mode 100755 index 0000000000..493558ad65 --- /dev/null +++ b/scripts-dev/check-newsfragment.sh @@ -0,0 +1,62 @@ +#!/usr/bin/env bash +# +# A script which checks that an appropriate news file has been added on this +# branch. + +echo -e "+++ \033[32mChecking newsfragment\033[m" + +set -e + +# make sure that origin/develop is up to date +git remote set-branches --add origin develop +git fetch -q origin develop + +pr="$PULL_REQUEST_NUMBER" + +# if there are changes in the debian directory, check that the debian changelog +# has been updated +if ! git diff --quiet FETCH_HEAD... -- debian; then + if git diff --quiet FETCH_HEAD... -- debian/changelog; then + echo "Updates to debian directory, but no update to the changelog." >&2 + echo "!! Please see the contributing guide for help writing your changelog entry:" >&2 + echo "https://github.com/matrix-org/synapse/blob/develop/CONTRIBUTING.md#debian-changelog" >&2 + exit 1 + fi +fi + +# if there are changes *outside* the debian directory, check that the +# newsfragments have been updated. +if ! git diff --name-only FETCH_HEAD... | grep -qv '^debian/'; then + exit 0 +fi + +# Print a link to the contributing guide if the user makes a mistake +CONTRIBUTING_GUIDE_TEXT="!! Please see the contributing guide for help writing your changelog entry: +https://github.com/matrix-org/synapse/blob/develop/CONTRIBUTING.md#changelog" + +# If check-newsfragment returns a non-zero exit code, print the contributing guide and exit +python -m towncrier.check --compare-with=origin/develop || (echo -e "$CONTRIBUTING_GUIDE_TEXT" >&2 && exit 1) + +echo +echo "--------------------------" +echo + +matched=0 +for f in $(git diff --diff-filter=d --name-only FETCH_HEAD... -- changelog.d); do + # check that any added newsfiles on this branch end with a full stop. + lastchar=$(tr -d '\n' < "$f" | tail -c 1) + if [ "$lastchar" != '.' ] && [ "$lastchar" != '!' ]; then + echo -e "\e[31mERROR: newsfragment $f does not end with a '.' or '!'\e[39m" >&2 + echo -e "$CONTRIBUTING_GUIDE_TEXT" >&2 + exit 1 + fi + + # see if this newsfile corresponds to the right PR + [[ -n "$pr" && "$f" == changelog.d/"$pr".* ]] && matched=1 +done + +if [[ -n "$pr" && "$matched" -eq 0 ]]; then + echo -e "\e[31mERROR: Did not find a news fragment with the right number: expected changelog.d/$pr.*.\e[39m" >&2 + echo -e "$CONTRIBUTING_GUIDE_TEXT" >&2 + exit 1 +fi diff --git a/scripts-dev/generate_sample_config b/scripts-dev/generate_sample_config deleted file mode 100755 index 185e277933..0000000000 --- a/scripts-dev/generate_sample_config +++ /dev/null @@ -1,28 +0,0 @@ -#!/usr/bin/env bash -# -# Update/check the docs/sample_config.yaml - -set -e - -cd "$(dirname "$0")/.." - -SAMPLE_CONFIG="docs/sample_config.yaml" -SAMPLE_LOG_CONFIG="docs/sample_log_config.yaml" - -check() { - diff -u "$SAMPLE_LOG_CONFIG" <(synapse/_scripts/generate_log_config.py) >/dev/null || return 1 -} - -if [ "$1" == "--check" ]; then - diff -u "$SAMPLE_CONFIG" <(synapse/_scripts/generate_config.py --header-file docs/.sample_config_header.yaml) >/dev/null || { - echo -e "\e[1m\e[31m$SAMPLE_CONFIG is not up-to-date. Regenerate it with \`scripts-dev/generate_sample_config\`.\e[0m" >&2 - exit 1 - } - diff -u "$SAMPLE_LOG_CONFIG" <(synapse/_scripts/generate_log_config.py) >/dev/null || { - echo -e "\e[1m\e[31m$SAMPLE_LOG_CONFIG is not up-to-date. Regenerate it with \`scripts-dev/generate_sample_config\`.\e[0m" >&2 - exit 1 - } -else - synapse/_scripts/generate_config.py --header-file docs/.sample_config_header.yaml -o "$SAMPLE_CONFIG" - synapse/_scripts/generate_log_config.py -o "$SAMPLE_LOG_CONFIG" -fi diff --git a/scripts-dev/generate_sample_config.sh b/scripts-dev/generate_sample_config.sh new file mode 100755 index 0000000000..375897eacb --- /dev/null +++ b/scripts-dev/generate_sample_config.sh @@ -0,0 +1,28 @@ +#!/usr/bin/env bash +# +# Update/check the docs/sample_config.yaml + +set -e + +cd "$(dirname "$0")/.." + +SAMPLE_CONFIG="docs/sample_config.yaml" +SAMPLE_LOG_CONFIG="docs/sample_log_config.yaml" + +check() { + diff -u "$SAMPLE_LOG_CONFIG" <(synapse/_scripts/generate_log_config.py) >/dev/null || return 1 +} + +if [ "$1" == "--check" ]; then + diff -u "$SAMPLE_CONFIG" <(synapse/_scripts/generate_config.py --header-file docs/.sample_config_header.yaml) >/dev/null || { + echo -e "\e[1m\e[31m$SAMPLE_CONFIG is not up-to-date. Regenerate it with \`scripts-dev/generate_sample_config.sh\`.\e[0m" >&2 + exit 1 + } + diff -u "$SAMPLE_LOG_CONFIG" <(synapse/_scripts/generate_log_config.py) >/dev/null || { + echo -e "\e[1m\e[31m$SAMPLE_LOG_CONFIG is not up-to-date. Regenerate it with \`scripts-dev/generate_sample_config.sh\`.\e[0m" >&2 + exit 1 + } +else + synapse/_scripts/generate_config.py --header-file docs/.sample_config_header.yaml -o "$SAMPLE_CONFIG" + synapse/_scripts/generate_log_config.py -o "$SAMPLE_LOG_CONFIG" +fi diff --git a/scripts-dev/lint.sh b/scripts-dev/lint.sh index df4d4934d0..2f5f2c3566 100755 --- a/scripts-dev/lint.sh +++ b/scripts-dev/lint.sh @@ -85,8 +85,6 @@ else "synapse" "docker" "tests" # annoyingly, black doesn't find these so we have to list them "scripts-dev" - "scripts-dev/build_debian_packages" - "scripts-dev/sign_json" "contrib" "synctl" "setup.py" "synmark" "stubs" ".ci" ) fi diff --git a/scripts-dev/sign_json b/scripts-dev/sign_json deleted file mode 100755 index 9459543106..0000000000 --- a/scripts-dev/sign_json +++ /dev/null @@ -1,166 +0,0 @@ -#!/usr/bin/env python -# -# Copyright 2020 The Matrix.org Foundation C.I.C. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -import argparse -import json -import sys -from json import JSONDecodeError - -import yaml -from signedjson.key import read_signing_keys -from signedjson.sign import sign_json - -from synapse.api.room_versions import KNOWN_ROOM_VERSIONS -from synapse.crypto.event_signing import add_hashes_and_signatures -from synapse.util import json_encoder - - -def main(): - parser = argparse.ArgumentParser( - description="""Adds a signature to a JSON object. - -Example usage: - - $ scripts-dev/sign_json.py -N test -k localhost.signing.key "{}" - {"signatures":{"test":{"ed25519:a_ZnZh":"LmPnml6iM0iR..."}}} -""", - formatter_class=argparse.RawDescriptionHelpFormatter, - ) - - parser.add_argument( - "-N", - "--server-name", - help="Name to give as the local homeserver. If unspecified, will be " - "read from the config file.", - ) - - parser.add_argument( - "-k", - "--signing-key-path", - help="Path to the file containing the private ed25519 key to sign the " - "request with.", - ) - - parser.add_argument( - "-K", - "--signing-key", - help="The private ed25519 key to sign the request with.", - ) - - parser.add_argument( - "-c", - "--config", - default="homeserver.yaml", - help=( - "Path to synapse config file, from which the server name and/or signing " - "key path will be read. Ignored if --server-name and --signing-key(-path) " - "are both given." - ), - ) - - parser.add_argument( - "--sign-event-room-version", - type=str, - help=( - "Sign the JSON as an event for the given room version, rather than raw JSON. " - "This means that we will add a 'hashes' object, and redact the event before " - "signing." - ), - ) - - input_args = parser.add_mutually_exclusive_group() - - input_args.add_argument("input_data", nargs="?", help="Raw JSON to be signed.") - - input_args.add_argument( - "-i", - "--input", - type=argparse.FileType("r"), - default=sys.stdin, - help=( - "A file from which to read the JSON to be signed. If neither --input nor " - "input_data are given, JSON will be read from stdin." - ), - ) - - parser.add_argument( - "-o", - "--output", - type=argparse.FileType("w"), - default=sys.stdout, - help="Where to write the signed JSON. Defaults to stdout.", - ) - - args = parser.parse_args() - - if not args.server_name or not (args.signing_key_path or args.signing_key): - read_args_from_config(args) - - if args.signing_key: - keys = read_signing_keys([args.signing_key]) - else: - with open(args.signing_key_path) as f: - keys = read_signing_keys(f) - - json_to_sign = args.input_data - if json_to_sign is None: - json_to_sign = args.input.read() - - try: - obj = json.loads(json_to_sign) - except JSONDecodeError as e: - print("Unable to parse input as JSON: %s" % e, file=sys.stderr) - sys.exit(1) - - if not isinstance(obj, dict): - print("Input json was not an object", file=sys.stderr) - sys.exit(1) - - if args.sign_event_room_version: - room_version = KNOWN_ROOM_VERSIONS.get(args.sign_event_room_version) - if not room_version: - print( - f"Unknown room version {args.sign_event_room_version}", file=sys.stderr - ) - sys.exit(1) - add_hashes_and_signatures(room_version, obj, args.server_name, keys[0]) - else: - sign_json(obj, args.server_name, keys[0]) - - for c in json_encoder.iterencode(obj): - args.output.write(c) - args.output.write("\n") - - -def read_args_from_config(args: argparse.Namespace) -> None: - with open(args.config, "r") as fh: - config = yaml.safe_load(fh) - if not args.server_name: - args.server_name = config["server_name"] - if not args.signing_key_path and not args.signing_key: - if "signing_key" in config: - args.signing_key = config["signing_key"] - elif "signing_key_path" in config: - args.signing_key_path = config["signing_key_path"] - else: - print( - "A signing key must be given on the commandline or in the config file.", - file=sys.stderr, - ) - sys.exit(1) - - -if __name__ == "__main__": - main() diff --git a/scripts-dev/sign_json.py b/scripts-dev/sign_json.py new file mode 100755 index 0000000000..9459543106 --- /dev/null +++ b/scripts-dev/sign_json.py @@ -0,0 +1,166 @@ +#!/usr/bin/env python +# +# Copyright 2020 The Matrix.org Foundation C.I.C. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import argparse +import json +import sys +from json import JSONDecodeError + +import yaml +from signedjson.key import read_signing_keys +from signedjson.sign import sign_json + +from synapse.api.room_versions import KNOWN_ROOM_VERSIONS +from synapse.crypto.event_signing import add_hashes_and_signatures +from synapse.util import json_encoder + + +def main(): + parser = argparse.ArgumentParser( + description="""Adds a signature to a JSON object. + +Example usage: + + $ scripts-dev/sign_json.py -N test -k localhost.signing.key "{}" + {"signatures":{"test":{"ed25519:a_ZnZh":"LmPnml6iM0iR..."}}} +""", + formatter_class=argparse.RawDescriptionHelpFormatter, + ) + + parser.add_argument( + "-N", + "--server-name", + help="Name to give as the local homeserver. If unspecified, will be " + "read from the config file.", + ) + + parser.add_argument( + "-k", + "--signing-key-path", + help="Path to the file containing the private ed25519 key to sign the " + "request with.", + ) + + parser.add_argument( + "-K", + "--signing-key", + help="The private ed25519 key to sign the request with.", + ) + + parser.add_argument( + "-c", + "--config", + default="homeserver.yaml", + help=( + "Path to synapse config file, from which the server name and/or signing " + "key path will be read. Ignored if --server-name and --signing-key(-path) " + "are both given." + ), + ) + + parser.add_argument( + "--sign-event-room-version", + type=str, + help=( + "Sign the JSON as an event for the given room version, rather than raw JSON. " + "This means that we will add a 'hashes' object, and redact the event before " + "signing." + ), + ) + + input_args = parser.add_mutually_exclusive_group() + + input_args.add_argument("input_data", nargs="?", help="Raw JSON to be signed.") + + input_args.add_argument( + "-i", + "--input", + type=argparse.FileType("r"), + default=sys.stdin, + help=( + "A file from which to read the JSON to be signed. If neither --input nor " + "input_data are given, JSON will be read from stdin." + ), + ) + + parser.add_argument( + "-o", + "--output", + type=argparse.FileType("w"), + default=sys.stdout, + help="Where to write the signed JSON. Defaults to stdout.", + ) + + args = parser.parse_args() + + if not args.server_name or not (args.signing_key_path or args.signing_key): + read_args_from_config(args) + + if args.signing_key: + keys = read_signing_keys([args.signing_key]) + else: + with open(args.signing_key_path) as f: + keys = read_signing_keys(f) + + json_to_sign = args.input_data + if json_to_sign is None: + json_to_sign = args.input.read() + + try: + obj = json.loads(json_to_sign) + except JSONDecodeError as e: + print("Unable to parse input as JSON: %s" % e, file=sys.stderr) + sys.exit(1) + + if not isinstance(obj, dict): + print("Input json was not an object", file=sys.stderr) + sys.exit(1) + + if args.sign_event_room_version: + room_version = KNOWN_ROOM_VERSIONS.get(args.sign_event_room_version) + if not room_version: + print( + f"Unknown room version {args.sign_event_room_version}", file=sys.stderr + ) + sys.exit(1) + add_hashes_and_signatures(room_version, obj, args.server_name, keys[0]) + else: + sign_json(obj, args.server_name, keys[0]) + + for c in json_encoder.iterencode(obj): + args.output.write(c) + args.output.write("\n") + + +def read_args_from_config(args: argparse.Namespace) -> None: + with open(args.config, "r") as fh: + config = yaml.safe_load(fh) + if not args.server_name: + args.server_name = config["server_name"] + if not args.signing_key_path and not args.signing_key: + if "signing_key" in config: + args.signing_key = config["signing_key"] + elif "signing_key_path" in config: + args.signing_key_path = config["signing_key_path"] + else: + print( + "A signing key must be given on the commandline or in the config file.", + file=sys.stderr, + ) + sys.exit(1) + + +if __name__ == "__main__": + main() diff --git a/tox.ini b/tox.ini index 8d6aa7580b..f4829200cc 100644 --- a/tox.ini +++ b/tox.ini @@ -40,8 +40,6 @@ lint_targets = tests # annoyingly, black doesn't find these so we have to list them scripts-dev - scripts-dev/build_debian_packages - scripts-dev/sign_json stubs contrib synctl -- cgit 1.5.1