diff options
Diffstat (limited to 'scripts-dev')
| -rwxr-xr-x | scripts-dev/build_debian_packages | 154 | ||||
| -rwxr-xr-x | scripts-dev/check-newsfragment | 36 | ||||
| -rwxr-xr-x | scripts-dev/federation_client.py | 9 | ||||
| -rwxr-xr-x | scripts-dev/make_identicons.pl | 39 |
4 files changed, 197 insertions, 41 deletions
diff --git a/scripts-dev/build_debian_packages b/scripts-dev/build_debian_packages new file mode 100755 index 0000000000..6b9be99060 --- /dev/null +++ b/scripts-dev/build_debian_packages @@ -0,0 +1,154 @@ +#!/usr/bin/env python3 + +# Build the Debian packages using Docker images. +# +# This script builds the Docker images and then executes them sequentially, each +# one building a Debian package for the targeted operating system. It is +# designed to be a "single command" to produce all the images. +# +# By default, builds for all known distributions, but a list of distributions +# can be passed on the commandline for debugging. + +import argparse +import os +import signal +import subprocess +import sys +import threading +from concurrent.futures import ThreadPoolExecutor + +DISTS = ( + "debian:stretch", + "debian:buster", + "debian:sid", + "ubuntu:xenial", + "ubuntu:bionic", + "ubuntu:cosmic", +) + +DESC = '''\ +Builds .debs for synapse, using a Docker image for the build environment. + +By default, builds for all known distributions, but a list of distributions +can be passed on the commandline for debugging. +''' + + +class Builder(object): + def __init__(self, redirect_stdout=False): + self.redirect_stdout = redirect_stdout + self.active_containers = set() + self._lock = threading.Lock() + self._failed = False + + def run_build(self, dist): + """Build deb for a single distribution""" + + if self._failed: + print("not building %s due to earlier failure" % (dist, )) + raise Exception("failed") + + try: + self._inner_build(dist) + except Exception as e: + print("build of %s failed: %s" % (dist, e), file=sys.stderr) + self._failed = True + raise + + def _inner_build(self, dist): + projdir = os.path.dirname(os.path.dirname(os.path.realpath(__file__))) + os.chdir(projdir) + + tag = dist.split(":", 1)[1] + + # Make the dir where the debs will live. + # + # Note that we deliberately put this outside the source tree, otherwise + # we tend to get source packages which are full of debs. (We could hack + # around that with more magic in the build_debian.sh script, but that + # doesn't solve the problem for natively-run dpkg-buildpakage). + debsdir = os.path.join(projdir, '../debs') + os.makedirs(debsdir, exist_ok=True) + + if self.redirect_stdout: + logfile = os.path.join(debsdir, "%s.buildlog" % (tag, )) + print("building %s: directing output to %s" % (dist, logfile)) + stdout = open(logfile, "w") + else: + stdout = None + + # first build a docker image for the build environment + subprocess.check_call([ + "docker", "build", + "--tag", "dh-venv-builder:" + tag, + "--build-arg", "distro=" + dist, + "-f", "docker/Dockerfile-dhvirtualenv", + "docker", + ], stdout=stdout, stderr=subprocess.STDOUT) + + container_name = "synapse_build_" + tag + with self._lock: + self.active_containers.add(container_name) + + # then run the build itself + subprocess.check_call([ + "docker", "run", + "--rm", + "--name", container_name, + "--volume=" + projdir + ":/synapse/source:ro", + "--volume=" + debsdir + ":/debs", + "-e", "TARGET_USERID=%i" % (os.getuid(), ), + "-e", "TARGET_GROUPID=%i" % (os.getgid(), ), + "dh-venv-builder:" + tag, + ], stdout=stdout, stderr=subprocess.STDOUT) + + with self._lock: + self.active_containers.remove(container_name) + + if stdout is not None: + stdout.close() + print("Completed build of %s" % (dist, )) + + def kill_containers(self): + with self._lock: + active = list(self.active_containers) + + for c in active: + print("killing container %s" % (c,)) + subprocess.run([ + "docker", "kill", c, + ], stdout=subprocess.DEVNULL) + with self._lock: + self.active_containers.remove(c) + + +def run_builds(dists, jobs=1): + builder = Builder(redirect_stdout=(jobs > 1)) + + def sig(signum, _frame): + print("Caught SIGINT") + builder.kill_containers() + signal.signal(signal.SIGINT, sig) + + with ThreadPoolExecutor(max_workers=jobs) as e: + res = e.map(builder.run_build, dists) + + # make sure we consume the iterable so that exceptions are raised. + for r in res: + pass + + +if __name__ == '__main__': + parser = argparse.ArgumentParser( + description=DESC, + ) + parser.add_argument( + '-j', '--jobs', type=int, default=1, + help='specify the number of builds to run in parallel', + ) + parser.add_argument( + 'dist', nargs='*', default=DISTS, + help='a list of distributions to build for. Default: %(default)s', + ) + args = parser.parse_args() + run_builds(dists=args.dist, jobs=args.jobs) diff --git a/scripts-dev/check-newsfragment b/scripts-dev/check-newsfragment new file mode 100755 index 0000000000..5da093e168 --- /dev/null +++ b/scripts-dev/check-newsfragment @@ -0,0 +1,36 @@ +#!/bin/bash +# +# A script which checks that an appropriate news file has been added on this +# branch. + +set -e + +# make sure that origin/develop is up to date +git fetch origin develop + +UPSTREAM=origin/develop + +# if there are changes in the debian directory, check that the debian changelog +# has been updated +if ! git diff --quiet $UPSTREAM... -- debian; then + if git diff --quiet $UPSTREAM... -- debian/changelog; then + echo "Updates to debian directory, but no update to the changelog." >&2 + exit 1 + fi +fi + +# if there are changes *outside* the debian directory, check that the +# newsfragments have been updated. +if git diff --name-only $UPSTREAM... | grep -qv '^develop/'; then + tox -e check-newsfragment +fi + +# check that any new newsfiles on this branch end with a full stop. +for f in git diff --name-only $UPSTREAM... -- changelog.d; do + lastchar=`tr -d '\n' < $f | tail -c 1` + if [ $lastchar != '.' ]; then + echo "Newsfragment $f does not end with a '.'" >&2 + exit 1 + fi +done + diff --git a/scripts-dev/federation_client.py b/scripts-dev/federation_client.py index 2566ce7cef..e0287c8c6c 100755 --- a/scripts-dev/federation_client.py +++ b/scripts-dev/federation_client.py @@ -154,10 +154,15 @@ def request_json(method, origin_name, origin_key, destination, path, content): s = requests.Session() s.mount("matrix://", MatrixConnectionAdapter()) + headers = {"Host": destination, "Authorization": authorization_headers[0]} + + if method == "POST": + headers["Content-Type"] = "application/json" + result = s.request( method=method, url=dest, - headers={"Host": destination, "Authorization": authorization_headers[0]}, + headers=headers, verify=False, data=content, ) @@ -203,7 +208,7 @@ def main(): parser.add_argument( "-X", "--method", - help="HTTP method to use for the request. Defaults to GET if --data is" + help="HTTP method to use for the request. Defaults to GET if --body is" "unspecified, POST if it is.", ) diff --git a/scripts-dev/make_identicons.pl b/scripts-dev/make_identicons.pl deleted file mode 100755 index cbff63e298..0000000000 --- a/scripts-dev/make_identicons.pl +++ /dev/null @@ -1,39 +0,0 @@ -#!/usr/bin/env perl - -use strict; -use warnings; - -use DBI; -use DBD::SQLite; -use JSON; -use Getopt::Long; - -my $db; # = "homeserver.db"; -my $server = "http://localhost:8008"; -my $size = 320; - -GetOptions("db|d=s", \$db, - "server|s=s", \$server, - "width|w=i", \$size) or usage(); - -usage() unless $db; - -my $dbh = DBI->connect("dbi:SQLite:dbname=$db","","") || die $DBI::errstr; - -my $res = $dbh->selectall_arrayref("select token, name from access_tokens, users where access_tokens.user_id = users.id group by user_id") || die $DBI::errstr; - -foreach (@$res) { - my ($token, $mxid) = ($_->[0], $_->[1]); - my ($user_id) = ($mxid =~ m/@(.*):/); - my ($url) = $dbh->selectrow_array("select avatar_url from profiles where user_id=?", undef, $user_id); - if (!$url || $url =~ /#auto$/) { - `curl -s -o tmp.png "$server/_matrix/media/v1/identicon?name=${mxid}&width=$size&height=$size"`; - my $json = `curl -s -X POST -H "Content-Type: image/png" -T "tmp.png" $server/_matrix/media/v1/upload?access_token=$token`; - my $content_uri = from_json($json)->{content_uri}; - `curl -X PUT -H "Content-Type: application/json" --data '{ "avatar_url": "${content_uri}#auto"}' $server/_matrix/client/api/v1/profile/${mxid}/avatar_url?access_token=$token`; - } -} - -sub usage { - die "usage: ./make-identicons.pl\n\t-d database [e.g. homeserver.db]\n\t-s homeserver (default: http://localhost:8008)\n\t-w identicon size in pixels (default 320)"; -} \ No newline at end of file |