Merge branch 'develop' of github.com:matrix-org/synapse into anoa/public_rooms_federate

4 files changed, 197 insertions, 41 deletions

diff --git a/scripts-dev/build_debian_packages b/scripts-dev/build_debian_packages
new file mode 100755
index 0000000000..6b9be99060
--- /dev/null
+++ b/scripts-dev/build_debian_packages
@@ -0,0 +1,154 @@
+#!/usr/bin/env python3
+
+# Build the Debian packages using Docker images.
+#
+# This script builds the Docker images and then executes them sequentially, each
+# one building a Debian package for the targeted operating system. It is
+# designed to be a "single command" to produce all the images.
+#
+# By default, builds for all known distributions, but a list of distributions
+# can be passed on the commandline for debugging.
+
+import argparse
+import os
+import signal
+import subprocess
+import sys
+import threading
+from concurrent.futures import ThreadPoolExecutor
+
+DISTS = (
+    "debian:stretch",
+    "debian:buster",
+    "debian:sid",
+    "ubuntu:xenial",
+    "ubuntu:bionic",
+    "ubuntu:cosmic",
+)
+
+DESC = '''\
+Builds .debs for synapse, using a Docker image for the build environment.
+
+By default, builds for all known distributions, but a list of distributions
+can be passed on the commandline for debugging.
+'''
+
+
+class Builder(object):
+    def __init__(self, redirect_stdout=False):
+        self.redirect_stdout = redirect_stdout
+        self.active_containers = set()
+        self._lock = threading.Lock()
+        self._failed = False
+
+    def run_build(self, dist):
+        """Build deb for a single distribution"""
+
+        if self._failed:
+            print("not building %s due to earlier failure" % (dist, ))
+            raise Exception("failed")
+
+        try:
+            self._inner_build(dist)
+        except Exception as e:
+            print("build of %s failed: %s" % (dist, e), file=sys.stderr)
+            self._failed = True
+            raise
+
+    def _inner_build(self, dist):
+        projdir = os.path.dirname(os.path.dirname(os.path.realpath(__file__)))
+        os.chdir(projdir)
+
+        tag = dist.split(":", 1)[1]
+
+        # Make the dir where the debs will live.
+        #
+        # Note that we deliberately put this outside the source tree, otherwise
+        # we tend to get source packages which are full of debs. (We could hack
+        # around that with more magic in the build_debian.sh script, but that
+        # doesn't solve the problem for natively-run dpkg-buildpakage).
+        debsdir = os.path.join(projdir, '../debs')
+        os.makedirs(debsdir, exist_ok=True)
+
+        if self.redirect_stdout:
+            logfile = os.path.join(debsdir, "%s.buildlog" % (tag, ))
+            print("building %s: directing output to %s" % (dist, logfile))
+            stdout = open(logfile, "w")
+        else:
+            stdout = None
+
+        # first build a docker image for the build environment
+        subprocess.check_call([
+            "docker", "build",
+            "--tag", "dh-venv-builder:" + tag,
+            "--build-arg", "distro=" + dist,
+            "-f", "docker/Dockerfile-dhvirtualenv",
+            "docker",
+        ], stdout=stdout, stderr=subprocess.STDOUT)
+
+        container_name = "synapse_build_" + tag
+        with self._lock:
+            self.active_containers.add(container_name)
+
+        # then run the build itself
+        subprocess.check_call([
+            "docker", "run",
+            "--rm",
+            "--name", container_name,
+            "--volume=" + projdir + ":/synapse/source:ro",
+            "--volume=" + debsdir + ":/debs",
+            "-e", "TARGET_USERID=%i" % (os.getuid(), ),
+            "-e", "TARGET_GROUPID=%i" % (os.getgid(), ),
+            "dh-venv-builder:" + tag,
+        ], stdout=stdout, stderr=subprocess.STDOUT)
+
+        with self._lock:
+            self.active_containers.remove(container_name)
+
+        if stdout is not None:
+            stdout.close()
+            print("Completed build of %s" % (dist, ))
+
+    def kill_containers(self):
+        with self._lock:
+            active = list(self.active_containers)
+
+        for c in active:
+            print("killing container %s" % (c,))
+            subprocess.run([
+                "docker", "kill", c,
+            ], stdout=subprocess.DEVNULL)
+            with self._lock:
+                self.active_containers.remove(c)
+
+
+def run_builds(dists, jobs=1):
+    builder = Builder(redirect_stdout=(jobs > 1))
+
+    def sig(signum, _frame):
+        print("Caught SIGINT")
+        builder.kill_containers()
+    signal.signal(signal.SIGINT, sig)
+
+    with ThreadPoolExecutor(max_workers=jobs) as e:
+        res = e.map(builder.run_build, dists)
+
+    # make sure we consume the iterable so that exceptions are raised.
+    for r in res:
+        pass
+
+
+if __name__ == '__main__':
+    parser = argparse.ArgumentParser(
+        description=DESC,
+    )
+    parser.add_argument(
+        '-j', '--jobs', type=int, default=1,
+        help='specify the number of builds to run in parallel',
+    )
+    parser.add_argument(
+        'dist', nargs='*', default=DISTS,
+        help='a list of distributions to build for. Default: %(default)s',
+    )
+    args = parser.parse_args()
+    run_builds(dists=args.dist, jobs=args.jobs)
diff --git a/scripts-dev/check-newsfragment b/scripts-dev/check-newsfragment
new file mode 100755
index 0000000000..5da093e168
--- /dev/null
+++ b/scripts-dev/check-newsfragment
@@ -0,0 +1,36 @@
+#!/bin/bash
+#
+# A script which checks that an appropriate news file has been added on this
+# branch.
+
+set -e
+
+# make sure that origin/develop is up to date
+git fetch origin develop
+
+UPSTREAM=origin/develop
+
+# if there are changes in the debian directory, check that the debian changelog
+# has been updated
+if ! git diff --quiet $UPSTREAM... -- debian; then
+    if git diff --quiet $UPSTREAM... -- debian/changelog; then
+        echo "Updates to debian directory, but no update to the changelog." >&2
+        exit 1
+    fi
+fi
+
+# if there are changes *outside* the debian directory, check that the
+# newsfragments have been updated.
+if git diff --name-only $UPSTREAM... | grep -qv '^develop/'; then
+    tox -e check-newsfragment
+fi
+
+# check that any new newsfiles on this branch end with a full stop.
+for f in git diff --name-only $UPSTREAM... -- changelog.d; do
+    lastchar=`tr -d '\n' < $f | tail -c 1`
+    if [ $lastchar != '.' ]; then
+        echo "Newsfragment $f does not end with a '.'" >&2
+        exit 1
+    fi
+done
+
diff --git a/scripts-dev/federation_client.py b/scripts-dev/federation_client.py
index 2566ce7cef..e0287c8c6c 100755
--- a/scripts-dev/federation_client.py
+++ b/scripts-dev/federation_client.py
@@ -154,10 +154,15 @@ def request_json(method, origin_name, origin_key, destination, path, content):
     s = requests.Session()
     s.mount("matrix://", MatrixConnectionAdapter())
 
+    headers = {"Host": destination, "Authorization": authorization_headers[0]}
+
+    if method == "POST":
+        headers["Content-Type"] = "application/json"
+
     result = s.request(
         method=method,
         url=dest,
-        headers={"Host": destination, "Authorization": authorization_headers[0]},
+        headers=headers,
         verify=False,
         data=content,
     )
@@ -203,7 +208,7 @@ def main():
     parser.add_argument(
         "-X",
         "--method",
-        help="HTTP method to use for the request. Defaults to GET if --data is"
+        help="HTTP method to use for the request. Defaults to GET if --body is"
         "unspecified, POST if it is.",
     )
 
diff --git a/scripts-dev/make_identicons.pl b/scripts-dev/make_identicons.pl
deleted file mode 100755
index cbff63e298..0000000000
--- a/scripts-dev/make_identicons.pl
+++ /dev/null
@@ -1,39 +0,0 @@
-#!/usr/bin/env perl
-
-use strict;
-use warnings;
-
-use DBI;
-use DBD::SQLite;
-use JSON;
-use Getopt::Long;
-
-my $db; # = "homeserver.db";
-my $server = "http://localhost:8008";
-my $size = 320;
-
-GetOptions("db|d=s",     \$db,
-           "server|s=s", \$server,
-           "width|w=i",  \$size) or usage();
-
-usage() unless $db;
-
-my $dbh = DBI->connect("dbi:SQLite:dbname=$db","","") || die $DBI::errstr;
-
-my $res = $dbh->selectall_arrayref("select token, name from access_tokens, users where access_tokens.user_id = users.id group by user_id") || die $DBI::errstr;
-
-foreach (@$res) {
-    my ($token, $mxid) = ($_->[0], $_->[1]);
-    my ($user_id) = ($mxid =~ m/@(.*):/);
-    my ($url) = $dbh->selectrow_array("select avatar_url from profiles where user_id=?", undef, $user_id);
-    if (!$url || $url =~ /#auto$/) {
-        `curl -s -o tmp.png "$server/_matrix/media/v1/identicon?name=${mxid}&width=$size&height=$size"`;
-        my $json = `curl -s -X POST -H "Content-Type: image/png" -T "tmp.png" $server/_matrix/media/v1/upload?access_token=$token`;
-        my $content_uri = from_json($json)->{content_uri};
-        `curl -X PUT -H "Content-Type: application/json" --data '{ "avatar_url": "${content_uri}#auto"}' $server/_matrix/client/api/v1/profile/${mxid}/avatar_url?access_token=$token`;
-    }
-}
-
-sub usage {
-    die "usage: ./make-identicons.pl\n\t-d database [e.g. homeserver.db]\n\t-s homeserver (default: http://localhost:8008)\n\t-w identicon size in pixels (default 320)";
-}
\ No newline at end of file