From 201178db1aea302d714d6da4dc188e9d103ee474 Mon Sep 17 00:00:00 2001
From: Richard van der Hoff <1389908+richvdh@users.noreply.github.com>
Date: Thu, 18 Mar 2021 20:31:47 +0000
Subject: federation_client: stop adding URL prefix (#9645)

---
 scripts-dev/federation_client.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

(limited to 'scripts-dev')

diff --git a/scripts-dev/federation_client.py b/scripts-dev/federation_client.py
index abcec48c4f..657919a0d7 100755
--- a/scripts-dev/federation_client.py
+++ b/scripts-dev/federation_client.py
@@ -223,7 +223,7 @@ def main():
     parser.add_argument("--body", help="Data to send as the body of the HTTP request")
 
     parser.add_argument(
-        "path", help="request path. We will add '/_matrix/federation/v1/' to this."
+        "path", help="request path, including the '/_matrix/federation/...' prefix."
     )
 
     args = parser.parse_args()
@@ -239,7 +239,7 @@ def main():
         args.server_name,
         key,
         args.destination,
-        "/_matrix/federation/v1/" + args.path,
+        args.path,
         content=args.body,
     )
 
-- 
cgit 1.5.1


From 0e355847347d6a2334dd8879174b1b6806b2a38b Mon Sep 17 00:00:00 2001
From: Richard van der Hoff <1389908+richvdh@users.noreply.github.com>
Date: Thu, 18 Mar 2021 21:12:07 +0000
Subject: federation_client: handle inline signing_keys in hs.yaml (#9647)

---
 changelog.d/9647.misc            |  1 +
 scripts-dev/federation_client.py | 71 ++++++++++------------------------------
 2 files changed, 18 insertions(+), 54 deletions(-)
 create mode 100644 changelog.d/9647.misc

(limited to 'scripts-dev')

diff --git a/changelog.d/9647.misc b/changelog.d/9647.misc
new file mode 100644
index 0000000000..303a8c6606
--- /dev/null
+++ b/changelog.d/9647.misc
@@ -0,0 +1 @@
+In the `federation_client` commandline client, handle inline `signing_key`s in `homeserver.yaml`.
diff --git a/scripts-dev/federation_client.py b/scripts-dev/federation_client.py
index 657919a0d7..6f76c08fcf 100755
--- a/scripts-dev/federation_client.py
+++ b/scripts-dev/federation_client.py
@@ -22,8 +22,8 @@ import sys
 from typing import Any, Optional
 from urllib import parse as urlparse
 
-import nacl.signing
 import requests
+import signedjson.key
 import signedjson.types
 import srvlookup
 import yaml
@@ -44,18 +44,6 @@ def encode_base64(input_bytes):
     return output_string
 
 
-def decode_base64(input_string):
-    """Decode a base64 string to bytes inferring padding from the length of the
-    string."""
-
-    input_bytes = input_string.encode("ascii")
-    input_len = len(input_bytes)
-    padding = b"=" * (3 - ((input_len + 3) % 4))
-    output_len = 3 * ((input_len + 2) // 4) + (input_len + 2) % 4 - 2
-    output_bytes = base64.b64decode(input_bytes + padding)
-    return output_bytes[:output_len]
-
-
 def encode_canonical_json(value):
     return json.dumps(
         value,
@@ -88,42 +76,6 @@ def sign_json(
     return json_object
 
 
-NACL_ED25519 = "ed25519"
-
-
-def decode_signing_key_base64(algorithm, version, key_base64):
-    """Decode a base64 encoded signing key
-    Args:
-        algorithm (str): The algorithm the key is for (currently "ed25519").
-        version (str): Identifies this key out of the keys for this entity.
-        key_base64 (str): Base64 encoded bytes of the key.
-    Returns:
-        A SigningKey object.
-    """
-    if algorithm == NACL_ED25519:
-        key_bytes = decode_base64(key_base64)
-        key = nacl.signing.SigningKey(key_bytes)
-        key.version = version
-        key.alg = NACL_ED25519
-        return key
-    else:
-        raise ValueError("Unsupported algorithm %s" % (algorithm,))
-
-
-def read_signing_keys(stream):
-    """Reads a list of keys from a stream
-    Args:
-        stream : A stream to iterate for keys.
-    Returns:
-        list of SigningKey objects.
-    """
-    keys = []
-    for line in stream:
-        algorithm, version, key_base64 = line.split()
-        keys.append(decode_signing_key_base64(algorithm, version, key_base64))
-    return keys
-
-
 def request(
     method: Optional[str],
     origin_name: str,
@@ -228,11 +180,16 @@ def main():
 
     args = parser.parse_args()
 
-    if not args.server_name or not args.signing_key_path:
+    args.signing_key = None
+    if args.signing_key_path:
+        with open(args.signing_key_path) as f:
+            args.signing_key = f.readline()
+
+    if not args.server_name or not args.signing_key:
         read_args_from_config(args)
 
-    with open(args.signing_key_path) as f:
-        key = read_signing_keys(f)[0]
+    algorithm, version, key_base64 = args.signing_key.split()
+    key = signedjson.key.decode_signing_key_base64(algorithm, version, key_base64)
 
     result = request(
         args.method,
@@ -255,10 +212,16 @@ def main():
 def read_args_from_config(args):
     with open(args.config, "r") as fh:
         config = yaml.safe_load(fh)
+
         if not args.server_name:
             args.server_name = config["server_name"]
-        if not args.signing_key_path:
-            args.signing_key_path = config["signing_key_path"]
+
+        if not args.signing_key:
+            if "signing_key" in config:
+                args.signing_key = config["signing_key"]
+            else:
+                with open(config["signing_key_path"]) as f:
+                    args.signing_key = f.readline()
 
 
 class MatrixConnectionAdapter(HTTPAdapter):
-- 
cgit 1.5.1


From d4c4798a2548a53b63546a176f6dd350c4ad26bc Mon Sep 17 00:00:00 2001
From: Quentin Gliech <quentingliech@gmail.com>
Date: Thu, 25 Mar 2021 17:53:54 +0100
Subject: Use interpreter from $PATH instead of absolute paths in various
 scripts using /usr/bin/env (#9689)

On NixOS, `bash` isn't under `/bin/bash` but rather in some directory in `$PATH`. Locally, I've been patching those scripts to make them work.

`/usr/bin/env` seems to be the only [portable way](https://unix.stackexchange.com/questions/29608/why-is-it-better-to-use-usr-bin-env-name-instead-of-path-to-name-as-my) to use binaries from the PATH as interpreters.

Signed-off-by: Quentin Gliech <quentingliech@gmail.com>
---
 .buildkite/scripts/test_old_deps.sh        | 2 +-
 .buildkite/scripts/test_synapse_port_db.sh | 2 +-
 changelog.d/9689.misc                      | 1 +
 contrib/purge_api/purge_history.sh         | 2 +-
 contrib/purge_api/purge_remote_media.sh    | 2 +-
 demo/clean.sh                              | 2 +-
 demo/start.sh                              | 2 +-
 demo/stop.sh                               | 2 +-
 docker/build_debian.sh                     | 2 +-
 docker/run_pg_tests.sh                     | 2 +-
 scripts-dev/check-newsfragment             | 2 +-
 scripts-dev/config-lint.sh                 | 2 +-
 scripts-dev/generate_sample_config         | 2 +-
 scripts-dev/lint.sh                        | 2 +-
 scripts-dev/make_full_schema.sh            | 2 +-
 scripts-dev/next_github_number.sh          | 4 ++--
 test_postgresql.sh                         | 2 +-
 17 files changed, 18 insertions(+), 17 deletions(-)
 create mode 100644 changelog.d/9689.misc

(limited to 'scripts-dev')

diff --git a/.buildkite/scripts/test_old_deps.sh b/.buildkite/scripts/test_old_deps.sh
index 28e6694b5d..9fe5b696b0 100755
--- a/.buildkite/scripts/test_old_deps.sh
+++ b/.buildkite/scripts/test_old_deps.sh
@@ -1,4 +1,4 @@
-#!/bin/bash
+#!/usr/bin/env bash
 
 # this script is run by buildkite in a plain `xenial` container; it installs the
 # minimal requirements for tox and hands over to the py35-old tox environment.
diff --git a/.buildkite/scripts/test_synapse_port_db.sh b/.buildkite/scripts/test_synapse_port_db.sh
index 9ed2177635..8914319e38 100755
--- a/.buildkite/scripts/test_synapse_port_db.sh
+++ b/.buildkite/scripts/test_synapse_port_db.sh
@@ -1,4 +1,4 @@
-#!/bin/bash
+#!/usr/bin/env bash
 #
 # Test script for 'synapse_port_db', which creates a virtualenv, installs Synapse along
 # with additional dependencies needed for the test (such as coverage or the PostgreSQL
diff --git a/changelog.d/9689.misc b/changelog.d/9689.misc
new file mode 100644
index 0000000000..a08d3482ad
--- /dev/null
+++ b/changelog.d/9689.misc
@@ -0,0 +1 @@
+Use interpreter from `$PATH` via `/usr/bin/env` instead of absolute paths in various scripts.
diff --git a/contrib/purge_api/purge_history.sh b/contrib/purge_api/purge_history.sh
index e7dd5d6468..c45136ff53 100644
--- a/contrib/purge_api/purge_history.sh
+++ b/contrib/purge_api/purge_history.sh
@@ -1,4 +1,4 @@
-#!/bin/bash
+#!/usr/bin/env bash
 
 # this script will use the api:
 #    https://github.com/matrix-org/synapse/blob/master/docs/admin_api/purge_history_api.rst
diff --git a/contrib/purge_api/purge_remote_media.sh b/contrib/purge_api/purge_remote_media.sh
index 77220d3bd5..4930d9529c 100644
--- a/contrib/purge_api/purge_remote_media.sh
+++ b/contrib/purge_api/purge_remote_media.sh
@@ -1,4 +1,4 @@
-#!/bin/bash
+#!/usr/bin/env bash
 
 DOMAIN=yourserver.tld
 # add this user as admin in your home server:
diff --git a/demo/clean.sh b/demo/clean.sh
index 418ca9457e..6b809f6e83 100755
--- a/demo/clean.sh
+++ b/demo/clean.sh
@@ -1,4 +1,4 @@
-#!/bin/bash
+#!/usr/bin/env bash
 
 set -e
 
diff --git a/demo/start.sh b/demo/start.sh
index f6b5ea137f..621a5698b8 100755
--- a/demo/start.sh
+++ b/demo/start.sh
@@ -1,4 +1,4 @@
-#!/bin/bash
+#!/usr/bin/env bash
 
 DIR="$( cd "$( dirname "$0" )" && pwd )"
 
diff --git a/demo/stop.sh b/demo/stop.sh
index 85a1d2c161..f9dddc5914 100755
--- a/demo/stop.sh
+++ b/demo/stop.sh
@@ -1,4 +1,4 @@
-#!/bin/bash
+#!/usr/bin/env bash
 
 DIR="$( cd "$( dirname "$0" )" && pwd )"
 
diff --git a/docker/build_debian.sh b/docker/build_debian.sh
index f312f0715f..f426d2b77b 100644
--- a/docker/build_debian.sh
+++ b/docker/build_debian.sh
@@ -1,4 +1,4 @@
-#!/bin/bash
+#!/usr/bin/env bash
 
 # The script to build the Debian package, as ran inside the Docker image.
 
diff --git a/docker/run_pg_tests.sh b/docker/run_pg_tests.sh
index d18d1e4c8e..1fd08cb62b 100755
--- a/docker/run_pg_tests.sh
+++ b/docker/run_pg_tests.sh
@@ -1,4 +1,4 @@
-#!/bin/bash
+#!/usr/bin/env bash
 
 # This script runs the PostgreSQL tests inside a Docker container. It expects
 # the relevant source files to be mounted into /src (done automatically by the
diff --git a/scripts-dev/check-newsfragment b/scripts-dev/check-newsfragment
index 448cadb829..af6d32e332 100755
--- a/scripts-dev/check-newsfragment
+++ b/scripts-dev/check-newsfragment
@@ -1,4 +1,4 @@
-#!/bin/bash
+#!/usr/bin/env bash
 #
 # A script which checks that an appropriate news file has been added on this
 # branch.
diff --git a/scripts-dev/config-lint.sh b/scripts-dev/config-lint.sh
index 9132160463..8c6323e59a 100755
--- a/scripts-dev/config-lint.sh
+++ b/scripts-dev/config-lint.sh
@@ -1,4 +1,4 @@
-#!/bin/bash
+#!/usr/bin/env bash
 # Find linting errors in Synapse's default config file.
 # Exits with 0 if there are no problems, or another code otherwise.
 
diff --git a/scripts-dev/generate_sample_config b/scripts-dev/generate_sample_config
index 9cb4630a5c..02739894b5 100755
--- a/scripts-dev/generate_sample_config
+++ b/scripts-dev/generate_sample_config
@@ -1,4 +1,4 @@
-#!/bin/bash
+#!/usr/bin/env bash
 #
 # Update/check the docs/sample_config.yaml
 
diff --git a/scripts-dev/lint.sh b/scripts-dev/lint.sh
index fe2965cd36..9761e97594 100755
--- a/scripts-dev/lint.sh
+++ b/scripts-dev/lint.sh
@@ -1,4 +1,4 @@
-#!/bin/bash
+#!/usr/bin/env bash
 #
 # Runs linting scripts over the local Synapse checkout
 # isort - sorts import statements
diff --git a/scripts-dev/make_full_schema.sh b/scripts-dev/make_full_schema.sh
index b8d1e636f1..bc8f978660 100755
--- a/scripts-dev/make_full_schema.sh
+++ b/scripts-dev/make_full_schema.sh
@@ -1,4 +1,4 @@
-#!/bin/bash
+#!/usr/bin/env bash
 #
 # This script generates SQL files for creating a brand new Synapse DB with the latest
 # schema, on both SQLite3 and Postgres.
diff --git a/scripts-dev/next_github_number.sh b/scripts-dev/next_github_number.sh
index 376280025a..00e9b14569 100755
--- a/scripts-dev/next_github_number.sh
+++ b/scripts-dev/next_github_number.sh
@@ -1,4 +1,4 @@
-#!/bin/bash
+#!/usr/bin/env bash
 
 set -e
 
@@ -6,4 +6,4 @@ set -e
 # next PR number.
 CURRENT_NUMBER=`curl -s "https://api.github.com/repos/matrix-org/synapse/issues?state=all&per_page=1" | jq -r ".[0].number"`
 CURRENT_NUMBER=$((CURRENT_NUMBER+1))
-echo $CURRENT_NUMBER
\ No newline at end of file
+echo $CURRENT_NUMBER
diff --git a/test_postgresql.sh b/test_postgresql.sh
index 1ffcaabd31..c10828fbbc 100755
--- a/test_postgresql.sh
+++ b/test_postgresql.sh
@@ -1,4 +1,4 @@
-#!/bin/bash
+#!/usr/bin/env bash
 
 # This script builds the Docker image to run the PostgreSQL tests, and then runs
 # the tests.
-- 
cgit 1.5.1


From f380bb77d19e7dd4ee6f61cd489e240ee4aa8fc2 Mon Sep 17 00:00:00 2001
From: Andrew Morgan <1342360+anoadragon453@users.noreply.github.com>
Date: Tue, 30 Mar 2021 10:30:43 +0100
Subject: Use 'dmypy run' in lint.sh instead of 'mypy' (#9701)

For it's obvious performance benefits. `dmypy` support landed in #9692.
---
 changelog.d/9701.misc | 1 +
 scripts-dev/lint.sh   | 2 +-
 2 files changed, 2 insertions(+), 1 deletion(-)
 create mode 100644 changelog.d/9701.misc

(limited to 'scripts-dev')

diff --git a/changelog.d/9701.misc b/changelog.d/9701.misc
new file mode 100644
index 0000000000..49fa7b3593
--- /dev/null
+++ b/changelog.d/9701.misc
@@ -0,0 +1 @@
+Use `dmypy run` in lint script for improved performance in type-checking while developing.
\ No newline at end of file
diff --git a/scripts-dev/lint.sh b/scripts-dev/lint.sh
index 9761e97594..41415ee07b 100755
--- a/scripts-dev/lint.sh
+++ b/scripts-dev/lint.sh
@@ -95,4 +95,4 @@ isort "${files[@]}"
 python3 -m black "${files[@]}"
 ./scripts-dev/config-lint.sh
 flake8 "${files[@]}"
-mypy
+dmypy run
-- 
cgit 1.5.1


From ac99774dacc31907ace76c6eda142b835a0dda25 Mon Sep 17 00:00:00 2001
From: Andrew Morgan <1342360+anoadragon453@users.noreply.github.com>
Date: Wed, 31 Mar 2021 11:58:12 +0100
Subject: Rewrite complement.sh (#9685)

This PR rewrites the original complement.sh script with a number of improvements:

* We can now use a local checkout of Complement (configurable with `COMPLEMENT_DIR`), though the default behaviour still downloads the master branch.
* You can now specify a regex of test names to run, or just run all tests.
* We now use the Synapse test blacklist tag (so all tests will pass).
---
 changelog.d/9685.misc     |  1 +
 scripts-dev/complement.sh | 49 ++++++++++++++++++++++++++++++++++++-----------
 2 files changed, 39 insertions(+), 11 deletions(-)
 create mode 100644 changelog.d/9685.misc

(limited to 'scripts-dev')

diff --git a/changelog.d/9685.misc b/changelog.d/9685.misc
new file mode 100644
index 0000000000..0506d8af0c
--- /dev/null
+++ b/changelog.d/9685.misc
@@ -0,0 +1 @@
+Update `scripts-dev/complement.sh` to use a local checkout of Complement, allow running a subset of tests and have it use Synapse's Complement test blacklist.
\ No newline at end of file
diff --git a/scripts-dev/complement.sh b/scripts-dev/complement.sh
index 3cde53f5c0..31cc20a826 100755
--- a/scripts-dev/complement.sh
+++ b/scripts-dev/complement.sh
@@ -1,22 +1,49 @@
-#! /bin/bash -eu
+#!/usr/bin/env bash
 # This script is designed for developers who want to test their code
 # against Complement.
 #
 # It makes a Synapse image which represents the current checkout,
-# then downloads Complement and runs it with that image.
+# builds a synapse-complement image on top, then runs tests with it.
+#
+# By default the script will fetch the latest Complement master branch and
+# run tests with that. This can be overridden to use a custom Complement
+# checkout by setting the COMPLEMENT_DIR environment variable to the
+# filepath of a local Complement checkout.
+#
+# A regular expression of test method names can be supplied as the first
+# argument to the script. Complement will then only run those tests. If
+# no regex is supplied, all tests are run. For example;
+#
+# ./complement.sh "TestOutboundFederation(Profile|Send)"
+#
+
+# Exit if a line returns a non-zero exit code
+set -e
 
+# Change to the repository root
 cd "$(dirname $0)/.."
 
+# Check for a user-specified Complement checkout
+if [[ -z "$COMPLEMENT_DIR" ]]; then
+  echo "COMPLEMENT_DIR not set. Fetching the latest Complement checkout..."
+  wget -Nq https://github.com/matrix-org/complement/archive/master.tar.gz
+  tar -xzf master.tar.gz
+  COMPLEMENT_DIR=complement-master
+  echo "Checkout available at 'complement-master'"
+fi
+
 # Build the base Synapse image from the local checkout
-docker build -t matrixdotorg/synapse:latest -f docker/Dockerfile .
+docker build -t matrixdotorg/synapse -f docker/Dockerfile .
+# Build the Synapse monolith image from Complement, based on the above image we just built
+docker build -t complement-synapse -f "$COMPLEMENT_DIR/dockerfiles/Synapse.Dockerfile" "$COMPLEMENT_DIR/dockerfiles"
 
-# Download Complement
-wget -N https://github.com/matrix-org/complement/archive/master.tar.gz
-tar -xzf master.tar.gz
-cd complement-master
+cd "$COMPLEMENT_DIR"
 
-# Build the Synapse image from Complement, based on the above image we just built
-docker build -t complement-synapse -f dockerfiles/Synapse.Dockerfile ./dockerfiles
+EXTRA_COMPLEMENT_ARGS=""
+if [[ -n "$1" ]]; then
+  # A test name regex has been set, supply it to Complement
+  EXTRA_COMPLEMENT_ARGS+="-run $1 "
+fi
 
-# Run the tests on the resulting image!
-COMPLEMENT_BASE_IMAGE=complement-synapse go test -v -count=1 ./tests
+# Run the tests!
+COMPLEMENT_BASE_IMAGE=complement-synapse go test -v -tags synapse_blacklist -count=1 $EXTRA_COMPLEMENT_ARGS ./tests
-- 
cgit 1.5.1


From 5fe38e07e74869574b79f0d91e3d607069313352 Mon Sep 17 00:00:00 2001
From: Patrick Cloke <clokep@users.noreply.github.com>
Date: Wed, 31 Mar 2021 14:17:52 -0400
Subject: Revert "Use 'dmypy run' in lint.sh instead of 'mypy' (#9701)" (#9720)

---
 changelog.d/9720.misc | 1 +
 scripts-dev/lint.sh   | 2 +-
 2 files changed, 2 insertions(+), 1 deletion(-)
 create mode 100644 changelog.d/9720.misc

(limited to 'scripts-dev')

diff --git a/changelog.d/9720.misc b/changelog.d/9720.misc
new file mode 100644
index 0000000000..9dd0bc9cff
--- /dev/null
+++ b/changelog.d/9720.misc
@@ -0,0 +1 @@
+Revert using `dmypy run` in lint script.
diff --git a/scripts-dev/lint.sh b/scripts-dev/lint.sh
index 41415ee07b..9761e97594 100755
--- a/scripts-dev/lint.sh
+++ b/scripts-dev/lint.sh
@@ -95,4 +95,4 @@ isort "${files[@]}"
 python3 -m black "${files[@]}"
 ./scripts-dev/config-lint.sh
 flake8 "${files[@]}"
-dmypy run
+mypy
-- 
cgit 1.5.1