diff --git a/scripts-dev/build_debian_packages b/scripts-dev/build_debian_packages
index d055cf3287..d0685c8b35 100755
--- a/scripts-dev/build_debian_packages
+++ b/scripts-dev/build_debian_packages
@@ -25,6 +25,7 @@ DISTS = (
"ubuntu:xenial",
"ubuntu:bionic",
"ubuntu:focal",
+ "ubuntu:groovy",
)
DESC = '''\
diff --git a/scripts-dev/check-newsfragment b/scripts-dev/check-newsfragment
index b55368ea6b..d742c522b5 100755
--- a/scripts-dev/check-newsfragment
+++ b/scripts-dev/check-newsfragment
@@ -3,6 +3,8 @@
# A script which checks that an appropriate news file has been added on this
# branch.
+echo -e "+++ \033[32mChecking newsfragment\033[m"
+
set -e
# make sure that origin/dinsic is up to date
@@ -16,6 +18,8 @@ pr="$BUILDKITE_PULL_REQUEST"
if ! git diff --quiet FETCH_HEAD... -- debian; then
if git diff --quiet FETCH_HEAD... -- debian/changelog; then
echo "Updates to debian directory, but no update to the changelog." >&2
+ echo "!! Please see the contributing guide for help writing your changelog entry:" >&2
+ echo "https://github.com/matrix-org/synapse/blob/develop/CONTRIBUTING.md#debian-changelog" >&2
exit 1
fi
fi
@@ -26,7 +30,12 @@ if ! git diff --name-only FETCH_HEAD... | grep -qv '^debian/'; then
exit 0
fi
-tox -qe check-newsfragment
+# Print a link to the contributing guide if the user makes a mistake
+CONTRIBUTING_GUIDE_TEXT="!! Please see the contributing guide for help writing your changelog entry:
+https://github.com/matrix-org/synapse/blob/develop/CONTRIBUTING.md#changelog"
+
+# If check-newsfragment returns a non-zero exit code, print the contributing guide and exit
+tox -qe check-newsfragment || (echo -e "$CONTRIBUTING_GUIDE_TEXT" >&2 && exit 1)
echo
echo "--------------------------"
@@ -38,6 +47,7 @@ for f in `git diff --name-only FETCH_HEAD... -- changelog.d`; do
lastchar=`tr -d '\n' < $f | tail -c 1`
if [ $lastchar != '.' -a $lastchar != '!' ]; then
echo -e "\e[31mERROR: newsfragment $f does not end with a '.' or '!'\e[39m" >&2
+ echo -e "$CONTRIBUTING_GUIDE_TEXT" >&2
exit 1
fi
@@ -47,5 +57,6 @@ done
if [[ -n "$pr" && "$matched" -eq 0 ]]; then
echo -e "\e[31mERROR: Did not find a news fragment with the right number: expected changelog.d/$pr.*.\e[39m" >&2
+ echo -e "$CONTRIBUTING_GUIDE_TEXT" >&2
exit 1
fi
diff --git a/scripts-dev/check_line_terminators.sh b/scripts-dev/check_line_terminators.sh
new file mode 100755
index 0000000000..c983956231
--- /dev/null
+++ b/scripts-dev/check_line_terminators.sh
@@ -0,0 +1,34 @@
+#!/bin/bash
+#
+# Copyright 2020 The Matrix.org Foundation C.I.C.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+# This script checks that line terminators in all repository files (excluding
+# those in the .git directory) feature unix line terminators.
+#
+# Usage:
+#
+# ./check_line_terminators.sh
+#
+# The script will emit exit code 1 if any files that do not use unix line
+# terminators are found, 0 otherwise.
+
+# cd to the root of the repository
+cd `dirname $0`/..
+
+# Find and print files with non-unix line terminators
+if find . -path './.git/*' -prune -o -type f -print0 | xargs -0 grep -I -l $'\r$'; then
+ echo -e '\e[31mERROR: found files with CRLF line endings. See above.\e[39m'
+ exit 1
+fi
diff --git a/scripts-dev/complement.sh b/scripts-dev/complement.sh
new file mode 100755
index 0000000000..3cde53f5c0
--- /dev/null
+++ b/scripts-dev/complement.sh
@@ -0,0 +1,22 @@
+#! /bin/bash -eu
+# This script is designed for developers who want to test their code
+# against Complement.
+#
+# It makes a Synapse image which represents the current checkout,
+# then downloads Complement and runs it with that image.
+
+cd "$(dirname $0)/.."
+
+# Build the base Synapse image from the local checkout
+docker build -t matrixdotorg/synapse:latest -f docker/Dockerfile .
+
+# Download Complement
+wget -N https://github.com/matrix-org/complement/archive/master.tar.gz
+tar -xzf master.tar.gz
+cd complement-master
+
+# Build the Synapse image from Complement, based on the above image we just built
+docker build -t complement-synapse -f dockerfiles/Synapse.Dockerfile ./dockerfiles
+
+# Run the tests on the resulting image!
+COMPLEMENT_BASE_IMAGE=complement-synapse go test -v -count=1 ./tests
diff --git a/scripts-dev/definitions.py b/scripts-dev/definitions.py
index 9eddb6d515..313860df13 100755
--- a/scripts-dev/definitions.py
+++ b/scripts-dev/definitions.py
@@ -1,7 +1,5 @@
#! /usr/bin/python
-from __future__ import print_function
-
import argparse
import ast
import os
@@ -13,7 +11,7 @@ import yaml
class DefinitionVisitor(ast.NodeVisitor):
def __init__(self):
- super(DefinitionVisitor, self).__init__()
+ super().__init__()
self.functions = {}
self.classes = {}
self.names = {}
diff --git a/scripts-dev/dump_macaroon.py b/scripts-dev/dump_macaroon.py
index 22b30fa78e..980b5e709f 100755
--- a/scripts-dev/dump_macaroon.py
+++ b/scripts-dev/dump_macaroon.py
@@ -1,7 +1,5 @@
#!/usr/bin/env python2
-from __future__ import print_function
-
import sys
import pymacaroons
diff --git a/scripts-dev/federation_client.py b/scripts-dev/federation_client.py
index 531010185d..abcec48c4f 100755
--- a/scripts-dev/federation_client.py
+++ b/scripts-dev/federation_client.py
@@ -15,16 +15,16 @@
# See the License for the specific language governing permissions and
# limitations under the License.
-from __future__ import print_function
-
import argparse
import base64
import json
import sys
+from typing import Any, Optional
from urllib import parse as urlparse
import nacl.signing
import requests
+import signedjson.types
import srvlookup
import yaml
from requests.adapters import HTTPAdapter
@@ -69,7 +69,9 @@ def encode_canonical_json(value):
).encode("UTF-8")
-def sign_json(json_object, signing_key, signing_name):
+def sign_json(
+ json_object: Any, signing_key: signedjson.types.SigningKey, signing_name: str
+) -> Any:
signatures = json_object.pop("signatures", {})
unsigned = json_object.pop("unsigned", None)
@@ -122,7 +124,14 @@ def read_signing_keys(stream):
return keys
-def request_json(method, origin_name, origin_key, destination, path, content):
+def request(
+ method: Optional[str],
+ origin_name: str,
+ origin_key: signedjson.types.SigningKey,
+ destination: str,
+ path: str,
+ content: Optional[str],
+) -> requests.Response:
if method is None:
if content is None:
method = "GET"
@@ -159,11 +168,14 @@ def request_json(method, origin_name, origin_key, destination, path, content):
if method == "POST":
headers["Content-Type"] = "application/json"
- result = s.request(
- method=method, url=dest, headers=headers, verify=False, data=content
+ return s.request(
+ method=method,
+ url=dest,
+ headers=headers,
+ verify=False,
+ data=content,
+ stream=True,
)
- sys.stderr.write("Status Code: %d\n" % (result.status_code,))
- return result.json()
def main():
@@ -222,7 +234,7 @@ def main():
with open(args.signing_key_path) as f:
key = read_signing_keys(f)[0]
- result = request_json(
+ result = request(
args.method,
args.server_name,
key,
@@ -231,7 +243,12 @@ def main():
content=args.body,
)
- json.dump(result, sys.stdout)
+ sys.stderr.write("Status Code: %d\n" % (result.status_code,))
+
+ for chunk in result.iter_content():
+ # we write raw utf8 to stdout.
+ sys.stdout.buffer.write(chunk)
+
print("")
@@ -304,7 +321,7 @@ class MatrixConnectionAdapter(HTTPAdapter):
url = urlparse.urlunparse(
("https", netloc, parsed.path, parsed.params, parsed.query, parsed.fragment)
)
- return super(MatrixConnectionAdapter, self).get_connection(url, proxies)
+ return super().get_connection(url, proxies)
if __name__ == "__main__":
diff --git a/scripts-dev/hash_history.py b/scripts-dev/hash_history.py
index bf3862a386..8d6c3d24db 100644
--- a/scripts-dev/hash_history.py
+++ b/scripts-dev/hash_history.py
@@ -1,5 +1,3 @@
-from __future__ import print_function
-
import sqlite3
import sys
@@ -15,7 +13,7 @@ from synapse.storage.pdu import PduStore
from synapse.storage.signatures import SignatureStore
-class Store(object):
+class Store:
_get_pdu_tuples = PduStore.__dict__["_get_pdu_tuples"]
_get_pdu_content_hashes_txn = SignatureStore.__dict__["_get_pdu_content_hashes_txn"]
_get_prev_pdu_hashes_txn = SignatureStore.__dict__["_get_prev_pdu_hashes_txn"]
diff --git a/scripts-dev/mypy_synapse_plugin.py b/scripts-dev/mypy_synapse_plugin.py
new file mode 100644
index 0000000000..a5b88731f1
--- /dev/null
+++ b/scripts-dev/mypy_synapse_plugin.py
@@ -0,0 +1,85 @@
+# -*- coding: utf-8 -*-
+# Copyright 2020 The Matrix.org Foundation C.I.C.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""This is a mypy plugin for Synpase to deal with some of the funky typing that
+can crop up, e.g the cache descriptors.
+"""
+
+from typing import Callable, Optional
+
+from mypy.plugin import MethodSigContext, Plugin
+from mypy.typeops import bind_self
+from mypy.types import CallableType
+
+
+class SynapsePlugin(Plugin):
+ def get_method_signature_hook(
+ self, fullname: str
+ ) -> Optional[Callable[[MethodSigContext], CallableType]]:
+ if fullname.startswith(
+ "synapse.util.caches.descriptors._CachedFunction.__call__"
+ ):
+ return cached_function_method_signature
+ return None
+
+
+def cached_function_method_signature(ctx: MethodSigContext) -> CallableType:
+ """Fixes the `_CachedFunction.__call__` signature to be correct.
+
+ It already has *almost* the correct signature, except:
+
+ 1. the `self` argument needs to be marked as "bound"; and
+ 2. any `cache_context` argument should be removed.
+ """
+
+ # First we mark this as a bound function signature.
+ signature = bind_self(ctx.default_signature)
+
+ # Secondly, we remove any "cache_context" args.
+ #
+ # Note: We should be only doing this if `cache_context=True` is set, but if
+ # it isn't then the code will raise an exception when its called anyway, so
+ # its not the end of the world.
+ context_arg_index = None
+ for idx, name in enumerate(signature.arg_names):
+ if name == "cache_context":
+ context_arg_index = idx
+ break
+
+ if context_arg_index:
+ arg_types = list(signature.arg_types)
+ arg_types.pop(context_arg_index)
+
+ arg_names = list(signature.arg_names)
+ arg_names.pop(context_arg_index)
+
+ arg_kinds = list(signature.arg_kinds)
+ arg_kinds.pop(context_arg_index)
+
+ signature = signature.copy_modified(
+ arg_types=arg_types, arg_names=arg_names, arg_kinds=arg_kinds,
+ )
+
+ return signature
+
+
+def plugin(version: str):
+ # This is the entry point of the plugin, and let's us deal with the fact
+ # that the mypy plugin interface is *not* stable by looking at the version
+ # string.
+ #
+ # However, since we pin the version of mypy Synapse uses in CI, we don't
+ # really care.
+ return SynapsePlugin
diff --git a/scripts-dev/update_database b/scripts-dev/update_database
index 94aa8758b4..56365e2b58 100755
--- a/scripts-dev/update_database
+++ b/scripts-dev/update_database
@@ -40,7 +40,7 @@ class MockHomeserver(HomeServer):
config.server_name, reactor=reactor, config=config, **kwargs
)
- self.version_string = "Synapse/"+get_version_string(synapse)
+ self.version_string = "Synapse/" + get_version_string(synapse)
if __name__ == "__main__":
@@ -86,7 +86,7 @@ if __name__ == "__main__":
store = hs.get_datastore()
async def run_background_updates():
- await store.db.updates.run_background_updates(sleep=False)
+ await store.db_pool.updates.run_background_updates(sleep=False)
# Stop the reactor to exit the script once every background update is run.
reactor.stop()
|
