Move scripts directory inside synapse, exposing as setuptools entry_points (#12118)

* Two scripts are basically entry_points already
* Move and rename scripts/* to synapse/_scripts/*.py
* Delete sync_room_to_group.pl
* Expose entry points in setup.py
* Update linter script and config
* Fixup scripts & docs mentioning scripts that moved

Co-authored-by: Andrew Morgan <1342360+anoadragon453@users.noreply.github.com>

11 files changed, 0 insertions, 1908 deletions

diff --git a/scripts/export_signing_key b/scripts/export_signing_key
deleted file mode 100755
index bf0139bd64..0000000000
--- a/scripts/export_signing_key
+++ /dev/null
@@ -1,100 +0,0 @@
-#!/usr/bin/env python
-# Copyright 2019 The Matrix.org Foundation C.I.C.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-import argparse
-import sys
-import time
-from typing import Optional
-
-import nacl.signing
-from signedjson.key import encode_verify_key_base64, get_verify_key, read_signing_keys
-
-
-def exit(status: int = 0, message: Optional[str] = None):
-    if message:
-        print(message, file=sys.stderr)
-    sys.exit(status)
-
-
-def format_plain(public_key: nacl.signing.VerifyKey):
-    print(
-        "%s:%s %s"
-        % (
-            public_key.alg,
-            public_key.version,
-            encode_verify_key_base64(public_key),
-        )
-    )
-
-
-def format_for_config(public_key: nacl.signing.VerifyKey, expiry_ts: int):
-    print(
-        '  "%s:%s": { key: "%s", expired_ts: %i }'
-        % (
-            public_key.alg,
-            public_key.version,
-            encode_verify_key_base64(public_key),
-            expiry_ts,
-        )
-    )
-
-
-if __name__ == "__main__":
-    parser = argparse.ArgumentParser()
-
-    parser.add_argument(
-        "key_file",
-        nargs="+",
-        type=argparse.FileType("r"),
-        help="The key file to read",
-    )
-
-    parser.add_argument(
-        "-x",
-        action="store_true",
-        dest="for_config",
-        help="format the output for inclusion in the old_signing_keys config setting",
-    )
-
-    parser.add_argument(
-        "--expiry-ts",
-        type=int,
-        default=int(time.time() * 1000) + 6 * 3600000,
-        help=(
-            "The expiry time to use for -x, in milliseconds since 1970. The default "
-            "is (now+6h)."
-        ),
-    )
-
-    args = parser.parse_args()
-
-    formatter = (
-        (lambda k: format_for_config(k, args.expiry_ts))
-        if args.for_config
-        else format_plain
-    )
-
-    keys = []
-    for file in args.key_file:
-        try:
-            res = read_signing_keys(file)
-        except Exception as e:
-            exit(
-                status=1,
-                message="Error reading key from file %s: %s %s"
-                % (file.name, type(e), e),
-            )
-            res = []
-        for key in res:
-            formatter(get_verify_key(key))
diff --git a/scripts/generate_config b/scripts/generate_config
deleted file mode 100755
index 931b40c045..0000000000
--- a/scripts/generate_config
+++ /dev/null
@@ -1,78 +0,0 @@
-#!/usr/bin/env python3
-
-import argparse
-import shutil
-import sys
-
-from synapse.config.homeserver import HomeServerConfig
-
-if __name__ == "__main__":
-    parser = argparse.ArgumentParser()
-    parser.add_argument(
-        "--config-dir",
-        default="CONFDIR",
-        help="The path where the config files are kept. Used to create filenames for "
-        "things like the log config and the signing key. Default: %(default)s",
-    )
-
-    parser.add_argument(
-        "--data-dir",
-        default="DATADIR",
-        help="The path where the data files are kept. Used to create filenames for "
-        "things like the database and media store. Default: %(default)s",
-    )
-
-    parser.add_argument(
-        "--server-name",
-        default="SERVERNAME",
-        help="The server name. Used to initialise the server_name config param, but also "
-        "used in the names of some of the config files. Default: %(default)s",
-    )
-
-    parser.add_argument(
-        "--report-stats",
-        action="store",
-        help="Whether the generated config reports anonymized usage statistics",
-        choices=["yes", "no"],
-    )
-
-    parser.add_argument(
-        "--generate-secrets",
-        action="store_true",
-        help="Enable generation of new secrets for things like the macaroon_secret_key."
-        "By default, these parameters will be left unset.",
-    )
-
-    parser.add_argument(
-        "-o",
-        "--output-file",
-        type=argparse.FileType("w"),
-        default=sys.stdout,
-        help="File to write the configuration to. Default: stdout",
-    )
-
-    parser.add_argument(
-        "--header-file",
-        type=argparse.FileType("r"),
-        help="File from which to read a header, which will be printed before the "
-        "generated config.",
-    )
-
-    args = parser.parse_args()
-
-    report_stats = args.report_stats
-    if report_stats is not None:
-        report_stats = report_stats == "yes"
-
-    conf = HomeServerConfig().generate_config(
-        config_dir_path=args.config_dir,
-        data_dir_path=args.data_dir,
-        server_name=args.server_name,
-        generate_secrets=args.generate_secrets,
-        report_stats=report_stats,
-    )
-
-    if args.header_file:
-        shutil.copyfileobj(args.header_file, args.output_file)
-
-    args.output_file.write(conf)
diff --git a/scripts/generate_log_config b/scripts/generate_log_config
deleted file mode 100755
index e72a0dafb7..0000000000
--- a/scripts/generate_log_config
+++ /dev/null
@@ -1,44 +0,0 @@
-#!/usr/bin/env python3
-
-# Copyright 2020 The Matrix.org Foundation C.I.C.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-import argparse
-import sys
-
-from synapse.config.logger import DEFAULT_LOG_CONFIG
-
-if __name__ == "__main__":
-    parser = argparse.ArgumentParser()
-
-    parser.add_argument(
-        "-o",
-        "--output-file",
-        type=argparse.FileType("w"),
-        default=sys.stdout,
-        help="File to write the configuration to. Default: stdout",
-    )
-
-    parser.add_argument(
-        "-f",
-        "--log-file",
-        type=str,
-        default="/var/log/matrix-synapse/homeserver.log",
-        help="name of the log file",
-    )
-
-    args = parser.parse_args()
-    out = args.output_file
-    out.write(DEFAULT_LOG_CONFIG.substitute(log_file=args.log_file))
-    out.flush()
diff --git a/scripts/generate_signing_key.py b/scripts/generate_signing_key.py
deleted file mode 100755
index 07df25a809..0000000000
--- a/scripts/generate_signing_key.py
+++ /dev/null
@@ -1,36 +0,0 @@
-#!/usr/bin/env python
-# Copyright 2019 The Matrix.org Foundation C.I.C.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-import argparse
-import sys
-
-from signedjson.key import generate_signing_key, write_signing_keys
-
-from synapse.util.stringutils import random_string
-
-if __name__ == "__main__":
-    parser = argparse.ArgumentParser()
-
-    parser.add_argument(
-        "-o",
-        "--output_file",
-        type=argparse.FileType("w"),
-        default=sys.stdout,
-        help="Where to write the output to",
-    )
-    args = parser.parse_args()
-
-    key_id = "a_" + random_string(4)
-    key = (generate_signing_key(key_id),)
-    write_signing_keys(args.output_file, key)
diff --git a/scripts/hash_password b/scripts/hash_password
deleted file mode 100755
index 1d6fb0d700..0000000000
--- a/scripts/hash_password
+++ /dev/null
@@ -1,79 +0,0 @@
-#!/usr/bin/env python
-
-import argparse
-import getpass
-import sys
-import unicodedata
-
-import bcrypt
-import yaml
-
-bcrypt_rounds = 12
-password_pepper = ""
-
-
-def prompt_for_pass():
-    password = getpass.getpass("Password: ")
-
-    if not password:
-        raise Exception("Password cannot be blank.")
-
-    confirm_password = getpass.getpass("Confirm password: ")
-
-    if password != confirm_password:
-        raise Exception("Passwords do not match.")
-
-    return password
-
-
-if __name__ == "__main__":
-    parser = argparse.ArgumentParser(
-        description=(
-            "Calculate the hash of a new password, so that passwords can be reset"
-        )
-    )
-    parser.add_argument(
-        "-p",
-        "--password",
-        default=None,
-        help="New password for user. Will prompt if omitted.",
-    )
-    parser.add_argument(
-        "-c",
-        "--config",
-        type=argparse.FileType("r"),
-        help=(
-            "Path to server config file. "
-            "Used to read in bcrypt_rounds and password_pepper."
-        ),
-    )
-
-    args = parser.parse_args()
-    if "config" in args and args.config:
-        config = yaml.safe_load(args.config)
-        bcrypt_rounds = config.get("bcrypt_rounds", bcrypt_rounds)
-        password_config = config.get("password_config", None) or {}
-        password_pepper = password_config.get("pepper", password_pepper)
-    password = args.password
-
-    if not password:
-        password = prompt_for_pass()
-
-    # On Python 2, make sure we decode it to Unicode before we normalise it
-    if isinstance(password, bytes):
-        try:
-            password = password.decode(sys.stdin.encoding)
-        except UnicodeDecodeError:
-            print(
-                "ERROR! Your password is not decodable using your terminal encoding (%s)."
-                % (sys.stdin.encoding,)
-            )
-
-    pw = unicodedata.normalize("NFKC", password)
-
-    hashed = bcrypt.hashpw(
-        pw.encode("utf8") + password_pepper.encode("utf8"),
-        bcrypt.gensalt(bcrypt_rounds),
-    ).decode("ascii")
-
-    print(hashed)
diff --git a/scripts/move_remote_media_to_new_store.py b/scripts/move_remote_media_to_new_store.py
deleted file mode 100755
index 875aa4781f..0000000000
--- a/scripts/move_remote_media_to_new_store.py
+++ /dev/null
@@ -1,118 +0,0 @@
-#!/usr/bin/env python
-# Copyright 2017 New Vector Ltd
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-"""
-Moves a list of remote media from one media store to another.
-
-The input should be a list of media files to be moved, one per line. Each line
-should be formatted::
-
-    <origin server>|<file id>
-
-This can be extracted from postgres with::
-
-    psql --tuples-only -A -c "select media_origin, filesystem_id from
-        matrix.remote_media_cache where ..."
-
-To use, pipe the above into::
-
-    PYTHON_PATH=. ./scripts/move_remote_media_to_new_store.py <source repo> <dest repo>
-"""
-
-import argparse
-import logging
-import os
-import shutil
-import sys
-
-from synapse.rest.media.v1.filepath import MediaFilePaths
-
-logger = logging.getLogger()
-
-
-def main(src_repo, dest_repo):
-    src_paths = MediaFilePaths(src_repo)
-    dest_paths = MediaFilePaths(dest_repo)
-    for line in sys.stdin:
-        line = line.strip()
-        parts = line.split("|")
-        if len(parts) != 2:
-            print("Unable to parse input line %s" % line, file=sys.stderr)
-            sys.exit(1)
-
-        move_media(parts[0], parts[1], src_paths, dest_paths)
-
-
-def move_media(origin_server, file_id, src_paths, dest_paths):
-    """Move the given file, and any thumbnails, to the dest repo
-
-    Args:
-        origin_server (str):
-        file_id (str):
-        src_paths (MediaFilePaths):
-        dest_paths (MediaFilePaths):
-    """
-    logger.info("%s/%s", origin_server, file_id)
-
-    # check that the original exists
-    original_file = src_paths.remote_media_filepath(origin_server, file_id)
-    if not os.path.exists(original_file):
-        logger.warning(
-            "Original for %s/%s (%s) does not exist",
-            origin_server,
-            file_id,
-            original_file,
-        )
-    else:
-        mkdir_and_move(
-            original_file, dest_paths.remote_media_filepath(origin_server, file_id)
-        )
-
-    # now look for thumbnails
-    original_thumb_dir = src_paths.remote_media_thumbnail_dir(origin_server, file_id)
-    if not os.path.exists(original_thumb_dir):
-        return
-
-    mkdir_and_move(
-        original_thumb_dir,
-        dest_paths.remote_media_thumbnail_dir(origin_server, file_id),
-    )
-
-
-def mkdir_and_move(original_file, dest_file):
-    dirname = os.path.dirname(dest_file)
-    if not os.path.exists(dirname):
-        logger.debug("mkdir %s", dirname)
-        os.makedirs(dirname)
-    logger.debug("mv %s %s", original_file, dest_file)
-    shutil.move(original_file, dest_file)
-
-
-if __name__ == "__main__":
-    parser = argparse.ArgumentParser(
-        description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter
-    )
-    parser.add_argument("-v", action="store_true", help="enable debug logging")
-    parser.add_argument("src_repo", help="Path to source content repo")
-    parser.add_argument("dest_repo", help="Path to source content repo")
-    args = parser.parse_args()
-
-    logging_config = {
-        "level": logging.DEBUG if args.v else logging.INFO,
-        "format": "%(asctime)s - %(name)s - %(lineno)d - %(levelname)s - %(message)s",
-    }
-    logging.basicConfig(**logging_config)
-
-    main(args.src_repo, args.dest_repo)
diff --git a/scripts/register_new_matrix_user b/scripts/register_new_matrix_user
deleted file mode 100755
index 00104b9d62..0000000000
--- a/scripts/register_new_matrix_user
+++ /dev/null
@@ -1,19 +0,0 @@
-#!/usr/bin/env python
-# Copyright 2015, 2016 OpenMarket Ltd
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-from synapse._scripts.register_new_matrix_user import main
-
-if __name__ == "__main__":
-    main()
diff --git a/scripts/synapse_port_db b/scripts/synapse_port_db
deleted file mode 100755
index db354b3c8c..0000000000
--- a/scripts/synapse_port_db
+++ /dev/null
@@ -1,1253 +0,0 @@
-#!/usr/bin/env python
-# Copyright 2015, 2016 OpenMarket Ltd
-# Copyright 2018 New Vector Ltd
-# Copyright 2019 The Matrix.org Foundation C.I.C.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-import argparse
-import curses
-import logging
-import sys
-import time
-import traceback
-from typing import Dict, Iterable, Optional, Set
-
-import yaml
-from matrix_common.versionstring import get_distribution_version_string
-
-from twisted.internet import defer, reactor
-
-from synapse.config.database import DatabaseConnectionConfig
-from synapse.config.homeserver import HomeServerConfig
-from synapse.logging.context import (
-    LoggingContext,
-    make_deferred_yieldable,
-    run_in_background,
-)
-from synapse.storage.database import DatabasePool, make_conn
-from synapse.storage.databases.main import PushRuleStore
-from synapse.storage.databases.main.account_data import AccountDataWorkerStore
-from synapse.storage.databases.main.client_ips import ClientIpBackgroundUpdateStore
-from synapse.storage.databases.main.deviceinbox import DeviceInboxBackgroundUpdateStore
-from synapse.storage.databases.main.devices import DeviceBackgroundUpdateStore
-from synapse.storage.databases.main.end_to_end_keys import EndToEndKeyBackgroundStore
-from synapse.storage.databases.main.events_bg_updates import (
-    EventsBackgroundUpdatesStore,
-)
-from synapse.storage.databases.main.group_server import GroupServerWorkerStore
-from synapse.storage.databases.main.media_repository import (
-    MediaRepositoryBackgroundUpdateStore,
-)
-from synapse.storage.databases.main.presence import PresenceBackgroundUpdateStore
-from synapse.storage.databases.main.pusher import PusherWorkerStore
-from synapse.storage.databases.main.registration import (
-    RegistrationBackgroundUpdateStore,
-    find_max_generated_user_id_localpart,
-)
-from synapse.storage.databases.main.room import RoomBackgroundUpdateStore
-from synapse.storage.databases.main.roommember import RoomMemberBackgroundUpdateStore
-from synapse.storage.databases.main.search import SearchBackgroundUpdateStore
-from synapse.storage.databases.main.state import MainStateBackgroundUpdateStore
-from synapse.storage.databases.main.stats import StatsStore
-from synapse.storage.databases.main.user_directory import (
-    UserDirectoryBackgroundUpdateStore,
-)
-from synapse.storage.databases.state.bg_updates import StateBackgroundUpdateStore
-from synapse.storage.engines import create_engine
-from synapse.storage.prepare_database import prepare_database
-from synapse.util import Clock
-
-logger = logging.getLogger("synapse_port_db")
-
-
-BOOLEAN_COLUMNS = {
-    "events": ["processed", "outlier", "contains_url"],
-    "rooms": ["is_public", "has_auth_chain_index"],
-    "event_edges": ["is_state"],
-    "presence_list": ["accepted"],
-    "presence_stream": ["currently_active"],
-    "public_room_list_stream": ["visibility"],
-    "devices": ["hidden"],
-    "device_lists_outbound_pokes": ["sent"],
-    "users_who_share_rooms": ["share_private"],
-    "groups": ["is_public"],
-    "group_rooms": ["is_public"],
-    "group_users": ["is_public", "is_admin"],
-    "group_summary_rooms": ["is_public"],
-    "group_room_categories": ["is_public"],
-    "group_summary_users": ["is_public"],
-    "group_roles": ["is_public"],
-    "local_group_membership": ["is_publicised", "is_admin"],
-    "e2e_room_keys": ["is_verified"],
-    "account_validity": ["email_sent"],
-    "redactions": ["have_censored"],
-    "room_stats_state": ["is_federatable"],
-    "local_media_repository": ["safe_from_quarantine"],
-    "users": ["shadow_banned"],
-    "e2e_fallback_keys_json": ["used"],
-    "access_tokens": ["used"],
-}
-
-
-APPEND_ONLY_TABLES = [
-    "event_reference_hashes",
-    "events",
-    "event_json",
-    "state_events",
-    "room_memberships",
-    "topics",
-    "room_names",
-    "rooms",
-    "local_media_repository",
-    "local_media_repository_thumbnails",
-    "remote_media_cache",
-    "remote_media_cache_thumbnails",
-    "redactions",
-    "event_edges",
-    "event_auth",
-    "received_transactions",
-    "sent_transactions",
-    "transaction_id_to_pdu",
-    "users",
-    "state_groups",
-    "state_groups_state",
-    "event_to_state_groups",
-    "rejections",
-    "event_search",
-    "presence_stream",
-    "push_rules_stream",
-    "ex_outlier_stream",
-    "cache_invalidation_stream_by_instance",
-    "public_room_list_stream",
-    "state_group_edges",
-    "stream_ordering_to_exterm",
-]
-
-
-IGNORED_TABLES = {
-    # We don't port these tables, as they're a faff and we can regenerate
-    # them anyway.
-    "user_directory",
-    "user_directory_search",
-    "user_directory_search_content",
-    "user_directory_search_docsize",
-    "user_directory_search_segdir",
-    "user_directory_search_segments",
-    "user_directory_search_stat",
-    "user_directory_search_pos",
-    "users_who_share_private_rooms",
-    "users_in_public_room",
-    # UI auth sessions have foreign keys so additional care needs to be taken,
-    # the sessions are transient anyway, so ignore them.
-    "ui_auth_sessions",
-    "ui_auth_sessions_credentials",
-    "ui_auth_sessions_ips",
-}
-
-
-# Error returned by the run function. Used at the top-level part of the script to
-# handle errors and return codes.
-end_error = None  # type: Optional[str]
-# The exec_info for the error, if any. If error is defined but not exec_info the script
-# will show only the error message without the stacktrace, if exec_info is defined but
-# not the error then the script will show nothing outside of what's printed in the run
-# function. If both are defined, the script will print both the error and the stacktrace.
-end_error_exec_info = None
-
-
-class Store(
-    ClientIpBackgroundUpdateStore,
-    DeviceInboxBackgroundUpdateStore,
-    DeviceBackgroundUpdateStore,
-    EventsBackgroundUpdatesStore,
-    MediaRepositoryBackgroundUpdateStore,
-    RegistrationBackgroundUpdateStore,
-    RoomBackgroundUpdateStore,
-    RoomMemberBackgroundUpdateStore,
-    SearchBackgroundUpdateStore,
-    StateBackgroundUpdateStore,
-    MainStateBackgroundUpdateStore,
-    UserDirectoryBackgroundUpdateStore,
-    EndToEndKeyBackgroundStore,
-    StatsStore,
-    AccountDataWorkerStore,
-    PushRuleStore,
-    PusherWorkerStore,
-    PresenceBackgroundUpdateStore,
-    GroupServerWorkerStore,
-):
-    def execute(self, f, *args, **kwargs):
-        return self.db_pool.runInteraction(f.__name__, f, *args, **kwargs)
-
-    def execute_sql(self, sql, *args):
-        def r(txn):
-            txn.execute(sql, args)
-            return txn.fetchall()
-
-        return self.db_pool.runInteraction("execute_sql", r)
-
-    def insert_many_txn(self, txn, table, headers, rows):
-        sql = "INSERT INTO %s (%s) VALUES (%s)" % (
-            table,
-            ", ".join(k for k in headers),
-            ", ".join("%s" for _ in headers),
-        )
-
-        try:
-            txn.executemany(sql, rows)
-        except Exception:
-            logger.exception("Failed to insert: %s", table)
-            raise
-
-    def set_room_is_public(self, room_id, is_public):
-        raise Exception(
-            "Attempt to set room_is_public during port_db: database not empty?"
-        )
-
-
-class MockHomeserver:
-    def __init__(self, config):
-        self.clock = Clock(reactor)
-        self.config = config
-        self.hostname = config.server.server_name
-        self.version_string = "Synapse/" + get_distribution_version_string(
-            "matrix-synapse"
-        )
-
-    def get_clock(self):
-        return self.clock
-
-    def get_reactor(self):
-        return reactor
-
-    def get_instance_name(self):
-        return "master"
-
-
-class Porter(object):
-    def __init__(self, **kwargs):
-        self.__dict__.update(kwargs)
-
-    async def setup_table(self, table):
-        if table in APPEND_ONLY_TABLES:
-            # It's safe to just carry on inserting.
-            row = await self.postgres_store.db_pool.simple_select_one(
-                table="port_from_sqlite3",
-                keyvalues={"table_name": table},
-                retcols=("forward_rowid", "backward_rowid"),
-                allow_none=True,
-            )
-
-            total_to_port = None
-            if row is None:
-                if table == "sent_transactions":
-                    (
-                        forward_chunk,
-                        already_ported,
-                        total_to_port,
-                    ) = await self._setup_sent_transactions()
-                    backward_chunk = 0
-                else:
-                    await self.postgres_store.db_pool.simple_insert(
-                        table="port_from_sqlite3",
-                        values={
-                            "table_name": table,
-                            "forward_rowid": 1,
-                            "backward_rowid": 0,
-                        },
-                    )
-
-                    forward_chunk = 1
-                    backward_chunk = 0
-                    already_ported = 0
-            else:
-                forward_chunk = row["forward_rowid"]
-                backward_chunk = row["backward_rowid"]
-
-            if total_to_port is None:
-                already_ported, total_to_port = await self._get_total_count_to_port(
-                    table, forward_chunk, backward_chunk
-                )
-        else:
-
-            def delete_all(txn):
-                txn.execute(
-                    "DELETE FROM port_from_sqlite3 WHERE table_name = %s", (table,)
-                )
-                txn.execute("TRUNCATE %s CASCADE" % (table,))
-
-            await self.postgres_store.execute(delete_all)
-
-            await self.postgres_store.db_pool.simple_insert(
-                table="port_from_sqlite3",
-                values={"table_name": table, "forward_rowid": 1, "backward_rowid": 0},
-            )
-
-            forward_chunk = 1
-            backward_chunk = 0
-
-            already_ported, total_to_port = await self._get_total_count_to_port(
-                table, forward_chunk, backward_chunk
-            )
-
-        return table, already_ported, total_to_port, forward_chunk, backward_chunk
-
-    async def get_table_constraints(self) -> Dict[str, Set[str]]:
-        """Returns a map of tables that have foreign key constraints to tables they depend on."""
-
-        def _get_constraints(txn):
-            # We can pull the information about foreign key constraints out from
-            # the postgres schema tables.
-            sql = """
-                SELECT DISTINCT
-                    tc.table_name,
-                    ccu.table_name AS foreign_table_name
-                FROM
-                    information_schema.table_constraints AS tc
-                    INNER JOIN information_schema.constraint_column_usage AS ccu
-                    USING (table_schema, constraint_name)
-                WHERE tc.constraint_type = 'FOREIGN KEY'
-                  AND tc.table_name != ccu.table_name;
-            """
-            txn.execute(sql)
-
-            results = {}
-            for table, foreign_table in txn:
-                results.setdefault(table, set()).add(foreign_table)
-            return results
-
-        return await self.postgres_store.db_pool.runInteraction(
-            "get_table_constraints", _get_constraints
-        )
-
-    async def handle_table(
-        self, table, postgres_size, table_size, forward_chunk, backward_chunk
-    ):
-        logger.info(
-            "Table %s: %i/%i (rows %i-%i) already ported",
-            table,
-            postgres_size,
-            table_size,
-            backward_chunk + 1,
-            forward_chunk - 1,
-        )
-
-        if not table_size:
-            return
-
-        self.progress.add_table(table, postgres_size, table_size)
-
-        if table == "event_search":
-            await self.handle_search_table(
-                postgres_size, table_size, forward_chunk, backward_chunk
-            )
-            return
-
-        if table in IGNORED_TABLES:
-            self.progress.update(table, table_size)  # Mark table as done
-            return
-
-        if table == "user_directory_stream_pos":
-            # We need to make sure there is a single row, `(X, null), as that is
-            # what synapse expects to be there.
-            await self.postgres_store.db_pool.simple_insert(
-                table=table, values={"stream_id": None}
-            )
-            self.progress.update(table, table_size)  # Mark table as done
-            return
-
-        forward_select = (
-            "SELECT rowid, * FROM %s WHERE rowid >= ? ORDER BY rowid LIMIT ?" % (table,)
-        )
-
-        backward_select = (
-            "SELECT rowid, * FROM %s WHERE rowid <= ? ORDER BY rowid LIMIT ?" % (table,)
-        )
-
-        do_forward = [True]
-        do_backward = [True]
-
-        while True:
-
-            def r(txn):
-                forward_rows = []
-                backward_rows = []
-                if do_forward[0]:
-                    txn.execute(forward_select, (forward_chunk, self.batch_size))
-                    forward_rows = txn.fetchall()
-                    if not forward_rows:
-                        do_forward[0] = False
-
-                if do_backward[0]:
-                    txn.execute(backward_select, (backward_chunk, self.batch_size))
-                    backward_rows = txn.fetchall()
-                    if not backward_rows:
-                        do_backward[0] = False
-
-                if forward_rows or backward_rows:
-                    headers = [column[0] for column in txn.description]
-                else:
-                    headers = None
-
-                return headers, forward_rows, backward_rows
-
-            headers, frows, brows = await self.sqlite_store.db_pool.runInteraction(
-                "select", r
-            )
-
-            if frows or brows:
-                if frows:
-                    forward_chunk = max(row[0] for row in frows) + 1
-                if brows:
-                    backward_chunk = min(row[0] for row in brows) - 1
-
-                rows = frows + brows
-                rows = self._convert_rows(table, headers, rows)
-
-                def insert(txn):
-                    self.postgres_store.insert_many_txn(txn, table, headers[1:], rows)
-
-                    self.postgres_store.db_pool.simple_update_one_txn(
-                        txn,
-                        table="port_from_sqlite3",
-                        keyvalues={"table_name": table},
-                        updatevalues={
-                            "forward_rowid": forward_chunk,
-                            "backward_rowid": backward_chunk,
-                        },
-                    )
-
-                await self.postgres_store.execute(insert)
-
-                postgres_size += len(rows)
-
-                self.progress.update(table, postgres_size)
-            else:
-                return
-
-    async def handle_search_table(
-        self, postgres_size, table_size, forward_chunk, backward_chunk
-    ):
-        select = (
-            "SELECT es.rowid, es.*, e.origin_server_ts, e.stream_ordering"
-            " FROM event_search as es"
-            " INNER JOIN events AS e USING (event_id, room_id)"
-            " WHERE es.rowid >= ?"
-            " ORDER BY es.rowid LIMIT ?"
-        )
-
-        while True:
-
-            def r(txn):
-                txn.execute(select, (forward_chunk, self.batch_size))
-                rows = txn.fetchall()
-                headers = [column[0] for column in txn.description]
-
-                return headers, rows
-
-            headers, rows = await self.sqlite_store.db_pool.runInteraction("select", r)
-
-            if rows:
-                forward_chunk = rows[-1][0] + 1
-
-                # We have to treat event_search differently since it has a
-                # different structure in the two different databases.
-                def insert(txn):
-                    sql = (
-                        "INSERT INTO event_search (event_id, room_id, key,"
-                        " sender, vector, origin_server_ts, stream_ordering)"
-                        " VALUES (?,?,?,?,to_tsvector('english', ?),?,?)"
-                    )
-
-                    rows_dict = []
-                    for row in rows:
-                        d = dict(zip(headers, row))
-                        if "\0" in d["value"]:
-                            logger.warning("dropping search row %s", d)
-                        else:
-                            rows_dict.append(d)
-
-                    txn.executemany(
-                        sql,
-                        [
-                            (
-                                row["event_id"],
-                                row["room_id"],
-                                row["key"],
-                                row["sender"],
-                                row["value"],
-                                row["origin_server_ts"],
-                                row["stream_ordering"],
-                            )
-                            for row in rows_dict
-                        ],
-                    )
-
-                    self.postgres_store.db_pool.simple_update_one_txn(
-                        txn,
-                        table="port_from_sqlite3",
-                        keyvalues={"table_name": "event_search"},
-                        updatevalues={
-                            "forward_rowid": forward_chunk,
-                            "backward_rowid": backward_chunk,
-                        },
-                    )
-
-                await self.postgres_store.execute(insert)
-
-                postgres_size += len(rows)
-
-                self.progress.update("event_search", postgres_size)
-
-            else:
-                return
-
-    def build_db_store(
-        self,
-        db_config: DatabaseConnectionConfig,
-        allow_outdated_version: bool = False,
-    ):
-        """Builds and returns a database store using the provided configuration.
-
-        Args:
-            db_config: The database configuration
-            allow_outdated_version: True to suppress errors about the database server
-                version being too old to run a complete synapse
-
-        Returns:
-            The built Store object.
-        """
-        self.progress.set_state("Preparing %s" % db_config.config["name"])
-
-        engine = create_engine(db_config.config)
-
-        hs = MockHomeserver(self.hs_config)
-
-        with make_conn(db_config, engine, "portdb") as db_conn:
-            engine.check_database(
-                db_conn, allow_outdated_version=allow_outdated_version
-            )
-            prepare_database(db_conn, engine, config=self.hs_config)
-            store = Store(DatabasePool(hs, db_config, engine), db_conn, hs)
-            db_conn.commit()
-
-        return store
-
-    async def run_background_updates_on_postgres(self):
-        # Manually apply all background updates on the PostgreSQL database.
-        postgres_ready = (
-            await self.postgres_store.db_pool.updates.has_completed_background_updates()
-        )
-
-        if not postgres_ready:
-            # Only say that we're running background updates when there are background
-            # updates to run.
-            self.progress.set_state("Running background updates on PostgreSQL")
-
-        while not postgres_ready:
-            await self.postgres_store.db_pool.updates.do_next_background_update(100)
-            postgres_ready = await (
-                self.postgres_store.db_pool.updates.has_completed_background_updates()
-            )
-
-    async def run(self):
-        """Ports the SQLite database to a PostgreSQL database.
-
-        When a fatal error is met, its message is assigned to the global "end_error"
-        variable. When this error comes with a stacktrace, its exec_info is assigned to
-        the global "end_error_exec_info" variable.
-        """
-        global end_error
-
-        try:
-            # we allow people to port away from outdated versions of sqlite.
-            self.sqlite_store = self.build_db_store(
-                DatabaseConnectionConfig("master-sqlite", self.sqlite_config),
-                allow_outdated_version=True,
-            )
-
-            # Check if all background updates are done, abort if not.
-            updates_complete = (
-                await self.sqlite_store.db_pool.updates.has_completed_background_updates()
-            )
-            if not updates_complete:
-                end_error = (
-                    "Pending background updates exist in the SQLite3 database."
-                    " Please start Synapse again and wait until every update has finished"
-                    " before running this script.\n"
-                )
-                return
-
-            self.postgres_store = self.build_db_store(
-                self.hs_config.database.get_single_database()
-            )
-
-            await self.run_background_updates_on_postgres()
-
-            self.progress.set_state("Creating port tables")
-
-            def create_port_table(txn):
-                txn.execute(
-                    "CREATE TABLE IF NOT EXISTS port_from_sqlite3 ("
-                    " table_name varchar(100) NOT NULL UNIQUE,"
-                    " forward_rowid bigint NOT NULL,"
-                    " backward_rowid bigint NOT NULL"
-                    ")"
-                )
-
-            # The old port script created a table with just a "rowid" column.
-            # We want people to be able to rerun this script from an old port
-            # so that they can pick up any missing events that were not
-            # ported across.
-            def alter_table(txn):
-                txn.execute(
-                    "ALTER TABLE IF EXISTS port_from_sqlite3"
-                    " RENAME rowid TO forward_rowid"
-                )
-                txn.execute(
-                    "ALTER TABLE IF EXISTS port_from_sqlite3"
-                    " ADD backward_rowid bigint NOT NULL DEFAULT 0"
-                )
-
-            try:
-                await self.postgres_store.db_pool.runInteraction(
-                    "alter_table", alter_table
-                )
-            except Exception:
-                # On Error Resume Next
-                pass
-
-            await self.postgres_store.db_pool.runInteraction(
-                "create_port_table", create_port_table
-            )
-
-            # Step 2. Set up sequences
-            #
-            # We do this before porting the tables so that event if we fail half
-            # way through the postgres DB always have sequences that are greater
-            # than their respective tables. If we don't then creating the
-            # `DataStore` object will fail due to the inconsistency.
-            self.progress.set_state("Setting up sequence generators")
-            await self._setup_state_group_id_seq()
-            await self._setup_user_id_seq()
-            await self._setup_events_stream_seqs()
-            await self._setup_sequence(
-                "device_inbox_sequence", ("device_inbox", "device_federation_outbox")
-            )
-            await self._setup_sequence(
-                "account_data_sequence",
-                ("room_account_data", "room_tags_revisions", "account_data"),
-            )
-            await self._setup_sequence("receipts_sequence", ("receipts_linearized",))
-            await self._setup_sequence("presence_stream_sequence", ("presence_stream",))
-            await self._setup_auth_chain_sequence()
-
-            # Step 3. Get tables.
-            self.progress.set_state("Fetching tables")
-            sqlite_tables = await self.sqlite_store.db_pool.simple_select_onecol(
-                table="sqlite_master", keyvalues={"type": "table"}, retcol="name"
-            )
-
-            postgres_tables = await self.postgres_store.db_pool.simple_select_onecol(
-                table="information_schema.tables",
-                keyvalues={},
-                retcol="distinct table_name",
-            )
-
-            tables = set(sqlite_tables) & set(postgres_tables)
-            logger.info("Found %d tables", len(tables))
-
-            # Step 4. Figure out what still needs copying
-            self.progress.set_state("Checking on port progress")
-            setup_res = await make_deferred_yieldable(
-                defer.gatherResults(
-                    [
-                        run_in_background(self.setup_table, table)
-                        for table in tables
-                        if table not in ["schema_version", "applied_schema_deltas"]
-                        and not table.startswith("sqlite_")
-                    ],
-                    consumeErrors=True,
-                )
-            )
-            # Map from table name to args passed to `handle_table`, i.e. a tuple
-            # of: `postgres_size`, `table_size`, `forward_chunk`, `backward_chunk`.
-            tables_to_port_info_map = {r[0]: r[1:] for r in setup_res}
-
-            # Step 5. Do the copying.
-            #
-            # This is slightly convoluted as we need to ensure tables are ported
-            # in the correct order due to foreign key constraints.
-            self.progress.set_state("Copying to postgres")
-
-            constraints = await self.get_table_constraints()
-            tables_ported = set()  # type: Set[str]
-
-            while tables_to_port_info_map:
-                # Pulls out all tables that are still to be ported and which
-                # only depend on tables that are already ported (if any).
-                tables_to_port = [
-                    table
-                    for table in tables_to_port_info_map
-                    if not constraints.get(table, set()) - tables_ported
-                ]
-
-                await make_deferred_yieldable(
-                    defer.gatherResults(
-                        [
-                            run_in_background(
-                                self.handle_table,
-                                table,
-                                *tables_to_port_info_map.pop(table),
-                            )
-                            for table in tables_to_port
-                        ],
-                        consumeErrors=True,
-                    )
-                )
-
-                tables_ported.update(tables_to_port)
-
-            self.progress.done()
-        except Exception as e:
-            global end_error_exec_info
-            end_error = str(e)
-            end_error_exec_info = sys.exc_info()
-            logger.exception("")
-        finally:
-            reactor.stop()
-
-    def _convert_rows(self, table, headers, rows):
-        bool_col_names = BOOLEAN_COLUMNS.get(table, [])
-
-        bool_cols = [i for i, h in enumerate(headers) if h in bool_col_names]
-
-        class BadValueException(Exception):
-            pass
-
-        def conv(j, col):
-            if j in bool_cols:
-                return bool(col)
-            if isinstance(col, bytes):
-                return bytearray(col)
-            elif isinstance(col, str) and "\0" in col:
-                logger.warning(
-                    "DROPPING ROW: NUL value in table %s col %s: %r",
-                    table,
-                    headers[j],
-                    col,
-                )
-                raise BadValueException()
-            return col
-
-        outrows = []
-        for row in rows:
-            try:
-                outrows.append(
-                    tuple(conv(j, col) for j, col in enumerate(row) if j > 0)
-                )
-            except BadValueException:
-                pass
-
-        return outrows
-
-    async def _setup_sent_transactions(self):
-        # Only save things from the last day
-        yesterday = int(time.time() * 1000) - 86400000
-
-        # And save the max transaction id from each destination
-        select = (
-            "SELECT rowid, * FROM sent_transactions WHERE rowid IN ("
-            "SELECT max(rowid) FROM sent_transactions"
-            " GROUP BY destination"
-            ")"
-        )
-
-        def r(txn):
-            txn.execute(select)
-            rows = txn.fetchall()
-            headers = [column[0] for column in txn.description]
-
-            ts_ind = headers.index("ts")
-
-            return headers, [r for r in rows if r[ts_ind] < yesterday]
-
-        headers, rows = await self.sqlite_store.db_pool.runInteraction("select", r)
-
-        rows = self._convert_rows("sent_transactions", headers, rows)
-
-        inserted_rows = len(rows)
-        if inserted_rows:
-            max_inserted_rowid = max(r[0] for r in rows)
-
-            def insert(txn):
-                self.postgres_store.insert_many_txn(
-                    txn, "sent_transactions", headers[1:], rows
-                )
-
-            await self.postgres_store.execute(insert)
-        else:
-            max_inserted_rowid = 0
-
-        def get_start_id(txn):
-            txn.execute(
-                "SELECT rowid FROM sent_transactions WHERE ts >= ?"
-                " ORDER BY rowid ASC LIMIT 1",
-                (yesterday,),
-            )
-
-            rows = txn.fetchall()
-            if rows:
-                return rows[0][0]
-            else:
-                return 1
-
-        next_chunk = await self.sqlite_store.execute(get_start_id)
-        next_chunk = max(max_inserted_rowid + 1, next_chunk)
-
-        await self.postgres_store.db_pool.simple_insert(
-            table="port_from_sqlite3",
-            values={
-                "table_name": "sent_transactions",
-                "forward_rowid": next_chunk,
-                "backward_rowid": 0,
-            },
-        )
-
-        def get_sent_table_size(txn):
-            txn.execute(
-                "SELECT count(*) FROM sent_transactions" " WHERE ts >= ?", (yesterday,)
-            )
-            (size,) = txn.fetchone()
-            return int(size)
-
-        remaining_count = await self.sqlite_store.execute(get_sent_table_size)
-
-        total_count = remaining_count + inserted_rows
-
-        return next_chunk, inserted_rows, total_count
-
-    async def _get_remaining_count_to_port(self, table, forward_chunk, backward_chunk):
-        frows = await self.sqlite_store.execute_sql(
-            "SELECT count(*) FROM %s WHERE rowid >= ?" % (table,), forward_chunk
-        )
-
-        brows = await self.sqlite_store.execute_sql(
-            "SELECT count(*) FROM %s WHERE rowid <= ?" % (table,), backward_chunk
-        )
-
-        return frows[0][0] + brows[0][0]
-
-    async def _get_already_ported_count(self, table):
-        rows = await self.postgres_store.execute_sql(
-            "SELECT count(*) FROM %s" % (table,)
-        )
-
-        return rows[0][0]
-
-    async def _get_total_count_to_port(self, table, forward_chunk, backward_chunk):
-        remaining, done = await make_deferred_yieldable(
-            defer.gatherResults(
-                [
-                    run_in_background(
-                        self._get_remaining_count_to_port,
-                        table,
-                        forward_chunk,
-                        backward_chunk,
-                    ),
-                    run_in_background(self._get_already_ported_count, table),
-                ],
-            )
-        )
-
-        remaining = int(remaining) if remaining else 0
-        done = int(done) if done else 0
-
-        return done, remaining + done
-
-    async def _setup_state_group_id_seq(self) -> None:
-        curr_id = await self.sqlite_store.db_pool.simple_select_one_onecol(
-            table="state_groups", keyvalues={}, retcol="MAX(id)", allow_none=True
-        )
-
-        if not curr_id:
-            return
-
-        def r(txn):
-            next_id = curr_id + 1
-            txn.execute("ALTER SEQUENCE state_group_id_seq RESTART WITH %s", (next_id,))
-
-        await self.postgres_store.db_pool.runInteraction("setup_state_group_id_seq", r)
-
-    async def _setup_user_id_seq(self) -> None:
-        curr_id = await self.sqlite_store.db_pool.runInteraction(
-            "setup_user_id_seq", find_max_generated_user_id_localpart
-        )
-
-        def r(txn):
-            next_id = curr_id + 1
-            txn.execute("ALTER SEQUENCE user_id_seq RESTART WITH %s", (next_id,))
-
-        await self.postgres_store.db_pool.runInteraction("setup_user_id_seq", r)
-
-    async def _setup_events_stream_seqs(self) -> None:
-        """Set the event stream sequences to the correct values."""
-
-        # We get called before we've ported the events table, so we need to
-        # fetch the current positions from the SQLite store.
-        curr_forward_id = await self.sqlite_store.db_pool.simple_select_one_onecol(
-            table="events", keyvalues={}, retcol="MAX(stream_ordering)", allow_none=True
-        )
-
-        curr_backward_id = await self.sqlite_store.db_pool.simple_select_one_onecol(
-            table="events",
-            keyvalues={},
-            retcol="MAX(-MIN(stream_ordering), 1)",
-            allow_none=True,
-        )
-
-        def _setup_events_stream_seqs_set_pos(txn):
-            if curr_forward_id:
-                txn.execute(
-                    "ALTER SEQUENCE events_stream_seq RESTART WITH %s",
-                    (curr_forward_id + 1,),
-                )
-
-            if curr_backward_id:
-                txn.execute(
-                    "ALTER SEQUENCE events_backfill_stream_seq RESTART WITH %s",
-                    (curr_backward_id + 1,),
-                )
-
-        await self.postgres_store.db_pool.runInteraction(
-            "_setup_events_stream_seqs",
-            _setup_events_stream_seqs_set_pos,
-        )
-
-    async def _setup_sequence(
-        self, sequence_name: str, stream_id_tables: Iterable[str]
-    ) -> None:
-        """Set a sequence to the correct value."""
-        current_stream_ids = []
-        for stream_id_table in stream_id_tables:
-            max_stream_id = await self.sqlite_store.db_pool.simple_select_one_onecol(
-                table=stream_id_table,
-                keyvalues={},
-                retcol="COALESCE(MAX(stream_id), 1)",
-                allow_none=True,
-            )
-            current_stream_ids.append(max_stream_id)
-
-        next_id = max(current_stream_ids) + 1
-
-        def r(txn):
-            sql = "ALTER SEQUENCE %s RESTART WITH" % (sequence_name,)
-            txn.execute(sql + " %s", (next_id,))
-
-        await self.postgres_store.db_pool.runInteraction(
-            "_setup_%s" % (sequence_name,), r
-        )
-
-    async def _setup_auth_chain_sequence(self) -> None:
-        curr_chain_id = await self.sqlite_store.db_pool.simple_select_one_onecol(
-            table="event_auth_chains",
-            keyvalues={},
-            retcol="MAX(chain_id)",
-            allow_none=True,
-        )
-
-        def r(txn):
-            txn.execute(
-                "ALTER SEQUENCE event_auth_chain_id RESTART WITH %s",
-                (curr_chain_id + 1,),
-            )
-
-        if curr_chain_id is not None:
-            await self.postgres_store.db_pool.runInteraction(
-                "_setup_event_auth_chain_id",
-                r,
-            )
-
-
-##############################################
-# The following is simply UI stuff
-##############################################
-
-
-class Progress(object):
-    """Used to report progress of the port"""
-
-    def __init__(self):
-        self.tables = {}
-
-        self.start_time = int(time.time())
-
-    def add_table(self, table, cur, size):
-        self.tables[table] = {
-            "start": cur,
-            "num_done": cur,
-            "total": size,
-            "perc": int(cur * 100 / size),
-        }
-
-    def update(self, table, num_done):
-        data = self.tables[table]
-        data["num_done"] = num_done
-        data["perc"] = int(num_done * 100 / data["total"])
-
-    def done(self):
-        pass
-
-
-class CursesProgress(Progress):
-    """Reports progress to a curses window"""
-
-    def __init__(self, stdscr):
-        self.stdscr = stdscr
-
-        curses.use_default_colors()
-        curses.curs_set(0)
-
-        curses.init_pair(1, curses.COLOR_RED, -1)
-        curses.init_pair(2, curses.COLOR_GREEN, -1)
-
-        self.last_update = 0
-
-        self.finished = False
-
-        self.total_processed = 0
-        self.total_remaining = 0
-
-        super(CursesProgress, self).__init__()
-
-    def update(self, table, num_done):
-        super(CursesProgress, self).update(table, num_done)
-
-        self.total_processed = 0
-        self.total_remaining = 0
-        for data in self.tables.values():
-            self.total_processed += data["num_done"] - data["start"]
-            self.total_remaining += data["total"] - data["num_done"]
-
-        self.render()
-
-    def render(self, force=False):
-        now = time.time()
-
-        if not force and now - self.last_update < 0.2:
-            # reactor.callLater(1, self.render)
-            return
-
-        self.stdscr.clear()
-
-        rows, cols = self.stdscr.getmaxyx()
-
-        duration = int(now) - int(self.start_time)
-
-        minutes, seconds = divmod(duration, 60)
-        duration_str = "%02dm %02ds" % (minutes, seconds)
-
-        if self.finished:
-            status = "Time spent: %s (Done!)" % (duration_str,)
-        else:
-
-            if self.total_processed > 0:
-                left = float(self.total_remaining) / self.total_processed
-
-                est_remaining = (int(now) - self.start_time) * left
-                est_remaining_str = "%02dm %02ds remaining" % divmod(est_remaining, 60)
-            else:
-                est_remaining_str = "Unknown"
-            status = "Time spent: %s (est. remaining: %s)" % (
-                duration_str,
-                est_remaining_str,
-            )
-
-        self.stdscr.addstr(0, 0, status, curses.A_BOLD)
-
-        max_len = max(len(t) for t in self.tables.keys())
-
-        left_margin = 5
-        middle_space = 1
-
-        items = self.tables.items()
-        items = sorted(items, key=lambda i: (i[1]["perc"], i[0]))
-
-        for i, (table, data) in enumerate(items):
-            if i + 2 >= rows:
-                break
-
-            perc = data["perc"]
-
-            color = curses.color_pair(2) if perc == 100 else curses.color_pair(1)
-
-            self.stdscr.addstr(
-                i + 2, left_margin + max_len - len(table), table, curses.A_BOLD | color
-            )
-
-            size = 20
-
-            progress = "[%s%s]" % (
-                "#" * int(perc * size / 100),
-                " " * (size - int(perc * size / 100)),
-            )
-
-            self.stdscr.addstr(
-                i + 2,
-                left_margin + max_len + middle_space,
-                "%s %3d%% (%d/%d)" % (progress, perc, data["num_done"], data["total"]),
-            )
-
-        if self.finished:
-            self.stdscr.addstr(rows - 1, 0, "Press any key to exit...")
-
-        self.stdscr.refresh()
-        self.last_update = time.time()
-
-    def done(self):
-        self.finished = True
-        self.render(True)
-        self.stdscr.getch()
-
-    def set_state(self, state):
-        self.stdscr.clear()
-        self.stdscr.addstr(0, 0, state + "...", curses.A_BOLD)
-        self.stdscr.refresh()
-
-
-class TerminalProgress(Progress):
-    """Just prints progress to the terminal"""
-
-    def update(self, table, num_done):
-        super(TerminalProgress, self).update(table, num_done)
-
-        data = self.tables[table]
-
-        print(
-            "%s: %d%% (%d/%d)" % (table, data["perc"], data["num_done"], data["total"])
-        )
-
-    def set_state(self, state):
-        print(state + "...")
-
-
-##############################################
-##############################################
-
-
-if __name__ == "__main__":
-    parser = argparse.ArgumentParser(
-        description="A script to port an existing synapse SQLite database to"
-        " a new PostgreSQL database."
-    )
-    parser.add_argument("-v", action="store_true")
-    parser.add_argument(
-        "--sqlite-database",
-        required=True,
-        help="The snapshot of the SQLite database file. This must not be"
-        " currently used by a running synapse server",
-    )
-    parser.add_argument(
-        "--postgres-config",
-        type=argparse.FileType("r"),
-        required=True,
-        help="The database config file for the PostgreSQL database",
-    )
-    parser.add_argument(
-        "--curses", action="store_true", help="display a curses based progress UI"
-    )
-
-    parser.add_argument(
-        "--batch-size",
-        type=int,
-        default=1000,
-        help="The number of rows to select from the SQLite table each"
-        " iteration [default=1000]",
-    )
-
-    args = parser.parse_args()
-
-    logging_config = {
-        "level": logging.DEBUG if args.v else logging.INFO,
-        "format": "%(asctime)s - %(name)s - %(lineno)d - %(levelname)s - %(message)s",
-    }
-
-    if args.curses:
-        logging_config["filename"] = "port-synapse.log"
-
-    logging.basicConfig(**logging_config)
-
-    sqlite_config = {
-        "name": "sqlite3",
-        "args": {
-            "database": args.sqlite_database,
-            "cp_min": 1,
-            "cp_max": 1,
-            "check_same_thread": False,
-        },
-    }
-
-    hs_config = yaml.safe_load(args.postgres_config)
-
-    if "database" not in hs_config:
-        sys.stderr.write("The configuration file must have a 'database' section.\n")
-        sys.exit(4)
-
-    postgres_config = hs_config["database"]
-
-    if "name" not in postgres_config:
-        sys.stderr.write("Malformed database config: no 'name'\n")
-        sys.exit(2)
-    if postgres_config["name"] != "psycopg2":
-        sys.stderr.write("Database must use the 'psycopg2' connector.\n")
-        sys.exit(3)
-
-    config = HomeServerConfig()
-    config.parse_config_dict(hs_config, "", "")
-
-    def start(stdscr=None):
-        if stdscr:
-            progress = CursesProgress(stdscr)
-        else:
-            progress = TerminalProgress()
-
-        porter = Porter(
-            sqlite_config=sqlite_config,
-            progress=progress,
-            batch_size=args.batch_size,
-            hs_config=config,
-        )
-
-        @defer.inlineCallbacks
-        def run():
-            with LoggingContext("synapse_port_db_run"):
-                yield defer.ensureDeferred(porter.run())
-
-        reactor.callWhenRunning(run)
-
-        reactor.run()
-
-    if args.curses:
-        curses.wrapper(start)
-    else:
-        start()
-
-    if end_error:
-        if end_error_exec_info:
-            exc_type, exc_value, exc_traceback = end_error_exec_info
-            traceback.print_exception(exc_type, exc_value, exc_traceback)
-
-        sys.stderr.write(end_error)
-
-        sys.exit(5)
diff --git a/scripts/synapse_review_recent_signups b/scripts/synapse_review_recent_signups
deleted file mode 100755
index a36d46e14c..0000000000
--- a/scripts/synapse_review_recent_signups
+++ /dev/null
@@ -1,19 +0,0 @@
-#!/usr/bin/env python
-# Copyright 2021 The Matrix.org Foundation C.I.C.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-from synapse._scripts.review_recent_signups import main
-
-if __name__ == "__main__":
-    main()
diff --git a/scripts/sync_room_to_group.pl b/scripts/sync_room_to_group.pl
deleted file mode 100755
index f0c2dfadfa..0000000000
--- a/scripts/sync_room_to_group.pl
+++ /dev/null
@@ -1,45 +0,0 @@
-#!/usr/bin/env perl
-
-use strict;
-use warnings;
-
-use JSON::XS;
-use LWP::UserAgent;
-use URI::Escape;
-
-if (@ARGV < 4) {
-    die "usage: $0 <homeserver url> <access_token> <room_id|room_alias> <group_id>\n";
-}
-
-my ($hs, $access_token, $room_id, $group_id) = @ARGV;
-my $ua = LWP::UserAgent->new();
-$ua->timeout(10);
-
-if ($room_id =~ /^#/) {
-    $room_id = uri_escape($room_id);
-    $room_id = decode_json($ua->get("${hs}/_matrix/client/r0/directory/room/${room_id}?access_token=${access_token}")->decoded_content)->{room_id};
-}
-
-my $room_users  = [ keys %{decode_json($ua->get("${hs}/_matrix/client/r0/rooms/${room_id}/joined_members?access_token=${access_token}")->decoded_content)->{joined}} ];
-my $group_users = [
-    (map { $_->{user_id} } @{decode_json($ua->get("${hs}/_matrix/client/unstable/groups/${group_id}/users?access_token=${access_token}" )->decoded_content)->{chunk}}),
-    (map { $_->{user_id} } @{decode_json($ua->get("${hs}/_matrix/client/unstable/groups/${group_id}/invited_users?access_token=${access_token}" )->decoded_content)->{chunk}}),
-];
-
-die "refusing to sync from empty room" unless (@$room_users);
-die "refusing to sync to empty group" unless (@$group_users);
-
-my $diff = {};
-foreach my $user (@$room_users) { $diff->{$user}++ }
-foreach my $user (@$group_users) { $diff->{$user}-- }
-
-foreach my $user (keys %$diff) {
-    if ($diff->{$user} == 1) {
-        warn "inviting $user";
-        print STDERR $ua->put("${hs}/_matrix/client/unstable/groups/${group_id}/admin/users/invite/${user}?access_token=${access_token}", Content=>'{}')->status_line."\n";
-    }
-    elsif ($diff->{$user} == -1) {
-        warn "removing $user";
-        print STDERR $ua->put("${hs}/_matrix/client/unstable/groups/${group_id}/admin/users/remove/${user}?access_token=${access_token}", Content=>'{}')->status_line."\n";
-    }
-}
diff --git a/scripts/update_synapse_database b/scripts/update_synapse_database
deleted file mode 100755
index f43676afaa..0000000000
--- a/scripts/update_synapse_database
+++ /dev/null
@@ -1,117 +0,0 @@
-#!/usr/bin/env python
-# Copyright 2019 The Matrix.org Foundation C.I.C.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-import argparse
-import logging
-import sys
-
-import yaml
-from matrix_common.versionstring import get_distribution_version_string
-
-from twisted.internet import defer, reactor
-
-from synapse.config.homeserver import HomeServerConfig
-from synapse.metrics.background_process_metrics import run_as_background_process
-from synapse.server import HomeServer
-from synapse.storage import DataStore
-
-logger = logging.getLogger("update_database")
-
-
-class MockHomeserver(HomeServer):
-    DATASTORE_CLASS = DataStore
-
-    def __init__(self, config, **kwargs):
-        super(MockHomeserver, self).__init__(
-            config.server.server_name, reactor=reactor, config=config, **kwargs
-        )
-
-        self.version_string = "Synapse/" + get_distribution_version_string(
-            "matrix-synapse"
-        )
-
-
-def run_background_updates(hs):
-    store = hs.get_datastores().main
-
-    async def run_background_updates():
-        await store.db_pool.updates.run_background_updates(sleep=False)
-        # Stop the reactor to exit the script once every background update is run.
-        reactor.stop()
-
-    def run():
-        # Apply all background updates on the database.
-        defer.ensureDeferred(
-            run_as_background_process("background_updates", run_background_updates)
-        )
-
-    reactor.callWhenRunning(run)
-
-    reactor.run()
-
-
-def main():
-    parser = argparse.ArgumentParser(
-        description=(
-            "Updates a synapse database to the latest schema and optionally runs background updates"
-            " on it."
-        )
-    )
-    parser.add_argument("-v", action="store_true")
-    parser.add_argument(
-        "--database-config",
-        type=argparse.FileType("r"),
-        required=True,
-        help="Synapse configuration file, giving the details of the database to be updated",
-    )
-    parser.add_argument(
-        "--run-background-updates",
-        action="store_true",
-        required=False,
-        help="run background updates after upgrading the database schema",
-    )
-
-    args = parser.parse_args()
-
-    logging_config = {
-        "level": logging.DEBUG if args.v else logging.INFO,
-        "format": "%(asctime)s - %(name)s - %(lineno)d - %(levelname)s - %(message)s",
-    }
-
-    logging.basicConfig(**logging_config)
-
-    # Load, process and sanity-check the config.
-    hs_config = yaml.safe_load(args.database_config)
-
-    if "database" not in hs_config:
-        sys.stderr.write("The configuration file must have a 'database' section.\n")
-        sys.exit(4)
-
-    config = HomeServerConfig()
-    config.parse_config_dict(hs_config, "", "")
-
-    # Instantiate and initialise the homeserver object.
-    hs = MockHomeserver(config)
-
-    # Setup instantiates the store within the homeserver object and updates the
-    # DB.
-    hs.setup()
-
-    if args.run_background_updates:
-        run_background_updates(hs)
-
-
-if __name__ == "__main__":
-    main()