Merge branch 'develop' into rav/cleanup_metrics

22 files changed, 443 insertions, 462 deletions

diff --git a/synapse/config/_base.py b/synapse/config/_base.py
index f7d7f153bb..36e9c04cee 100644
--- a/synapse/config/_base.py
+++ b/synapse/config/_base.py
@@ -196,6 +196,12 @@ class Config(object):
 
     @classmethod
     def load_config(cls, description, argv):
+        """Parse the commandline and config files
+
+        Doesn't support config-file-generation: used by the worker apps.
+
+        Returns: Config object.
+        """
         config_parser = argparse.ArgumentParser(description=description)
         config_parser.add_argument(
             "-c",
@@ -222,9 +228,10 @@ class Config(object):
 
         config_files = find_config_files(search_paths=config_args.config_path)
 
-        obj.read_config_files(
-            config_files, keys_directory=config_args.keys_directory, generate_keys=False
+        config_dict = obj.read_config_files(
+            config_files, keys_directory=config_args.keys_directory
         )
+        obj.parse_config_dict(config_dict)
 
         obj.invoke_all("read_arguments", config_args)
 
@@ -232,6 +239,12 @@ class Config(object):
 
     @classmethod
     def load_or_generate_config(cls, description, argv):
+        """Parse the commandline and config files
+
+        Supports generation of config files, so is used for the main homeserver app.
+
+        Returns: Config object, or None if --generate-config or --generate-keys was set
+        """
         config_parser = argparse.ArgumentParser(add_help=False)
         config_parser.add_argument(
             "-c",
@@ -241,37 +254,43 @@ class Config(object):
             help="Specify config file. Can be given multiple times and"
             " may specify directories containing *.yaml files.",
         )
-        config_parser.add_argument(
+
+        generate_group = config_parser.add_argument_group("Config generation")
+        generate_group.add_argument(
             "--generate-config",
             action="store_true",
-            help="Generate a config file for the server name",
+            help="Generate a config file, then exit.",
         )
-        config_parser.add_argument(
+        generate_group.add_argument(
+            "--generate-missing-configs",
+            "--generate-keys",
+            action="store_true",
+            help="Generate any missing additional config files, then exit.",
+        )
+        generate_group.add_argument(
+            "-H", "--server-name", help="The server name to generate a config file for."
+        )
+        generate_group.add_argument(
             "--report-stats",
             action="store",
-            help="Whether the generated config reports anonymized usage statistics",
+            help="Whether the generated config reports anonymized usage statistics.",
             choices=["yes", "no"],
         )
-        config_parser.add_argument(
-            "--generate-keys",
-            action="store_true",
-            help="Generate any missing key files then exit",
-        )
-        config_parser.add_argument(
+        generate_group.add_argument(
+            "--config-directory",
             "--keys-directory",
             metavar="DIRECTORY",
-            help="Used with 'generate-*' options to specify where files such as"
-            " signing keys should be stored, unless explicitly"
-            " specified in the config.",
-        )
-        config_parser.add_argument(
-            "-H", "--server-name", help="The server name to generate a config file for"
+            help=(
+                "Specify where additional config files such as signing keys and log"
+                " config should be stored. Defaults to the same directory as the main"
+                " config file."
+            ),
         )
         config_args, remaining_args = config_parser.parse_known_args(argv)
 
         config_files = find_config_files(search_paths=config_args.config_path)
 
-        generate_keys = config_args.generate_keys
+        generate_missing_configs = config_args.generate_missing_configs
 
         obj = cls()
 
@@ -284,13 +303,14 @@ class Config(object):
             if not config_files:
                 config_parser.error(
                     "Must supply a config file.\nA config file can be automatically"
-                    " generated using \"--generate-config -H SERVER_NAME"
-                    " -c CONFIG-FILE\""
+                    ' generated using "--generate-config -H SERVER_NAME'
+                    ' -c CONFIG-FILE"'
                 )
             (config_path,) = config_files
             if not cls.path_exists(config_path):
-                if config_args.keys_directory:
-                    config_dir_path = config_args.keys_directory
+                print("Generating config file %s" % (config_path,))
+                if config_args.config_directory:
+                    config_dir_path = config_args.config_directory
                 else:
                     config_dir_path = os.path.dirname(config_path)
                 config_dir_path = os.path.abspath(config_dir_path)
@@ -313,9 +333,7 @@ class Config(object):
                 if not cls.path_exists(config_dir_path):
                     os.makedirs(config_dir_path)
                 with open(config_path, "w") as config_file:
-                    config_file.write(
-                        "# vim:ft=yaml\n\n"
-                    )
+                    config_file.write("# vim:ft=yaml\n\n")
                     config_file.write(config_str)
 
                 config = yaml.safe_load(config_str)
@@ -333,12 +351,12 @@ class Config(object):
             else:
                 print(
                     (
-                        "Config file %r already exists. Generating any missing key"
+                        "Config file %r already exists. Generating any missing config"
                         " files."
                     )
                     % (config_path,)
                 )
-                generate_keys = True
+                generate_missing_configs = True
 
         parser = argparse.ArgumentParser(
             parents=[config_parser],
@@ -352,37 +370,43 @@ class Config(object):
         if not config_files:
             config_parser.error(
                 "Must supply a config file.\nA config file can be automatically"
-                " generated using \"--generate-config -H SERVER_NAME"
-                " -c CONFIG-FILE\""
+                ' generated using "--generate-config -H SERVER_NAME'
+                ' -c CONFIG-FILE"'
             )
 
-        obj.read_config_files(
-            config_files,
-            keys_directory=config_args.keys_directory,
-            generate_keys=generate_keys,
+        config_dict = obj.read_config_files(
+            config_files, keys_directory=config_args.config_directory
         )
 
-        if generate_keys:
+        if generate_missing_configs:
+            obj.generate_missing_files(config_dict)
             return None
 
+        obj.parse_config_dict(config_dict)
         obj.invoke_all("read_arguments", args)
 
         return obj
 
-    def read_config_files(self, config_files, keys_directory=None, generate_keys=False):
+    def read_config_files(self, config_files, keys_directory=None):
+        """Read the config files into a dict
+
+        Returns: dict
+        """
         if not keys_directory:
             keys_directory = os.path.dirname(config_files[-1])
 
         self.config_dir_path = os.path.abspath(keys_directory)
 
+        # first we read the config files into a dict
         specified_config = {}
         for config_file in config_files:
             yaml_config = self.read_config_file(config_file)
             specified_config.update(yaml_config)
 
+        # not all of the options have sensible defaults in code, so we now need to
+        # generate a default config file suitable for the specified server name...
         if "server_name" not in specified_config:
             raise ConfigError(MISSING_SERVER_NAME)
-
         server_name = specified_config["server_name"]
         config_string = self.generate_config(
             config_dir_path=self.config_dir_path,
@@ -390,7 +414,11 @@ class Config(object):
             server_name=server_name,
             generate_secrets=False,
         )
+
+        # ... and read it into a base config dict ...
         config = yaml.safe_load(config_string)
+
+        # ... and finally, overlay it with the actual configuration.
         config.pop("log_config")
         config.update(specified_config)
 
@@ -400,16 +428,14 @@ class Config(object):
                 + "\n"
                 + MISSING_REPORT_STATS_SPIEL
             )
-
-        if generate_keys:
-            self.invoke_all("generate_files", config)
-            return
-
-        self.parse_config_dict(config)
+        return config
 
     def parse_config_dict(self, config_dict):
         self.invoke_all("read_config", config_dict)
 
+    def generate_missing_files(self, config_dict):
+        self.invoke_all("generate_files", config_dict)
+
 
 def find_config_files(search_paths):
     """Finds config files using a list of search paths. If a path is a file
diff --git a/synapse/config/api.py b/synapse/config/api.py
index 5eb4f86fa2..23b0ea6962 100644
--- a/synapse/config/api.py
+++ b/synapse/config/api.py
@@ -18,15 +18,17 @@ from ._base import Config
 
 
 class ApiConfig(Config):
-
     def read_config(self, config):
-        self.room_invite_state_types = config.get("room_invite_state_types", [
-            EventTypes.JoinRules,
-            EventTypes.CanonicalAlias,
-            EventTypes.RoomAvatar,
-            EventTypes.RoomEncryption,
-            EventTypes.Name,
-        ])
+        self.room_invite_state_types = config.get(
+            "room_invite_state_types",
+            [
+                EventTypes.JoinRules,
+                EventTypes.CanonicalAlias,
+                EventTypes.RoomAvatar,
+                EventTypes.RoomEncryption,
+                EventTypes.Name,
+            ],
+        )
 
     def default_config(cls, **kwargs):
         return """\
@@ -40,4 +42,6 @@ class ApiConfig(Config):
         #  - "{RoomAvatar}"
         #  - "{RoomEncryption}"
         #  - "{Name}"
-        """.format(**vars(EventTypes))
+        """.format(
+            **vars(EventTypes)
+        )
diff --git a/synapse/config/appservice.py b/synapse/config/appservice.py
index 7e89d345d8..679ee62480 100644
--- a/synapse/config/appservice.py
+++ b/synapse/config/appservice.py
@@ -29,7 +29,6 @@ logger = logging.getLogger(__name__)
 
 
 class AppServiceConfig(Config):
-
     def read_config(self, config):
         self.app_service_config_files = config.get("app_service_config_files", [])
         self.notify_appservices = config.get("notify_appservices", True)
@@ -53,9 +52,7 @@ class AppServiceConfig(Config):
 def load_appservices(hostname, config_files):
     """Returns a list of Application Services from the config files."""
     if not isinstance(config_files, list):
-        logger.warning(
-            "Expected %s to be a list of AS config files.", config_files
-        )
+        logger.warning("Expected %s to be a list of AS config files.", config_files)
         return []
 
     # Dicts of value -> filename
@@ -66,22 +63,20 @@ def load_appservices(hostname, config_files):
 
     for config_file in config_files:
         try:
-            with open(config_file, 'r') as f:
-                appservice = _load_appservice(
-                    hostname, yaml.safe_load(f), config_file
-                )
+            with open(config_file, "r") as f:
+                appservice = _load_appservice(hostname, yaml.safe_load(f), config_file)
                 if appservice.id in seen_ids:
                     raise ConfigError(
                         "Cannot reuse ID across application services: "
-                        "%s (files: %s, %s)" % (
-                            appservice.id, config_file, seen_ids[appservice.id],
-                        )
+                        "%s (files: %s, %s)"
+                        % (appservice.id, config_file, seen_ids[appservice.id])
                     )
                 seen_ids[appservice.id] = config_file
                 if appservice.token in seen_as_tokens:
                     raise ConfigError(
                         "Cannot reuse as_token across application services: "
-                        "%s (files: %s, %s)" % (
+                        "%s (files: %s, %s)"
+                        % (
                             appservice.token,
                             config_file,
                             seen_as_tokens[appservice.token],
@@ -98,28 +93,26 @@ def load_appservices(hostname, config_files):
 
 
 def _load_appservice(hostname, as_info, config_filename):
-    required_string_fields = [
-        "id", "as_token", "hs_token", "sender_localpart"
-    ]
+    required_string_fields = ["id", "as_token", "hs_token", "sender_localpart"]
     for field in required_string_fields:
         if not isinstance(as_info.get(field), string_types):
-            raise KeyError("Required string field: '%s' (%s)" % (
-                field, config_filename,
-            ))
+            raise KeyError(
+                "Required string field: '%s' (%s)" % (field, config_filename)
+            )
 
     # 'url' must either be a string or explicitly null, not missing
     # to avoid accidentally turning off push for ASes.
-    if (not isinstance(as_info.get("url"), string_types) and
-            as_info.get("url", "") is not None):
+    if (
+        not isinstance(as_info.get("url"), string_types)
+        and as_info.get("url", "") is not None
+    ):
         raise KeyError(
             "Required string field or explicit null: 'url' (%s)" % (config_filename,)
         )
 
     localpart = as_info["sender_localpart"]
     if urlparse.quote(localpart) != localpart:
-        raise ValueError(
-            "sender_localpart needs characters which are not URL encoded."
-        )
+        raise ValueError("sender_localpart needs characters which are not URL encoded.")
     user = UserID(localpart, hostname)
     user_id = user.to_string()
 
@@ -138,13 +131,12 @@ def _load_appservice(hostname, as_info, config_filename):
             for regex_obj in as_info["namespaces"][ns]:
                 if not isinstance(regex_obj, dict):
                     raise ValueError(
-                        "Expected namespace entry in %s to be an object,"
-                        " but got %s", ns, regex_obj
+                        "Expected namespace entry in %s to be an object," " but got %s",
+                        ns,
+                        regex_obj,
                     )
                 if not isinstance(regex_obj.get("regex"), string_types):
-                    raise ValueError(
-                        "Missing/bad type 'regex' key in %s", regex_obj
-                    )
+                    raise ValueError("Missing/bad type 'regex' key in %s", regex_obj)
                 if not isinstance(regex_obj.get("exclusive"), bool):
                     raise ValueError(
                         "Missing/bad type 'exclusive' key in %s", regex_obj
@@ -167,10 +159,8 @@ def _load_appservice(hostname, as_info, config_filename):
         )
 
     ip_range_whitelist = None
-    if as_info.get('ip_range_whitelist'):
-        ip_range_whitelist = IPSet(
-            as_info.get('ip_range_whitelist')
-        )
+    if as_info.get("ip_range_whitelist"):
+        ip_range_whitelist = IPSet(as_info.get("ip_range_whitelist"))
 
     return ApplicationService(
         token=as_info["as_token"],
diff --git a/synapse/config/captcha.py b/synapse/config/captcha.py
index f7eebf26d2..e2eb473a92 100644
--- a/synapse/config/captcha.py
+++ b/synapse/config/captcha.py
@@ -16,7 +16,6 @@ from ._base import Config
 
 
 class CaptchaConfig(Config):
-
     def read_config(self, config):
         self.recaptcha_private_key = config.get("recaptcha_private_key")
         self.recaptcha_public_key = config.get("recaptcha_public_key")
diff --git a/synapse/config/consent_config.py b/synapse/config/consent_config.py
index abeb0180d3..5b0bf919c7 100644
--- a/synapse/config/consent_config.py
+++ b/synapse/config/consent_config.py
@@ -89,29 +89,26 @@ class ConsentConfig(Config):
         if consent_config is None:
             return
         self.user_consent_version = str(consent_config["version"])
-        self.user_consent_template_dir = self.abspath(
-            consent_config["template_dir"]
-        )
+        self.user_consent_template_dir = self.abspath(consent_config["template_dir"])
         if not path.isdir(self.user_consent_template_dir):
             raise ConfigError(
-                "Could not find template directory '%s'" % (
-                    self.user_consent_template_dir,
-                ),
+                "Could not find template directory '%s'"
+                % (self.user_consent_template_dir,)
             )
         self.user_consent_server_notice_content = consent_config.get(
-            "server_notice_content",
+            "server_notice_content"
         )
         self.block_events_without_consent_error = consent_config.get(
-            "block_events_error",
+            "block_events_error"
+        )
+        self.user_consent_server_notice_to_guests = bool(
+            consent_config.get("send_server_notice_to_guests", False)
+        )
+        self.user_consent_at_registration = bool(
+            consent_config.get("require_at_registration", False)
         )
-        self.user_consent_server_notice_to_guests = bool(consent_config.get(
-            "send_server_notice_to_guests", False,
-        ))
-        self.user_consent_at_registration = bool(consent_config.get(
-            "require_at_registration", False,
-        ))
         self.user_consent_policy_name = consent_config.get(
-            "policy_name", "Privacy Policy",
+            "policy_name", "Privacy Policy"
         )
 
     def default_config(self, **kwargs):
diff --git a/synapse/config/database.py b/synapse/config/database.py
index 3c27ed6b4a..adc0a47ddf 100644
--- a/synapse/config/database.py
+++ b/synapse/config/database.py
@@ -18,29 +18,21 @@ from ._base import Config
 
 
 class DatabaseConfig(Config):
-
     def read_config(self, config):
-        self.event_cache_size = self.parse_size(
-            config.get("event_cache_size", "10K")
-        )
+        self.event_cache_size = self.parse_size(config.get("event_cache_size", "10K"))
 
         self.database_config = config.get("database")
 
         if self.database_config is None:
-            self.database_config = {
-                "name": "sqlite3",
-                "args": {},
-            }
+            self.database_config = {"name": "sqlite3", "args": {}}
 
         name = self.database_config.get("name", None)
         if name == "psycopg2":
             pass
         elif name == "sqlite3":
-            self.database_config.setdefault("args", {}).update({
-                "cp_min": 1,
-                "cp_max": 1,
-                "check_same_thread": False,
-            })
+            self.database_config.setdefault("args", {}).update(
+                {"cp_min": 1, "cp_max": 1, "check_same_thread": False}
+            )
         else:
             raise RuntimeError("Unsupported database type '%s'" % (name,))
 
@@ -48,7 +40,8 @@ class DatabaseConfig(Config):
 
     def default_config(self, data_dir_path, **kwargs):
         database_path = os.path.join(data_dir_path, "homeserver.db")
-        return """\
+        return (
+            """\
         ## Database ##
 
         database:
@@ -62,7 +55,9 @@ class DatabaseConfig(Config):
         # Number of events to cache in memory.
         #
         #event_cache_size: 10K
-        """ % locals()
+        """
+            % locals()
+        )
 
     def read_arguments(self, args):
         self.set_databasepath(args.database_path)
@@ -77,6 +72,8 @@ class DatabaseConfig(Config):
     def add_arguments(self, parser):
         db_group = parser.add_argument_group("database")
         db_group.add_argument(
-            "-d", "--database-path", metavar="SQLITE_DATABASE_PATH",
-            help="The path to a sqlite database to use."
+            "-d",
+            "--database-path",
+            metavar="SQLITE_DATABASE_PATH",
+            help="The path to a sqlite database to use.",
         )
diff --git a/synapse/config/emailconfig.py b/synapse/config/emailconfig.py
index 86018dfcce..3a6cb07206 100644
--- a/synapse/config/emailconfig.py
+++ b/synapse/config/emailconfig.py
@@ -56,7 +56,7 @@ class EmailConfig(Config):
         if self.email_notif_from is not None:
             # make sure it's valid
             parsed = email.utils.parseaddr(self.email_notif_from)
-            if parsed[1] == '':
+            if parsed[1] == "":
                 raise RuntimeError("Invalid notif_from address")
 
         template_dir = email_config.get("template_dir")
@@ -65,19 +65,17 @@ class EmailConfig(Config):
         # (Note that loading as package_resources with jinja.PackageLoader doesn't
         # work for the same reason.)
         if not template_dir:
-            template_dir = pkg_resources.resource_filename(
-                'synapse', 'res/templates'
-            )
+            template_dir = pkg_resources.resource_filename("synapse", "res/templates")
 
         self.email_template_dir = os.path.abspath(template_dir)
 
         self.email_enable_notifs = email_config.get("enable_notifs", False)
-        account_validity_renewal_enabled = config.get(
-            "account_validity", {},
-        ).get("renew_at")
+        account_validity_renewal_enabled = config.get("account_validity", {}).get(
+            "renew_at"
+        )
 
         email_trust_identity_server_for_password_resets = email_config.get(
-            "trust_identity_server_for_password_resets", False,
+            "trust_identity_server_for_password_resets", False
         )
         self.email_password_reset_behaviour = (
             "remote" if email_trust_identity_server_for_password_resets else "local"
@@ -103,62 +101,59 @@ class EmailConfig(Config):
             # make sure we can import the required deps
             import jinja2
             import bleach
+
             # prevent unused warnings
             jinja2
             bleach
 
         if self.email_password_reset_behaviour == "local":
-            required = [
-                "smtp_host",
-                "smtp_port",
-                "notif_from",
-            ]
+            required = ["smtp_host", "smtp_port", "notif_from"]
 
             missing = []
             for k in required:
                 if k not in email_config:
                     missing.append(k)
 
-            if (len(missing) > 0):
+            if len(missing) > 0:
                 raise RuntimeError(
                     "email.password_reset_behaviour is set to 'local' "
-                    "but required keys are missing: %s" %
-                    (", ".join(["email." + k for k in missing]),)
+                    "but required keys are missing: %s"
+                    % (", ".join(["email." + k for k in missing]),)
                 )
 
             # Templates for password reset emails
             self.email_password_reset_template_html = email_config.get(
-                "password_reset_template_html", "password_reset.html",
+                "password_reset_template_html", "password_reset.html"
             )
             self.email_password_reset_template_text = email_config.get(
-                "password_reset_template_text", "password_reset.txt",
+                "password_reset_template_text", "password_reset.txt"
             )
             self.email_password_reset_failure_template = email_config.get(
-                "password_reset_failure_template", "password_reset_failure.html",
+                "password_reset_failure_template", "password_reset_failure.html"
             )
             # This template does not support any replaceable variables, so we will
             # read it from the disk once during setup
             email_password_reset_success_template = email_config.get(
-                "password_reset_success_template", "password_reset_success.html",
+                "password_reset_success_template", "password_reset_success.html"
             )
 
             # Check templates exist
-            for f in [self.email_password_reset_template_html,
-                      self.email_password_reset_template_text,
-                      self.email_password_reset_failure_template,
-                      email_password_reset_success_template]:
+            for f in [
+                self.email_password_reset_template_html,
+                self.email_password_reset_template_text,
+                self.email_password_reset_failure_template,
+                email_password_reset_success_template,
+            ]:
                 p = os.path.join(self.email_template_dir, f)
                 if not os.path.isfile(p):
-                    raise ConfigError("Unable to find template file %s" % (p, ))
+                    raise ConfigError("Unable to find template file %s" % (p,))
 
             # Retrieve content of web templates
             filepath = os.path.join(
-                self.email_template_dir,
-                email_password_reset_success_template,
+                self.email_template_dir, email_password_reset_success_template
             )
             self.email_password_reset_success_html_content = self.read_file(
-                filepath,
-                "email.password_reset_template_success_html",
+                filepath, "email.password_reset_template_success_html"
             )
 
             if config.get("public_baseurl") is None:
@@ -182,10 +177,10 @@ class EmailConfig(Config):
                 if k not in email_config:
                     missing.append(k)
 
-            if (len(missing) > 0):
+            if len(missing) > 0:
                 raise RuntimeError(
-                    "email.enable_notifs is True but required keys are missing: %s" %
-                    (", ".join(["email." + k for k in missing]),)
+                    "email.enable_notifs is True but required keys are missing: %s"
+                    % (", ".join(["email." + k for k in missing]),)
                 )
 
             if config.get("public_baseurl") is None:
@@ -199,27 +194,25 @@ class EmailConfig(Config):
             for f in self.email_notif_template_text, self.email_notif_template_html:
                 p = os.path.join(self.email_template_dir, f)
                 if not os.path.isfile(p):
-                    raise ConfigError("Unable to find email template file %s" % (p, ))
+                    raise ConfigError("Unable to find email template file %s" % (p,))
 
             self.email_notif_for_new_users = email_config.get(
                 "notif_for_new_users", True
             )
-            self.email_riot_base_url = email_config.get(
-                "riot_base_url", None
-            )
+            self.email_riot_base_url = email_config.get("riot_base_url", None)
 
         if account_validity_renewal_enabled:
             self.email_expiry_template_html = email_config.get(
-                "expiry_template_html", "notice_expiry.html",
+                "expiry_template_html", "notice_expiry.html"
             )
             self.email_expiry_template_text = email_config.get(
-                "expiry_template_text", "notice_expiry.txt",
+                "expiry_template_text", "notice_expiry.txt"
             )
 
             for f in self.email_expiry_template_text, self.email_expiry_template_html:
                 p = os.path.join(self.email_template_dir, f)
                 if not os.path.isfile(p):
-                    raise ConfigError("Unable to find email template file %s" % (p, ))
+                    raise ConfigError("Unable to find email template file %s" % (p,))
 
     def default_config(self, config_dir_path, server_name, **kwargs):
         return """
diff --git a/synapse/config/jwt_config.py b/synapse/config/jwt_config.py
index ecb4124096..b190dcbe38 100644
--- a/synapse/config/jwt_config.py
+++ b/synapse/config/jwt_config.py
@@ -15,13 +15,11 @@
 
 from ._base import Config, ConfigError
 
-MISSING_JWT = (
-    """Missing jwt library. This is required for jwt login.
+MISSING_JWT = """Missing jwt library. This is required for jwt login.
 
     Install by running:
         pip install pyjwt
     """
-)
 
 
 class JWTConfig(Config):
@@ -34,6 +32,7 @@ class JWTConfig(Config):
 
             try:
                 import jwt
+
                 jwt  # To stop unused lint.
             except ImportError:
                 raise ConfigError(MISSING_JWT)
diff --git a/synapse/config/key.py b/synapse/config/key.py
index 424875feae..21c4f5c51c 100644
--- a/synapse/config/key.py
+++ b/synapse/config/key.py
@@ -241,6 +241,7 @@ class KeyConfig(Config):
         signing_key_path = config["signing_key_path"]
 
         if not self.path_exists(signing_key_path):
+            print("Generating signing key file %s" % (signing_key_path,))
             with open(signing_key_path, "w") as signing_key_file:
                 key_id = "a_" + random_string(4)
                 write_signing_keys(signing_key_file, (generate_signing_key(key_id),))
@@ -348,9 +349,8 @@ def _parse_key_servers(key_servers, federation_verify_certificates):
 
                 result.verify_keys[key_id] = verify_key
 
-        if (
-            not federation_verify_certificates and
-            not server.get("accept_keys_insecurely")
+        if not federation_verify_certificates and not server.get(
+            "accept_keys_insecurely"
         ):
             _assert_keyserver_has_verify_keys(result)
 
diff --git a/synapse/config/logger.py b/synapse/config/logger.py
index c1febbe9d3..9db2e087e4 100644
--- a/synapse/config/logger.py
+++ b/synapse/config/logger.py
@@ -29,7 +29,8 @@ from synapse.util.versionstring import get_version_string
 
 from ._base import Config
 
-DEFAULT_LOG_CONFIG = Template("""
+DEFAULT_LOG_CONFIG = Template(
+    """
 version: 1
 
 formatters:
@@ -68,11 +69,11 @@ loggers:
 root:
     level: INFO
     handlers: [file, console]
-""")
+"""
+)
 
 
 class LoggingConfig(Config):
-
     def read_config(self, config):
         self.verbosity = config.get("verbose", 0)
         self.no_redirect_stdio = config.get("no_redirect_stdio", False)
@@ -81,13 +82,16 @@ class LoggingConfig(Config):
 
     def default_config(self, config_dir_path, server_name, **kwargs):
         log_config = os.path.join(config_dir_path, server_name + ".log.config")
-        return """\
+        return (
+            """\
         ## Logging ##
 
         # A yaml python logging config file
         #
         log_config: "%(log_config)s"
-        """ % locals()
+        """
+            % locals()
+        )
 
     def read_arguments(self, args):
         if args.verbose is not None:
@@ -102,32 +106,43 @@ class LoggingConfig(Config):
     def add_arguments(cls, parser):
         logging_group = parser.add_argument_group("logging")
         logging_group.add_argument(
-            '-v', '--verbose', dest="verbose", action='count',
+            "-v",
+            "--verbose",
+            dest="verbose",
+            action="count",
             help="The verbosity level. Specify multiple times to increase "
-            "verbosity. (Ignored if --log-config is specified.)"
+            "verbosity. (Ignored if --log-config is specified.)",
         )
         logging_group.add_argument(
-            '-f', '--log-file', dest="log_file",
-            help="File to log to. (Ignored if --log-config is specified.)"
+            "-f",
+            "--log-file",
+            dest="log_file",
+            help="File to log to. (Ignored if --log-config is specified.)",
         )
         logging_group.add_argument(
-            '--log-config', dest="log_config", default=None,
-            help="Python logging config file"
+            "--log-config",
+            dest="log_config",
+            default=None,
+            help="Python logging config file",
         )
         logging_group.add_argument(
-            '-n', '--no-redirect-stdio',
-            action='store_true', default=None,
-            help="Do not redirect stdout/stderr to the log"
+            "-n",
+            "--no-redirect-stdio",
+            action="store_true",
+            default=None,
+            help="Do not redirect stdout/stderr to the log",
         )
 
     def generate_files(self, config):
         log_config = config.get("log_config")
         if log_config and not os.path.exists(log_config):
             log_file = self.abspath("homeserver.log")
+            print(
+                "Generating log config file %s which will log to %s"
+                % (log_config, log_file)
+            )
             with open(log_config, "w") as log_config_file:
-                log_config_file.write(
-                    DEFAULT_LOG_CONFIG.substitute(log_file=log_file)
-                )
+                log_config_file.write(DEFAULT_LOG_CONFIG.substitute(log_file=log_file))
 
 
 def setup_logging(config, use_worker_options=False):
@@ -143,10 +158,8 @@ def setup_logging(config, use_worker_options=False):
         register_sighup (func | None): Function to call to register a
             sighup handler.
     """
-    log_config = (config.worker_log_config if use_worker_options
-                  else config.log_config)
-    log_file = (config.worker_log_file if use_worker_options
-                else config.log_file)
+    log_config = config.worker_log_config if use_worker_options else config.log_config
+    log_file = config.worker_log_file if use_worker_options else config.log_file
 
     log_format = (
         "%(asctime)s - %(name)s - %(lineno)d - %(levelname)s - %(request)s"
@@ -164,23 +177,23 @@ def setup_logging(config, use_worker_options=False):
             if config.verbosity > 1:
                 level_for_storage = logging.DEBUG
 
-        logger = logging.getLogger('')
+        logger = logging.getLogger("")
         logger.setLevel(level)
 
-        logging.getLogger('synapse.storage.SQL').setLevel(level_for_storage)
+        logging.getLogger("synapse.storage.SQL").setLevel(level_for_storage)
 
         formatter = logging.Formatter(log_format)
         if log_file:
             # TODO: Customisable file size / backup count
             handler = logging.handlers.RotatingFileHandler(
-                log_file, maxBytes=(1000 * 1000 * 100), backupCount=3,
-                encoding='utf8'
+                log_file, maxBytes=(1000 * 1000 * 100), backupCount=3, encoding="utf8"
             )
 
             def sighup(signum, stack):
                 logger.info("Closing log file due to SIGHUP")
                 handler.doRollover()
                 logger.info("Opened new log file due to SIGHUP")
+
         else:
             handler = logging.StreamHandler()
 
@@ -193,8 +206,9 @@ def setup_logging(config, use_worker_options=False):
 
         logger.addHandler(handler)
     else:
+
         def load_log_config():
-            with open(log_config, 'r') as f:
+            with open(log_config, "r") as f:
                 logging.config.dictConfig(yaml.safe_load(f))
 
         def sighup(*args):
@@ -209,10 +223,7 @@ def setup_logging(config, use_worker_options=False):
     # make sure that the first thing we log is a thing we can grep backwards
     # for
     logging.warn("***** STARTING SERVER *****")
-    logging.warn(
-        "Server %s version %s",
-        sys.argv[0], get_version_string(synapse),
-    )
+    logging.warn("Server %s version %s", sys.argv[0], get_version_string(synapse))
     logging.info("Server hostname: %s", config.server_name)
 
     # It's critical to point twisted's internal logging somewhere, otherwise it
@@ -242,8 +253,7 @@ def setup_logging(config, use_worker_options=False):
         return observer(event)
 
     globalLogBeginner.beginLoggingTo(
-        [_log],
-        redirectStandardIO=not config.no_redirect_stdio,
+        [_log], redirectStandardIO=not config.no_redirect_stdio
     )
     if not config.no_redirect_stdio:
         print("Redirected stdout/stderr to logs")
diff --git a/synapse/config/metrics.py b/synapse/config/metrics.py
index 2de51979d8..c85e234d22 100644
--- a/synapse/config/metrics.py
+++ b/synapse/config/metrics.py
@@ -15,11 +15,9 @@
 
 from ._base import Config, ConfigError
 
-MISSING_SENTRY = (
-    """Missing sentry-sdk library. This is required to enable sentry
+MISSING_SENTRY = """Missing sentry-sdk library. This is required to enable sentry
     integration.
     """
-)
 
 
 class MetricsConfig(Config):
@@ -39,7 +37,7 @@ class MetricsConfig(Config):
             self.sentry_dsn = config["sentry"].get("dsn")
             if not self.sentry_dsn:
                 raise ConfigError(
-                    "sentry.dsn field is required when sentry integration is enabled",
+                    "sentry.dsn field is required when sentry integration is enabled"
                 )
 
     def default_config(self, report_stats=None, **kwargs):
@@ -66,6 +64,6 @@ class MetricsConfig(Config):
         if report_stats is None:
             res += "# report_stats: true|false\n"
         else:
-            res += "report_stats: %s\n" % ('true' if report_stats else 'false')
+            res += "report_stats: %s\n" % ("true" if report_stats else "false")
 
         return res
diff --git a/synapse/config/password_auth_providers.py b/synapse/config/password_auth_providers.py
index f0a6be0679..fcf279e8e1 100644
--- a/synapse/config/password_auth_providers.py
+++ b/synapse/config/password_auth_providers.py
@@ -17,7 +17,7 @@ from synapse.util.module_loader import load_module
 
 from ._base import Config
 
-LDAP_PROVIDER = 'ldap_auth_provider.LdapAuthProvider'
+LDAP_PROVIDER = "ldap_auth_provider.LdapAuthProvider"
 
 
 class PasswordAuthProviderConfig(Config):
@@ -29,24 +29,20 @@ class PasswordAuthProviderConfig(Config):
         # param.
         ldap_config = config.get("ldap_config", {})
         if ldap_config.get("enabled", False):
-            providers.append({
-                'module': LDAP_PROVIDER,
-                'config': ldap_config,
-            })
+            providers.append({"module": LDAP_PROVIDER, "config": ldap_config})
 
         providers.extend(config.get("password_providers", []))
         for provider in providers:
-            mod_name = provider['module']
+            mod_name = provider["module"]
 
             # This is for backwards compat when the ldap auth provider resided
             # in this package.
             if mod_name == "synapse.util.ldap_auth_provider.LdapAuthProvider":
                 mod_name = LDAP_PROVIDER
 
-            (provider_class, provider_config) = load_module({
-                "module": mod_name,
-                "config": provider['config'],
-            })
+            (provider_class, provider_config) = load_module(
+                {"module": mod_name, "config": provider["config"]}
+            )
 
             self.password_providers.append((provider_class, provider_config))
 
diff --git a/synapse/config/registration.py b/synapse/config/registration.py
index aad3400819..a1e27ba66c 100644
--- a/synapse/config/registration.py
+++ b/synapse/config/registration.py
@@ -23,7 +23,7 @@ from synapse.util.stringutils import random_string_with_symbols
 class AccountValidityConfig(Config):
     def __init__(self, config, synapse_config):
         self.enabled = config.get("enabled", False)
-        self.renew_by_email_enabled = ("renew_at" in config)
+        self.renew_by_email_enabled = "renew_at" in config
 
         if self.enabled:
             if "period" in config:
@@ -39,14 +39,13 @@ class AccountValidityConfig(Config):
             else:
                 self.renew_email_subject = "Renew your %(app)s account"
 
-            self.startup_job_max_delta = self.period * 10. / 100.
+            self.startup_job_max_delta = self.period * 10.0 / 100.0
 
         if self.renew_by_email_enabled and "public_baseurl" not in synapse_config:
             raise ConfigError("Can't send renewal emails without 'public_baseurl'")
 
 
 class RegistrationConfig(Config):
-
     def read_config(self, config):
         self.enable_registration = bool(
             strtobool(str(config.get("enable_registration", False)))
@@ -57,7 +56,7 @@ class RegistrationConfig(Config):
             )
 
         self.account_validity = AccountValidityConfig(
-            config.get("account_validity", {}), config,
+            config.get("account_validity", {}), config
         )
 
         self.registrations_require_3pid = config.get("registrations_require_3pid", [])
@@ -67,24 +66,23 @@ class RegistrationConfig(Config):
 
         self.bcrypt_rounds = config.get("bcrypt_rounds", 12)
         self.trusted_third_party_id_servers = config.get(
-            "trusted_third_party_id_servers",
-            ["matrix.org", "vector.im"],
+            "trusted_third_party_id_servers", ["matrix.org", "vector.im"]
         )
         self.default_identity_server = config.get("default_identity_server")
         self.allow_guest_access = config.get("allow_guest_access", False)
 
-        self.invite_3pid_guest = (
-            self.allow_guest_access and config.get("invite_3pid_guest", False)
+        self.invite_3pid_guest = self.allow_guest_access and config.get(
+            "invite_3pid_guest", False
         )
 
         self.auto_join_rooms = config.get("auto_join_rooms", [])
         for room_alias in self.auto_join_rooms:
             if not RoomAlias.is_valid(room_alias):
-                raise ConfigError('Invalid auto_join_rooms entry %s' % (room_alias,))
+                raise ConfigError("Invalid auto_join_rooms entry %s" % (room_alias,))
         self.autocreate_auto_join_rooms = config.get("autocreate_auto_join_rooms", True)
 
-        self.disable_msisdn_registration = (
-            config.get("disable_msisdn_registration", False)
+        self.disable_msisdn_registration = config.get(
+            "disable_msisdn_registration", False
         )
 
     def default_config(self, generate_secrets=False, **kwargs):
@@ -93,9 +91,12 @@ class RegistrationConfig(Config):
                 random_string_with_symbols(50),
             )
         else:
-            registration_shared_secret = '# registration_shared_secret: <PRIVATE STRING>'
+            registration_shared_secret = (
+                "# registration_shared_secret: <PRIVATE STRING>"
+            )
 
-        return """\
+        return (
+            """\
         ## Registration ##
         #
         # Registration can be rate-limited using the parameters in the "Ratelimiting"
@@ -217,17 +218,19 @@ class RegistrationConfig(Config):
         # users cannot be auto-joined since they do not exist.
         #
         #autocreate_auto_join_rooms: true
-        """ % locals()
+        """
+            % locals()
+        )
 
     def add_arguments(self, parser):
         reg_group = parser.add_argument_group("registration")
         reg_group.add_argument(
-            "--enable-registration", action="store_true", default=None,
-            help="Enable registration for new users."
+            "--enable-registration",
+            action="store_true",
+            default=None,
+            help="Enable registration for new users.",
         )
 
     def read_arguments(self, args):
         if args.enable_registration is not None:
-            self.enable_registration = bool(
-                strtobool(str(args.enable_registration))
-            )
+            self.enable_registration = bool(strtobool(str(args.enable_registration)))
diff --git a/synapse/config/repository.py b/synapse/config/repository.py
index fbfcecc240..9f9669ebb1 100644
--- a/synapse/config/repository.py
+++ b/synapse/config/repository.py
@@ -20,27 +20,11 @@ from synapse.util.module_loader import load_module
 from ._base import Config, ConfigError
 
 DEFAULT_THUMBNAIL_SIZES = [
-    {
-        "width": 32,
-        "height": 32,
-        "method": "crop",
-    }, {
-        "width": 96,
-        "height": 96,
-        "method": "crop",
-    }, {
-        "width": 320,
-        "height": 240,
-        "method": "scale",
-    }, {
-        "width": 640,
-        "height": 480,
-        "method": "scale",
-    }, {
-        "width": 800,
-        "height": 600,
-        "method": "scale"
-    },
+    {"width": 32, "height": 32, "method": "crop"},
+    {"width": 96, "height": 96, "method": "crop"},
+    {"width": 320, "height": 240, "method": "scale"},
+    {"width": 640, "height": 480, "method": "scale"},
+    {"width": 800, "height": 600, "method": "scale"},
 ]
 
 THUMBNAIL_SIZE_YAML = """\
@@ -49,19 +33,15 @@ THUMBNAIL_SIZE_YAML = """\
         #    method: %(method)s
 """
 
-MISSING_NETADDR = (
-    "Missing netaddr library. This is required for URL preview API."
-)
+MISSING_NETADDR = "Missing netaddr library. This is required for URL preview API."
 
-MISSING_LXML = (
-    """Missing lxml library. This is required for URL preview API.
+MISSING_LXML = """Missing lxml library. This is required for URL preview API.
 
     Install by running:
         pip install lxml
 
     Requires libxslt1-dev system package.
     """
-)
 
 
 ThumbnailRequirement = namedtuple(
@@ -69,7 +49,8 @@ ThumbnailRequirement = namedtuple(
 )
 
 MediaStorageProviderConfig = namedtuple(
-    "MediaStorageProviderConfig", (
+    "MediaStorageProviderConfig",
+    (
         "store_local",  # Whether to store newly uploaded local files
         "store_remote",  # Whether to store newly downloaded remote files
         "store_synchronous",  # Whether to wait for successful storage for local uploads
@@ -100,8 +81,7 @@ def parse_thumbnail_requirements(thumbnail_sizes):
         requirements.setdefault("image/gif", []).append(png_thumbnail)
         requirements.setdefault("image/png", []).append(png_thumbnail)
     return {
-        media_type: tuple(thumbnails)
-        for media_type, thumbnails in requirements.items()
+        media_type: tuple(thumbnails) for media_type, thumbnails in requirements.items()
     }
 
 
@@ -127,15 +107,15 @@ class ContentRepositoryConfig(Config):
                     "Cannot use both 'backup_media_store_path' and 'storage_providers'"
                 )
 
-            storage_providers = [{
-                "module": "file_system",
-                "store_local": True,
-                "store_synchronous": synchronous_backup_media_store,
-                "store_remote": True,
-                "config": {
-                    "directory": backup_media_store_path,
+            storage_providers = [
+                {
+                    "module": "file_system",
+                    "store_local": True,
+                    "store_synchronous": synchronous_backup_media_store,
+                    "store_remote": True,
+                    "config": {"directory": backup_media_store_path},
                 }
-            }]
+            ]
 
         # This is a list of config that can be used to create the storage
         # providers. The entries are tuples of (Class, class_config,
@@ -165,18 +145,19 @@ class ContentRepositoryConfig(Config):
             )
 
             self.media_storage_providers.append(
-                (provider_class, parsed_config, wrapper_config,)
+                (provider_class, parsed_config, wrapper_config)
             )
 
         self.uploads_path = self.ensure_directory(config["uploads_path"])
         self.dynamic_thumbnails = config.get("dynamic_thumbnails", False)
         self.thumbnail_requirements = parse_thumbnail_requirements(
-            config.get("thumbnail_sizes", DEFAULT_THUMBNAIL_SIZES),
+            config.get("thumbnail_sizes", DEFAULT_THUMBNAIL_SIZES)
         )
         self.url_preview_enabled = config.get("url_preview_enabled", False)
         if self.url_preview_enabled:
             try:
                 import lxml
+
                 lxml  # To stop unused lint.
             except ImportError:
                 raise ConfigError(MISSING_LXML)
@@ -199,15 +180,13 @@ class ContentRepositoryConfig(Config):
 
             # we always blacklist '0.0.0.0' and '::', which are supposed to be
             # unroutable addresses.
-            self.url_preview_ip_range_blacklist.update(['0.0.0.0', '::'])
+            self.url_preview_ip_range_blacklist.update(["0.0.0.0", "::"])
 
             self.url_preview_ip_range_whitelist = IPSet(
                 config.get("url_preview_ip_range_whitelist", ())
             )
 
-            self.url_preview_url_blacklist = config.get(
-                "url_preview_url_blacklist", ()
-            )
+            self.url_preview_url_blacklist = config.get("url_preview_url_blacklist", ())
 
     def default_config(self, data_dir_path, **kwargs):
         media_store = os.path.join(data_dir_path, "media_store")
@@ -219,7 +198,8 @@ class ContentRepositoryConfig(Config):
         # strip final NL
         formatted_thumbnail_sizes = formatted_thumbnail_sizes[:-1]
 
-        return r"""
+        return (
+            r"""
         # Directory where uploaded images and attachments are stored.
         #
         media_store_path: "%(media_store)s"
@@ -342,4 +322,6 @@ class ContentRepositoryConfig(Config):
         # The largest allowed URL preview spidering size in bytes
         #
         #max_spider_size: 10M
-        """ % locals()
+        """
+            % locals()
+        )
diff --git a/synapse/config/room_directory.py b/synapse/config/room_directory.py
index 8a9fded4c5..c1da0e20e0 100644
--- a/synapse/config/room_directory.py
+++ b/synapse/config/room_directory.py
@@ -20,9 +20,7 @@ from ._base import Config, ConfigError
 
 class RoomDirectoryConfig(Config):
     def read_config(self, config):
-        self.enable_room_list_search = config.get(
-            "enable_room_list_search", True,
-        )
+        self.enable_room_list_search = config.get("enable_room_list_search", True)
 
         alias_creation_rules = config.get("alias_creation_rules")
 
@@ -33,11 +31,7 @@ class RoomDirectoryConfig(Config):
             ]
         else:
             self._alias_creation_rules = [
-                _RoomDirectoryRule(
-                    "alias_creation_rules", {
-                        "action": "allow",
-                    }
-                )
+                _RoomDirectoryRule("alias_creation_rules", {"action": "allow"})
             ]
 
         room_list_publication_rules = config.get("room_list_publication_rules")
@@ -49,11 +43,7 @@ class RoomDirectoryConfig(Config):
             ]
         else:
             self._room_list_publication_rules = [
-                _RoomDirectoryRule(
-                    "room_list_publication_rules", {
-                        "action": "allow",
-                    }
-                )
+                _RoomDirectoryRule("room_list_publication_rules", {"action": "allow"})
             ]
 
     def default_config(self, config_dir_path, server_name, **kwargs):
@@ -178,8 +168,7 @@ class _RoomDirectoryRule(object):
             self.action = action
         else:
             raise ConfigError(
-                "%s rules can only have action of 'allow'"
-                " or 'deny'" % (option_name,)
+                "%s rules can only have action of 'allow'" " or 'deny'" % (option_name,)
             )
 
         self._alias_matches_all = alias == "*"
diff --git a/synapse/config/saml2_config.py b/synapse/config/saml2_config.py
index aa6eac271f..2ec38e48e9 100644
--- a/synapse/config/saml2_config.py
+++ b/synapse/config/saml2_config.py
@@ -28,6 +28,7 @@ class SAML2Config(Config):
         self.saml2_enabled = True
 
         import saml2.config
+
         self.saml2_sp_config = saml2.config.SPConfig()
         self.saml2_sp_config.load(self._default_saml_config_dict())
         self.saml2_sp_config.load(saml2_config.get("sp_config", {}))
@@ -41,26 +42,23 @@ class SAML2Config(Config):
 
         public_baseurl = self.public_baseurl
         if public_baseurl is None:
-            raise ConfigError(
-                "saml2_config requires a public_baseurl to be set"
-            )
+            raise ConfigError("saml2_config requires a public_baseurl to be set")
 
         metadata_url = public_baseurl + "_matrix/saml2/metadata.xml"
         response_url = public_baseurl + "_matrix/saml2/authn_response"
         return {
             "entityid": metadata_url,
-
             "service": {
                 "sp": {
                     "endpoints": {
                         "assertion_consumer_service": [
-                            (response_url, saml2.BINDING_HTTP_POST),
-                        ],
+                            (response_url, saml2.BINDING_HTTP_POST)
+                        ]
                     },
                     "required_attributes": ["uid"],
                     "optional_attributes": ["mail", "surname", "givenname"],
-                },
-            }
+                }
+            },
         }
 
     def default_config(self, config_dir_path, server_name, **kwargs):
@@ -106,4 +104,6 @@ class SAML2Config(Config):
         #  # separate pysaml2 configuration file:
         #  #
         #  config_path: "%(config_dir_path)s/sp_conf.py"
-        """ % {"config_dir_path": config_dir_path}
+        """ % {
+            "config_dir_path": config_dir_path
+        }
diff --git a/synapse/config/server.py b/synapse/config/server.py
index 7d56e2d141..6d3f1da96c 100644
--- a/synapse/config/server.py
+++ b/synapse/config/server.py
@@ -34,13 +34,12 @@ logger = logging.Logger(__name__)
 #
 # We later check for errors when binding to 0.0.0.0 and ignore them if :: is also in
 # in the list.
-DEFAULT_BIND_ADDRESSES = ['::', '0.0.0.0']
+DEFAULT_BIND_ADDRESSES = ["::", "0.0.0.0"]
 
 DEFAULT_ROOM_VERSION = "4"
 
 
 class ServerConfig(Config):
-
     def read_config(self, config):
         self.server_name = config["server_name"]
         self.server_context = config.get("server_context", None)
@@ -81,27 +80,25 @@ class ServerConfig(Config):
         # Whether to require authentication to retrieve profile data (avatars,
         # display names) of other users through the client API.
         self.require_auth_for_profile_requests = config.get(
-            "require_auth_for_profile_requests", False,
+            "require_auth_for_profile_requests", False
         )
 
         # If set to 'True', requires authentication to access the server's
         # public rooms directory through the client API, and forbids any other
         # homeserver to fetch it via federation.
         self.restrict_public_rooms_to_local_users = config.get(
-            "restrict_public_rooms_to_local_users", False,
+            "restrict_public_rooms_to_local_users", False
         )
 
-        default_room_version = config.get(
-            "default_room_version", DEFAULT_ROOM_VERSION,
-        )
+        default_room_version = config.get("default_room_version", DEFAULT_ROOM_VERSION)
 
         # Ensure room version is a str
         default_room_version = str(default_room_version)
 
         if default_room_version not in KNOWN_ROOM_VERSIONS:
             raise ConfigError(
-                "Unknown default_room_version: %s, known room versions: %s" %
-                (default_room_version, list(KNOWN_ROOM_VERSIONS.keys()))
+                "Unknown default_room_version: %s, known room versions: %s"
+                % (default_room_version, list(KNOWN_ROOM_VERSIONS.keys()))
             )
 
         # Get the actual room version object rather than just the identifier
@@ -116,31 +113,25 @@ class ServerConfig(Config):
 
         # Whether we should block invites sent to users on this server
         # (other than those sent by local server admins)
-        self.block_non_admin_invites = config.get(
-            "block_non_admin_invites", False,
-        )
+        self.block_non_admin_invites = config.get("block_non_admin_invites", False)
 
         # Whether to enable experimental MSC1849 (aka relations) support
         self.experimental_msc1849_support_enabled = config.get(
-            "experimental_msc1849_support_enabled", False,
+            "experimental_msc1849_support_enabled", False
         )
 
         # Options to control access by tracking MAU
         self.limit_usage_by_mau = config.get("limit_usage_by_mau", False)
         self.max_mau_value = 0
         if self.limit_usage_by_mau:
-            self.max_mau_value = config.get(
-                "max_mau_value", 0,
-            )
+            self.max_mau_value = config.get("max_mau_value", 0)
         self.mau_stats_only = config.get("mau_stats_only", False)
 
         self.mau_limits_reserved_threepids = config.get(
             "mau_limit_reserved_threepids", []
         )
 
-        self.mau_trial_days = config.get(
-            "mau_trial_days", 0,
-        )
+        self.mau_trial_days = config.get("mau_trial_days", 0)
 
         # Options to disable HS
         self.hs_disabled = config.get("hs_disabled", False)
@@ -153,9 +144,7 @@ class ServerConfig(Config):
 
         # FIXME: federation_domain_whitelist needs sytests
         self.federation_domain_whitelist = None
-        federation_domain_whitelist = config.get(
-            "federation_domain_whitelist", None,
-        )
+        federation_domain_whitelist = config.get("federation_domain_whitelist", None)
 
         if federation_domain_whitelist is not None:
             # turn the whitelist into a hash for speed of lookup
@@ -165,7 +154,7 @@ class ServerConfig(Config):
                 self.federation_domain_whitelist[domain] = True
 
         self.federation_ip_range_blacklist = config.get(
-            "federation_ip_range_blacklist", [],
+            "federation_ip_range_blacklist", []
         )
 
         # Attempt to create an IPSet from the given ranges
@@ -178,13 +167,12 @@ class ServerConfig(Config):
             self.federation_ip_range_blacklist.update(["0.0.0.0", "::"])
         except Exception as e:
             raise ConfigError(
-                "Invalid range(s) provided in "
-                "federation_ip_range_blacklist: %s" % e
+                "Invalid range(s) provided in " "federation_ip_range_blacklist: %s" % e
             )
 
         if self.public_baseurl is not None:
-            if self.public_baseurl[-1] != '/':
-                self.public_baseurl += '/'
+            if self.public_baseurl[-1] != "/":
+                self.public_baseurl += "/"
         self.start_pushers = config.get("start_pushers", True)
 
         # (undocumented) option for torturing the worker-mode replication a bit,
@@ -195,7 +183,7 @@ class ServerConfig(Config):
         # Whether to require a user to be in the room to add an alias to it.
         # Defaults to True.
         self.require_membership_for_aliases = config.get(
-            "require_membership_for_aliases", True,
+            "require_membership_for_aliases", True
         )
 
         # Whether to allow per-room membership profiles through the send of membership
@@ -227,9 +215,9 @@ class ServerConfig(Config):
 
             # if we still have an empty list of addresses, use the default list
             if not bind_addresses:
-                if listener['type'] == 'metrics':
+                if listener["type"] == "metrics":
                     # the metrics listener doesn't support IPv6
-                    bind_addresses.append('0.0.0.0')
+                    bind_addresses.append("0.0.0.0")
                 else:
                     bind_addresses.extend(DEFAULT_BIND_ADDRESSES)
 
@@ -249,74 +237,74 @@ class ServerConfig(Config):
             bind_host = config.get("bind_host", "")
             gzip_responses = config.get("gzip_responses", True)
 
-            self.listeners.append({
-                "port": bind_port,
-                "bind_addresses": [bind_host],
-                "tls": True,
-                "type": "http",
-                "resources": [
-                    {
-                        "names": ["client"],
-                        "compress": gzip_responses,
-                    },
-                    {
-                        "names": ["federation"],
-                        "compress": False,
-                    }
-                ]
-            })
-
-            unsecure_port = config.get("unsecure_port", bind_port - 400)
-            if unsecure_port:
-                self.listeners.append({
-                    "port": unsecure_port,
+            self.listeners.append(
+                {
+                    "port": bind_port,
                     "bind_addresses": [bind_host],
-                    "tls": False,
+                    "tls": True,
                     "type": "http",
                     "resources": [
-                        {
-                            "names": ["client"],
-                            "compress": gzip_responses,
-                        },
-                        {
-                            "names": ["federation"],
-                            "compress": False,
-                        }
-                    ]
-                })
+                        {"names": ["client"], "compress": gzip_responses},
+                        {"names": ["federation"], "compress": False},
+                    ],
+                }
+            )
+
+            unsecure_port = config.get("unsecure_port", bind_port - 400)
+            if unsecure_port:
+                self.listeners.append(
+                    {
+                        "port": unsecure_port,
+                        "bind_addresses": [bind_host],
+                        "tls": False,
+                        "type": "http",
+                        "resources": [
+                            {"names": ["client"], "compress": gzip_responses},
+                            {"names": ["federation"], "compress": False},
+                        ],
+                    }
+                )
 
         manhole = config.get("manhole")
         if manhole:
-            self.listeners.append({
-                "port": manhole,
-                "bind_addresses": ["127.0.0.1"],
-                "type": "manhole",
-                "tls": False,
-            })
+            self.listeners.append(
+                {
+                    "port": manhole,
+                    "bind_addresses": ["127.0.0.1"],
+                    "type": "manhole",
+                    "tls": False,
+                }
+            )
 
         metrics_port = config.get("metrics_port")
         if metrics_port:
             logger.warn(
-                ("The metrics_port configuration option is deprecated in Synapse 0.31 "
-                 "in favour of a listener. Please see "
-                 "http://github.com/matrix-org/synapse/blob/master/docs/metrics-howto.rst"
-                 " on how to configure the new listener."))
-
-            self.listeners.append({
-                "port": metrics_port,
-                "bind_addresses": [config.get("metrics_bind_host", "127.0.0.1")],
-                "tls": False,
-                "type": "http",
-                "resources": [
-                    {
-                        "names": ["metrics"],
-                        "compress": False,
-                    },
-                ]
-            })
+                (
+                    "The metrics_port configuration option is deprecated in Synapse 0.31 "
+                    "in favour of a listener. Please see "
+                    "http://github.com/matrix-org/synapse/blob/master/docs/metrics-howto.rst"
+                    " on how to configure the new listener."
+                )
+            )
+
+            self.listeners.append(
+                {
+                    "port": metrics_port,
+                    "bind_addresses": [config.get("metrics_bind_host", "127.0.0.1")],
+                    "tls": False,
+                    "type": "http",
+                    "resources": [{"names": ["metrics"], "compress": False}],
+                }
+            )
 
         _check_resource_config(self.listeners)
 
+        # An experimental option to try and periodically clean up extremities
+        # by sending dummy events.
+        self.cleanup_extremities_with_dummy_events = config.get(
+            "cleanup_extremities_with_dummy_events", False
+        )
+
     def has_tls_listener(self):
         return any(l["tls"] for l in self.listeners)
 
@@ -333,7 +321,8 @@ class ServerConfig(Config):
         # Bring DEFAULT_ROOM_VERSION into the local-scope for use in the
         # default config string
         default_room_version = DEFAULT_ROOM_VERSION
-        return """\
+        return (
+            """\
         ## Server ##
 
         # The domain name of the server, with optional explicit port.
@@ -631,7 +620,9 @@ class ServerConfig(Config):
         # Defaults to 'true'.
         #
         #allow_per_room_profiles: false
-        """ % locals()
+        """
+            % locals()
+        )
 
     def read_arguments(self, args):
         if args.manhole is not None:
@@ -643,17 +634,26 @@ class ServerConfig(Config):
 
     def add_arguments(self, parser):
         server_group = parser.add_argument_group("server")
-        server_group.add_argument("-D", "--daemonize", action='store_true',
-                                  default=None,
-                                  help="Daemonize the home server")
-        server_group.add_argument("--print-pidfile", action='store_true',
-                                  default=None,
-                                  help="Print the path to the pidfile just"
-                                  " before daemonizing")
-        server_group.add_argument("--manhole", metavar="PORT", dest="manhole",
-                                  type=int,
-                                  help="Turn on the twisted telnet manhole"
-                                  " service on the given port.")
+        server_group.add_argument(
+            "-D",
+            "--daemonize",
+            action="store_true",
+            default=None,
+            help="Daemonize the home server",
+        )
+        server_group.add_argument(
+            "--print-pidfile",
+            action="store_true",
+            default=None,
+            help="Print the path to the pidfile just" " before daemonizing",
+        )
+        server_group.add_argument(
+            "--manhole",
+            metavar="PORT",
+            dest="manhole",
+            type=int,
+            help="Turn on the twisted telnet manhole" " service on the given port.",
+        )
 
 
 def is_threepid_reserved(reserved_threepids, threepid):
@@ -667,7 +667,7 @@ def is_threepid_reserved(reserved_threepids, threepid):
     """
 
     for tp in reserved_threepids:
-        if (threepid['medium'] == tp['medium'] and threepid['address'] == tp['address']):
+        if threepid["medium"] == tp["medium"] and threepid["address"] == tp["address"]:
             return True
     return False
 
@@ -680,9 +680,7 @@ def read_gc_thresholds(thresholds):
         return None
     try:
         assert len(thresholds) == 3
-        return (
-            int(thresholds[0]), int(thresholds[1]), int(thresholds[2]),
-        )
+        return (int(thresholds[0]), int(thresholds[1]), int(thresholds[2]))
     except Exception:
         raise ConfigError(
             "Value of `gc_threshold` must be a list of three integers if set"
@@ -700,22 +698,22 @@ def _warn_if_webclient_configured(listeners):
     for listener in listeners:
         for res in listener.get("resources", []):
             for name in res.get("names", []):
-                if name == 'webclient':
+                if name == "webclient":
                     logger.warning(NO_MORE_WEB_CLIENT_WARNING)
                     return
 
 
 KNOWN_RESOURCES = (
-    'client',
-    'consent',
-    'federation',
-    'keys',
-    'media',
-    'metrics',
-    'openid',
-    'replication',
-    'static',
-    'webclient',
+    "client",
+    "consent",
+    "federation",
+    "keys",
+    "media",
+    "metrics",
+    "openid",
+    "replication",
+    "static",
+    "webclient",
 )
 
 
@@ -729,11 +727,9 @@ def _check_resource_config(listeners):
 
     for resource in resource_names:
         if resource not in KNOWN_RESOURCES:
-            raise ConfigError(
-                "Unknown listener resource '%s'" % (resource, )
-            )
+            raise ConfigError("Unknown listener resource '%s'" % (resource,))
         if resource == "consent":
             try:
-                check_requirements('resources.consent')
+                check_requirements("resources.consent")
             except DependencyException as e:
                 raise ConfigError(e.message)
diff --git a/synapse/config/server_notices_config.py b/synapse/config/server_notices_config.py
index 529dc0a617..d930eb33b5 100644
--- a/synapse/config/server_notices_config.py
+++ b/synapse/config/server_notices_config.py
@@ -58,6 +58,7 @@ class ServerNoticesConfig(Config):
             The name to use for the server notices room.
             None if server notices are not enabled.
     """
+
     def __init__(self):
         super(ServerNoticesConfig, self).__init__()
         self.server_notices_mxid = None
@@ -70,18 +71,12 @@ class ServerNoticesConfig(Config):
         if c is None:
             return
 
-        mxid_localpart = c['system_mxid_localpart']
-        self.server_notices_mxid = UserID(
-            mxid_localpart, self.server_name,
-        ).to_string()
-        self.server_notices_mxid_display_name = c.get(
-            'system_mxid_display_name', None,
-        )
-        self.server_notices_mxid_avatar_url = c.get(
-            'system_mxid_avatar_url', None,
-        )
+        mxid_localpart = c["system_mxid_localpart"]
+        self.server_notices_mxid = UserID(mxid_localpart, self.server_name).to_string()
+        self.server_notices_mxid_display_name = c.get("system_mxid_display_name", None)
+        self.server_notices_mxid_avatar_url = c.get("system_mxid_avatar_url", None)
         # todo: i18n
-        self.server_notices_room_name = c.get('room_name', "Server Notices")
+        self.server_notices_room_name = c.get("room_name", "Server Notices")
 
     def default_config(self, **kwargs):
         return DEFAULT_CONFIG
diff --git a/synapse/config/tls.py b/synapse/config/tls.py
index 658f9dd361..7951bf21fa 100644
--- a/synapse/config/tls.py
+++ b/synapse/config/tls.py
@@ -42,11 +42,11 @@ class TlsConfig(Config):
         self.acme_enabled = acme_config.get("enabled", False)
 
         # hyperlink complains on py2 if this is not a Unicode
-        self.acme_url = six.text_type(acme_config.get(
-            "url", u"https://acme-v01.api.letsencrypt.org/directory"
-        ))
+        self.acme_url = six.text_type(
+            acme_config.get("url", "https://acme-v01.api.letsencrypt.org/directory")
+        )
         self.acme_port = acme_config.get("port", 80)
-        self.acme_bind_addresses = acme_config.get("bind_addresses", ['::', '0.0.0.0'])
+        self.acme_bind_addresses = acme_config.get("bind_addresses", ["::", "0.0.0.0"])
         self.acme_reprovision_threshold = acme_config.get("reprovision_threshold", 30)
         self.acme_domain = acme_config.get("domain", config.get("server_name"))
 
@@ -74,12 +74,12 @@ class TlsConfig(Config):
 
         # Whether to verify certificates on outbound federation traffic
         self.federation_verify_certificates = config.get(
-            "federation_verify_certificates", True,
+            "federation_verify_certificates", True
         )
 
         # Whitelist of domains to not verify certificates for
         fed_whitelist_entries = config.get(
-            "federation_certificate_verification_whitelist", [],
+            "federation_certificate_verification_whitelist", []
         )
 
         # Support globs (*) in whitelist values
@@ -90,9 +90,7 @@ class TlsConfig(Config):
             self.federation_certificate_verification_whitelist.append(entry_regex)
 
         # List of custom certificate authorities for federation traffic validation
-        custom_ca_list = config.get(
-            "federation_custom_ca_list", None,
-        )
+        custom_ca_list = config.get("federation_custom_ca_list", None)
 
         # Read in and parse custom CA certificates
         self.federation_ca_trust_root = None
@@ -101,8 +99,10 @@ class TlsConfig(Config):
                 # A trustroot cannot be generated without any CA certificates.
                 # Raise an error if this option has been specified without any
                 # corresponding certificates.
-                raise ConfigError("federation_custom_ca_list specified without "
-                                  "any certificate files")
+                raise ConfigError(
+                    "federation_custom_ca_list specified without "
+                    "any certificate files"
+                )
 
             certs = []
             for ca_file in custom_ca_list:
@@ -114,8 +114,9 @@ class TlsConfig(Config):
                     cert_base = Certificate.loadPEM(content)
                     certs.append(cert_base)
                 except Exception as e:
-                    raise ConfigError("Error parsing custom CA certificate file %s: %s"
-                                      % (ca_file, e))
+                    raise ConfigError(
+                        "Error parsing custom CA certificate file %s: %s" % (ca_file, e)
+                    )
 
             self.federation_ca_trust_root = trustRootFromCertificates(certs)
 
@@ -146,17 +147,21 @@ class TlsConfig(Config):
             return None
 
         try:
-            with open(self.tls_certificate_file, 'rb') as f:
+            with open(self.tls_certificate_file, "rb") as f:
                 cert_pem = f.read()
         except Exception as e:
-            raise ConfigError("Failed to read existing certificate file %s: %s"
-                              % (self.tls_certificate_file, e))
+            raise ConfigError(
+                "Failed to read existing certificate file %s: %s"
+                % (self.tls_certificate_file, e)
+            )
 
         try:
             tls_certificate = crypto.load_certificate(crypto.FILETYPE_PEM, cert_pem)
         except Exception as e:
-            raise ConfigError("Failed to parse existing certificate file %s: %s"
-                              % (self.tls_certificate_file, e))
+            raise ConfigError(
+                "Failed to parse existing certificate file %s: %s"
+                % (self.tls_certificate_file, e)
+            )
 
         if not allow_self_signed:
             if tls_certificate.get_subject() == tls_certificate.get_issuer():
@@ -166,7 +171,7 @@ class TlsConfig(Config):
 
         # YYYYMMDDhhmmssZ -- in UTC
         expires_on = datetime.strptime(
-            tls_certificate.get_notAfter().decode('ascii'), "%Y%m%d%H%M%SZ"
+            tls_certificate.get_notAfter().decode("ascii"), "%Y%m%d%H%M%SZ"
         )
         now = datetime.utcnow()
         days_remaining = (expires_on - now).days
@@ -191,7 +196,8 @@ class TlsConfig(Config):
             except Exception as e:
                 logger.info(
                     "Unable to read TLS certificate (%s). Ignoring as no "
-                    "tls listeners enabled.", e,
+                    "tls listeners enabled.",
+                    e,
                 )
 
         self.tls_fingerprints = list(self._original_tls_fingerprints)
@@ -205,7 +211,7 @@ class TlsConfig(Config):
             sha256_fingerprint = encode_base64(sha256(x509_certificate_bytes).digest())
             sha256_fingerprints = set(f["sha256"] for f in self.tls_fingerprints)
             if sha256_fingerprint not in sha256_fingerprints:
-                self.tls_fingerprints.append({u"sha256": sha256_fingerprint})
+                self.tls_fingerprints.append({"sha256": sha256_fingerprint})
 
     def default_config(self, config_dir_path, server_name, **kwargs):
         base_key_name = os.path.join(config_dir_path, server_name)
@@ -215,8 +221,8 @@ class TlsConfig(Config):
 
         # this is to avoid the max line length. Sorrynotsorry
         proxypassline = (
-            'ProxyPass /.well-known/acme-challenge '
-            'http://localhost:8009/.well-known/acme-challenge'
+            "ProxyPass /.well-known/acme-challenge "
+            "http://localhost:8009/.well-known/acme-challenge"
         )
 
         return (
diff --git a/synapse/config/user_directory.py b/synapse/config/user_directory.py
index 023997ccde..e031b11599 100644
--- a/synapse/config/user_directory.py
+++ b/synapse/config/user_directory.py
@@ -26,11 +26,11 @@ class UserDirectoryConfig(Config):
         self.user_directory_search_all_users = False
         user_directory_config = config.get("user_directory", None)
         if user_directory_config:
-            self.user_directory_search_enabled = (
-                user_directory_config.get("enabled", True)
+            self.user_directory_search_enabled = user_directory_config.get(
+                "enabled", True
             )
-            self.user_directory_search_all_users = (
-                user_directory_config.get("search_all_users", False)
+            self.user_directory_search_all_users = user_directory_config.get(
+                "search_all_users", False
             )
 
     def default_config(self, config_dir_path, server_name, **kwargs):
diff --git a/synapse/config/voip.py b/synapse/config/voip.py
index 2a1f005a37..82cf8c53a8 100644
--- a/synapse/config/voip.py
+++ b/synapse/config/voip.py
@@ -16,14 +16,13 @@ from ._base import Config
 
 
 class VoipConfig(Config):
-
     def read_config(self, config):
         self.turn_uris = config.get("turn_uris", [])
         self.turn_shared_secret = config.get("turn_shared_secret")
         self.turn_username = config.get("turn_username")
         self.turn_password = config.get("turn_password")
         self.turn_user_lifetime = self.parse_duration(
-            config.get("turn_user_lifetime", "1h"),
+            config.get("turn_user_lifetime", "1h")
         )
         self.turn_allow_guests = config.get("turn_allow_guests", True)
 
diff --git a/synapse/config/workers.py b/synapse/config/workers.py
index bfbd8b6c91..75993abf35 100644
--- a/synapse/config/workers.py
+++ b/synapse/config/workers.py
@@ -52,12 +52,14 @@ class WorkerConfig(Config):
         # argument.
         manhole = config.get("worker_manhole")
         if manhole:
-            self.worker_listeners.append({
-                "port": manhole,
-                "bind_addresses": ["127.0.0.1"],
-                "type": "manhole",
-                "tls": False,
-            })
+            self.worker_listeners.append(
+                {
+                    "port": manhole,
+                    "bind_addresses": ["127.0.0.1"],
+                    "type": "manhole",
+                    "tls": False,
+                }
+            )
 
         if self.worker_listeners:
             for listener in self.worker_listeners:
@@ -67,7 +69,7 @@ class WorkerConfig(Config):
                 if bind_address:
                     bind_addresses.append(bind_address)
                 elif not bind_addresses:
-                    bind_addresses.append('')
+                    bind_addresses.append("")
 
     def read_arguments(self, args):
         # We support a bunch of command line arguments that override options in