Merge branch 'develop' into rav/cleanup_metrics

12 files changed, 425 insertions, 408 deletions

diff --git a/synapse/rest/media/v0/content_repository.py b/synapse/rest/media/v0/content_repository.py
index 5a426ff2f6..86884c0ef4 100644
--- a/synapse/rest/media/v0/content_repository.py
+++ b/synapse/rest/media/v0/content_repository.py
@@ -44,6 +44,7 @@ class ContentRepoResource(resource.Resource):
         - Content type base64d (so we can return it when clients GET it)
 
     """
+
     isLeaf = True
 
     def __init__(self, hs, directory):
@@ -56,7 +57,7 @@ class ContentRepoResource(resource.Resource):
         # servers.
 
         # TODO: A little crude here, we could do this better.
-        filename = request.path.decode('ascii').split('/')[-1]
+        filename = request.path.decode("ascii").split("/")[-1]
         # be paranoid
         filename = re.sub("[^0-9A-z.-_]", "", filename)
 
@@ -69,17 +70,15 @@ class ContentRepoResource(resource.Resource):
             base64_contentype = filename.split(".")[1]
             content_type = base64.urlsafe_b64decode(base64_contentype)
             logger.info("Sending file %s", file_path)
-            f = open(file_path, 'rb')
-            request.setHeader('Content-Type', content_type)
+            f = open(file_path, "rb")
+            request.setHeader("Content-Type", content_type)
 
             # cache for at least a day.
             # XXX: we might want to turn this off for data we don't want to
             # recommend caching as it's sensitive or private - or at least
             # select private. don't bother setting Expires as all our matrix
             # clients are smart enough to be happy with Cache-Control (right?)
-            request.setHeader(
-                b"Cache-Control", b"public,max-age=86400,s-maxage=86400"
-            )
+            request.setHeader(b"Cache-Control", b"public,max-age=86400,s-maxage=86400")
 
             d = FileSender().beginFileTransfer(f, request)
 
@@ -87,13 +86,15 @@ class ContentRepoResource(resource.Resource):
             def cbFinished(ignored):
                 f.close()
                 finish_request(request)
+
             d.addCallback(cbFinished)
         else:
             respond_with_json_bytes(
                 request,
                 404,
                 json.dumps(cs_error("Not found", code=Codes.NOT_FOUND)),
-                send_cors=True)
+                send_cors=True,
+            )
 
         return server.NOT_DONE_YET
 
diff --git a/synapse/rest/media/v1/_base.py b/synapse/rest/media/v1/_base.py
index 2dcc8f74d6..3318638d3e 100644
--- a/synapse/rest/media/v1/_base.py
+++ b/synapse/rest/media/v1/_base.py
@@ -38,8 +38,8 @@ def parse_media_id(request):
         server_name, media_id = request.postpath[:2]
 
         if isinstance(server_name, bytes):
-            server_name = server_name.decode('utf-8')
-            media_id = media_id.decode('utf8')
+            server_name = server_name.decode("utf-8")
+            media_id = media_id.decode("utf8")
 
         file_name = None
         if len(request.postpath) > 2:
@@ -120,11 +120,11 @@ def add_file_headers(request, media_type, file_size, upload_name):
         # correctly interpret those as of 0.99.2 and (b) they are a bit of a pain and we
         # may as well just do the filename* version.
         if _can_encode_filename_as_token(upload_name):
-            disposition = 'inline; filename=%s' % (upload_name, )
+            disposition = "inline; filename=%s" % (upload_name,)
         else:
-            disposition = "inline; filename*=utf-8''%s" % (_quote(upload_name), )
+            disposition = "inline; filename*=utf-8''%s" % (_quote(upload_name),)
 
-        request.setHeader(b"Content-Disposition", disposition.encode('ascii'))
+        request.setHeader(b"Content-Disposition", disposition.encode("ascii"))
 
     # cache for at least a day.
     # XXX: we might want to turn this off for data we don't want to
@@ -137,10 +137,27 @@ def add_file_headers(request, media_type, file_size, upload_name):
 
 # separators as defined in RFC2616. SP and HT are handled separately.
 # see _can_encode_filename_as_token.
-_FILENAME_SEPARATOR_CHARS = set((
-    "(", ")", "<", ">", "@", ",", ";", ":", "\\", '"',
-    "/", "[", "]", "?", "=", "{", "}",
-))
+_FILENAME_SEPARATOR_CHARS = set(
+    (
+        "(",
+        ")",
+        "<",
+        ">",
+        "@",
+        ",",
+        ";",
+        ":",
+        "\\",
+        '"',
+        "/",
+        "[",
+        "]",
+        "?",
+        "=",
+        "{",
+        "}",
+    )
+)
 
 
 def _can_encode_filename_as_token(x):
@@ -271,7 +288,7 @@ def get_filename_from_headers(headers):
     Returns:
         A Unicode string of the filename, or None.
     """
-    content_disposition = headers.get(b"Content-Disposition", [b''])
+    content_disposition = headers.get(b"Content-Disposition", [b""])
 
     # No header, bail out.
     if not content_disposition[0]:
@@ -293,7 +310,7 @@ def get_filename_from_headers(headers):
                     # Once it is decoded, we can then unquote the %-encoded
                     # parts strictly into a unicode string.
                     upload_name = urllib.parse.unquote(
-                        upload_name_utf8.decode('ascii'), errors="strict"
+                        upload_name_utf8.decode("ascii"), errors="strict"
                     )
                 except UnicodeDecodeError:
                     # Incorrect UTF-8.
@@ -302,7 +319,7 @@ def get_filename_from_headers(headers):
                 # On Python 2, we first unquote the %-encoded parts and then
                 # decode it strictly using UTF-8.
                 try:
-                    upload_name = urllib.parse.unquote(upload_name_utf8).decode('utf8')
+                    upload_name = urllib.parse.unquote(upload_name_utf8).decode("utf8")
                 except UnicodeDecodeError:
                     pass
 
@@ -310,7 +327,7 @@ def get_filename_from_headers(headers):
     if not upload_name:
         upload_name_ascii = params.get(b"filename", None)
         if upload_name_ascii and is_ascii(upload_name_ascii):
-            upload_name = upload_name_ascii.decode('ascii')
+            upload_name = upload_name_ascii.decode("ascii")
 
     # This may be None here, indicating we did not find a matching name.
     return upload_name
@@ -328,19 +345,19 @@ def _parse_header(line):
         Tuple[bytes, dict[bytes, bytes]]:
             the main content-type, followed by the parameter dictionary
     """
-    parts = _parseparam(b';' + line)
+    parts = _parseparam(b";" + line)
     key = next(parts)
     pdict = {}
     for p in parts:
-        i = p.find(b'=')
+        i = p.find(b"=")
         if i >= 0:
             name = p[:i].strip().lower()
-            value = p[i + 1:].strip()
+            value = p[i + 1 :].strip()
 
             # strip double-quotes
             if len(value) >= 2 and value[0:1] == value[-1:] == b'"':
                 value = value[1:-1]
-                value = value.replace(b'\\\\', b'\\').replace(b'\\"', b'"')
+                value = value.replace(b"\\\\", b"\\").replace(b'\\"', b'"')
             pdict[name] = value
 
     return key, pdict
@@ -357,16 +374,16 @@ def _parseparam(s):
     Returns:
         Iterable[bytes]: the split input
     """
-    while s[:1] == b';':
+    while s[:1] == b";":
         s = s[1:]
 
         # look for the next ;
-        end = s.find(b';')
+        end = s.find(b";")
 
         # if there is an odd number of " marks between here and the next ;, skip to the
         # next ; instead
         while end > 0 and (s.count(b'"', 0, end) - s.count(b'\\"', 0, end)) % 2:
-            end = s.find(b';', end + 1)
+            end = s.find(b";", end + 1)
 
         if end < 0:
             end = len(s)
diff --git a/synapse/rest/media/v1/config_resource.py b/synapse/rest/media/v1/config_resource.py
index 77316033f7..fa3d6680fc 100644
--- a/synapse/rest/media/v1/config_resource.py
+++ b/synapse/rest/media/v1/config_resource.py
@@ -29,9 +29,7 @@ class MediaConfigResource(Resource):
         config = hs.get_config()
         self.clock = hs.get_clock()
         self.auth = hs.get_auth()
-        self.limits_dict = {
-            "m.upload.size": config.max_upload_size,
-        }
+        self.limits_dict = {"m.upload.size": config.max_upload_size}
 
     def render_GET(self, request):
         self._async_render_GET(request)
diff --git a/synapse/rest/media/v1/download_resource.py b/synapse/rest/media/v1/download_resource.py
index bdc5daecc1..a21a35f843 100644
--- a/synapse/rest/media/v1/download_resource.py
+++ b/synapse/rest/media/v1/download_resource.py
@@ -54,18 +54,20 @@ class DownloadResource(Resource):
             b" plugin-types application/pdf;"
             b" style-src 'unsafe-inline';"
             b" media-src 'self';"
-            b" object-src 'self';"
+            b" object-src 'self';",
         )
         server_name, media_id, name = parse_media_id(request)
         if server_name == self.server_name:
             yield self.media_repo.get_local_media(request, media_id, name)
         else:
             allow_remote = synapse.http.servlet.parse_boolean(
-                request, "allow_remote", default=True)
+                request, "allow_remote", default=True
+            )
             if not allow_remote:
                 logger.info(
                     "Rejecting request for remote media %s/%s due to allow_remote",
-                    server_name, media_id,
+                    server_name,
+                    media_id,
                 )
                 respond_404(request)
                 return
diff --git a/synapse/rest/media/v1/filepath.py b/synapse/rest/media/v1/filepath.py
index c8586fa280..e25c382c9c 100644
--- a/synapse/rest/media/v1/filepath.py
+++ b/synapse/rest/media/v1/filepath.py
@@ -24,6 +24,7 @@ def _wrap_in_base_path(func):
     """Takes a function that returns a relative path and turns it into an
     absolute path based on the location of the primary media store
     """
+
     @functools.wraps(func)
     def _wrapped(self, *args, **kwargs):
         path = func(self, *args, **kwargs)
@@ -43,125 +44,102 @@ class MediaFilePaths(object):
     def __init__(self, primary_base_path):
         self.base_path = primary_base_path
 
-    def default_thumbnail_rel(self, default_top_level, default_sub_type, width,
-                              height, content_type, method):
+    def default_thumbnail_rel(
+        self, default_top_level, default_sub_type, width, height, content_type, method
+    ):
         top_level_type, sub_type = content_type.split("/")
-        file_name = "%i-%i-%s-%s-%s" % (
-            width, height, top_level_type, sub_type, method
-        )
+        file_name = "%i-%i-%s-%s-%s" % (width, height, top_level_type, sub_type, method)
         return os.path.join(
-            "default_thumbnails", default_top_level,
-            default_sub_type, file_name
+            "default_thumbnails", default_top_level, default_sub_type, file_name
         )
 
     default_thumbnail = _wrap_in_base_path(default_thumbnail_rel)
 
     def local_media_filepath_rel(self, media_id):
-        return os.path.join(
-            "local_content",
-            media_id[0:2], media_id[2:4], media_id[4:]
-        )
+        return os.path.join("local_content", media_id[0:2], media_id[2:4], media_id[4:])
 
     local_media_filepath = _wrap_in_base_path(local_media_filepath_rel)
 
-    def local_media_thumbnail_rel(self, media_id, width, height, content_type,
-                                  method):
+    def local_media_thumbnail_rel(self, media_id, width, height, content_type, method):
         top_level_type, sub_type = content_type.split("/")
-        file_name = "%i-%i-%s-%s-%s" % (
-            width, height, top_level_type, sub_type, method
-        )
+        file_name = "%i-%i-%s-%s-%s" % (width, height, top_level_type, sub_type, method)
         return os.path.join(
-            "local_thumbnails",
-            media_id[0:2], media_id[2:4], media_id[4:],
-            file_name
+            "local_thumbnails", media_id[0:2], media_id[2:4], media_id[4:], file_name
         )
 
     local_media_thumbnail = _wrap_in_base_path(local_media_thumbnail_rel)
 
     def remote_media_filepath_rel(self, server_name, file_id):
         return os.path.join(
-            "remote_content", server_name,
-            file_id[0:2], file_id[2:4], file_id[4:]
+            "remote_content", server_name, file_id[0:2], file_id[2:4], file_id[4:]
         )
 
     remote_media_filepath = _wrap_in_base_path(remote_media_filepath_rel)
 
-    def remote_media_thumbnail_rel(self, server_name, file_id, width, height,
-                                   content_type, method):
+    def remote_media_thumbnail_rel(
+        self, server_name, file_id, width, height, content_type, method
+    ):
         top_level_type, sub_type = content_type.split("/")
         file_name = "%i-%i-%s-%s" % (width, height, top_level_type, sub_type)
         return os.path.join(
-            "remote_thumbnail", server_name,
-            file_id[0:2], file_id[2:4], file_id[4:],
-            file_name
+            "remote_thumbnail",
+            server_name,
+            file_id[0:2],
+            file_id[2:4],
+            file_id[4:],
+            file_name,
         )
 
     remote_media_thumbnail = _wrap_in_base_path(remote_media_thumbnail_rel)
 
     def remote_media_thumbnail_dir(self, server_name, file_id):
         return os.path.join(
-            self.base_path, "remote_thumbnail", server_name,
-            file_id[0:2], file_id[2:4], file_id[4:],
+            self.base_path,
+            "remote_thumbnail",
+            server_name,
+            file_id[0:2],
+            file_id[2:4],
+            file_id[4:],
         )
 
     def url_cache_filepath_rel(self, media_id):
         if NEW_FORMAT_ID_RE.match(media_id):
             # Media id is of the form <DATE><RANDOM_STRING>
             # E.g.: 2017-09-28-fsdRDt24DS234dsf
-            return os.path.join(
-                "url_cache",
-                media_id[:10], media_id[11:]
-            )
+            return os.path.join("url_cache", media_id[:10], media_id[11:])
         else:
-            return os.path.join(
-                "url_cache",
-                media_id[0:2], media_id[2:4], media_id[4:],
-            )
+            return os.path.join("url_cache", media_id[0:2], media_id[2:4], media_id[4:])
 
     url_cache_filepath = _wrap_in_base_path(url_cache_filepath_rel)
 
     def url_cache_filepath_dirs_to_delete(self, media_id):
         "The dirs to try and remove if we delete the media_id file"
         if NEW_FORMAT_ID_RE.match(media_id):
-            return [
-                os.path.join(
-                    self.base_path, "url_cache",
-                    media_id[:10],
-                ),
-            ]
+            return [os.path.join(self.base_path, "url_cache", media_id[:10])]
         else:
             return [
-                os.path.join(
-                    self.base_path, "url_cache",
-                    media_id[0:2], media_id[2:4],
-                ),
-                os.path.join(
-                    self.base_path, "url_cache",
-                    media_id[0:2],
-                ),
+                os.path.join(self.base_path, "url_cache", media_id[0:2], media_id[2:4]),
+                os.path.join(self.base_path, "url_cache", media_id[0:2]),
             ]
 
-    def url_cache_thumbnail_rel(self, media_id, width, height, content_type,
-                                method):
+    def url_cache_thumbnail_rel(self, media_id, width, height, content_type, method):
         # Media id is of the form <DATE><RANDOM_STRING>
         # E.g.: 2017-09-28-fsdRDt24DS234dsf
 
         top_level_type, sub_type = content_type.split("/")
-        file_name = "%i-%i-%s-%s-%s" % (
-            width, height, top_level_type, sub_type, method
-        )
+        file_name = "%i-%i-%s-%s-%s" % (width, height, top_level_type, sub_type, method)
 
         if NEW_FORMAT_ID_RE.match(media_id):
             return os.path.join(
-                "url_cache_thumbnails",
-                media_id[:10], media_id[11:],
-                file_name
+                "url_cache_thumbnails", media_id[:10], media_id[11:], file_name
             )
         else:
             return os.path.join(
                 "url_cache_thumbnails",
-                media_id[0:2], media_id[2:4], media_id[4:],
-                file_name
+                media_id[0:2],
+                media_id[2:4],
+                media_id[4:],
+                file_name,
             )
 
     url_cache_thumbnail = _wrap_in_base_path(url_cache_thumbnail_rel)
@@ -172,13 +150,15 @@ class MediaFilePaths(object):
 
         if NEW_FORMAT_ID_RE.match(media_id):
             return os.path.join(
-                self.base_path, "url_cache_thumbnails",
-                media_id[:10], media_id[11:],
+                self.base_path, "url_cache_thumbnails", media_id[:10], media_id[11:]
             )
         else:
             return os.path.join(
-                self.base_path, "url_cache_thumbnails",
-                media_id[0:2], media_id[2:4], media_id[4:],
+                self.base_path,
+                "url_cache_thumbnails",
+                media_id[0:2],
+                media_id[2:4],
+                media_id[4:],
             )
 
     def url_cache_thumbnail_dirs_to_delete(self, media_id):
@@ -188,26 +168,21 @@ class MediaFilePaths(object):
         if NEW_FORMAT_ID_RE.match(media_id):
             return [
                 os.path.join(
-                    self.base_path, "url_cache_thumbnails",
-                    media_id[:10], media_id[11:],
-                ),
-                os.path.join(
-                    self.base_path, "url_cache_thumbnails",
-                    media_id[:10],
+                    self.base_path, "url_cache_thumbnails", media_id[:10], media_id[11:]
                 ),
+                os.path.join(self.base_path, "url_cache_thumbnails", media_id[:10]),
             ]
         else:
             return [
                 os.path.join(
-                    self.base_path, "url_cache_thumbnails",
-                    media_id[0:2], media_id[2:4], media_id[4:],
-                ),
-                os.path.join(
-                    self.base_path, "url_cache_thumbnails",
-                    media_id[0:2], media_id[2:4],
+                    self.base_path,
+                    "url_cache_thumbnails",
+                    media_id[0:2],
+                    media_id[2:4],
+                    media_id[4:],
                 ),
                 os.path.join(
-                    self.base_path, "url_cache_thumbnails",
-                    media_id[0:2],
+                    self.base_path, "url_cache_thumbnails", media_id[0:2], media_id[2:4]
                 ),
+                os.path.join(self.base_path, "url_cache_thumbnails", media_id[0:2]),
             ]
diff --git a/synapse/rest/media/v1/media_repository.py b/synapse/rest/media/v1/media_repository.py
index a4929dd5db..df3d985a38 100644
--- a/synapse/rest/media/v1/media_repository.py
+++ b/synapse/rest/media/v1/media_repository.py
@@ -100,17 +100,16 @@ class MediaRepository(object):
             storage_providers.append(provider)
 
         self.media_storage = MediaStorage(
-            self.hs, self.primary_base_path, self.filepaths, storage_providers,
+            self.hs, self.primary_base_path, self.filepaths, storage_providers
         )
 
         self.clock.looping_call(
-            self._start_update_recently_accessed,
-            UPDATE_RECENTLY_ACCESSED_TS,
+            self._start_update_recently_accessed, UPDATE_RECENTLY_ACCESSED_TS
         )
 
     def _start_update_recently_accessed(self):
         return run_as_background_process(
-            "update_recently_accessed_media", self._update_recently_accessed,
+            "update_recently_accessed_media", self._update_recently_accessed
         )
 
     @defer.inlineCallbacks
@@ -138,8 +137,9 @@ class MediaRepository(object):
             self.recently_accessed_locals.add(media_id)
 
     @defer.inlineCallbacks
-    def create_content(self, media_type, upload_name, content, content_length,
-                       auth_user):
+    def create_content(
+        self, media_type, upload_name, content, content_length, auth_user
+    ):
         """Store uploaded content for a local user and return the mxc URL
 
         Args:
@@ -154,10 +154,7 @@ class MediaRepository(object):
         """
         media_id = random_string(24)
 
-        file_info = FileInfo(
-            server_name=None,
-            file_id=media_id,
-        )
+        file_info = FileInfo(server_name=None, file_id=media_id)
 
         fname = yield self.media_storage.store_file(content, file_info)
 
@@ -172,9 +169,7 @@ class MediaRepository(object):
             user_id=auth_user,
         )
 
-        yield self._generate_thumbnails(
-            None, media_id, media_id, media_type,
-        )
+        yield self._generate_thumbnails(None, media_id, media_id, media_type)
 
         defer.returnValue("mxc://%s/%s" % (self.server_name, media_id))
 
@@ -205,14 +200,11 @@ class MediaRepository(object):
         upload_name = name if name else media_info["upload_name"]
         url_cache = media_info["url_cache"]
 
-        file_info = FileInfo(
-            None, media_id,
-            url_cache=url_cache,
-        )
+        file_info = FileInfo(None, media_id, url_cache=url_cache)
 
         responder = yield self.media_storage.fetch_media(file_info)
         yield respond_with_responder(
-            request, responder, media_type, media_length, upload_name,
+            request, responder, media_type, media_length, upload_name
         )
 
     @defer.inlineCallbacks
@@ -232,8 +224,8 @@ class MediaRepository(object):
                 to request
         """
         if (
-            self.federation_domain_whitelist is not None and
-            server_name not in self.federation_domain_whitelist
+            self.federation_domain_whitelist is not None
+            and server_name not in self.federation_domain_whitelist
         ):
             raise FederationDeniedError(server_name)
 
@@ -244,7 +236,7 @@ class MediaRepository(object):
         key = (server_name, media_id)
         with (yield self.remote_media_linearizer.queue(key)):
             responder, media_info = yield self._get_remote_media_impl(
-                server_name, media_id,
+                server_name, media_id
             )
 
         # We deliberately stream the file outside the lock
@@ -253,7 +245,7 @@ class MediaRepository(object):
             media_length = media_info["media_length"]
             upload_name = name if name else media_info["upload_name"]
             yield respond_with_responder(
-                request, responder, media_type, media_length, upload_name,
+                request, responder, media_type, media_length, upload_name
             )
         else:
             respond_404(request)
@@ -272,8 +264,8 @@ class MediaRepository(object):
             Deferred[dict]: The media_info of the file
         """
         if (
-            self.federation_domain_whitelist is not None and
-            server_name not in self.federation_domain_whitelist
+            self.federation_domain_whitelist is not None
+            and server_name not in self.federation_domain_whitelist
         ):
             raise FederationDeniedError(server_name)
 
@@ -282,7 +274,7 @@ class MediaRepository(object):
         key = (server_name, media_id)
         with (yield self.remote_media_linearizer.queue(key)):
             responder, media_info = yield self._get_remote_media_impl(
-                server_name, media_id,
+                server_name, media_id
             )
 
         # Ensure we actually use the responder so that it releases resources
@@ -305,9 +297,7 @@ class MediaRepository(object):
         Returns:
             Deferred[(Responder, media_info)]
         """
-        media_info = yield self.store.get_cached_remote_media(
-            server_name, media_id
-        )
+        media_info = yield self.store.get_cached_remote_media(server_name, media_id)
 
         # file_id is the ID we use to track the file locally. If we've already
         # seen the file then reuse the existing ID, otherwise genereate a new
@@ -331,9 +321,7 @@ class MediaRepository(object):
 
         # Failed to find the file anywhere, lets download it.
 
-        media_info = yield self._download_remote_file(
-            server_name, media_id, file_id
-        )
+        media_info = yield self._download_remote_file(server_name, media_id, file_id)
 
         responder = yield self.media_storage.fetch_media(file_info)
         defer.returnValue((responder, media_info))
@@ -354,54 +342,60 @@ class MediaRepository(object):
             Deferred[MediaInfo]
         """
 
-        file_info = FileInfo(
-            server_name=server_name,
-            file_id=file_id,
-        )
+        file_info = FileInfo(server_name=server_name, file_id=file_id)
 
         with self.media_storage.store_into_file(file_info) as (f, fname, finish):
-            request_path = "/".join((
-                "/_matrix/media/v1/download", server_name, media_id,
-            ))
+            request_path = "/".join(
+                ("/_matrix/media/v1/download", server_name, media_id)
+            )
             try:
                 length, headers = yield self.client.get_file(
-                    server_name, request_path, output_stream=f,
-                    max_size=self.max_upload_size, args={
+                    server_name,
+                    request_path,
+                    output_stream=f,
+                    max_size=self.max_upload_size,
+                    args={
                         # tell the remote server to 404 if it doesn't
                         # recognise the server_name, to make sure we don't
                         # end up with a routing loop.
-                        "allow_remote": "false",
-                    }
+                        "allow_remote": "false"
+                    },
                 )
             except RequestSendFailed as e:
-                logger.warn("Request failed fetching remote media %s/%s: %r",
-                            server_name, media_id, e)
+                logger.warn(
+                    "Request failed fetching remote media %s/%s: %r",
+                    server_name,
+                    media_id,
+                    e,
+                )
                 raise SynapseError(502, "Failed to fetch remote media")
 
             except HttpResponseException as e:
-                logger.warn("HTTP error fetching remote media %s/%s: %s",
-                            server_name, media_id, e.response)
+                logger.warn(
+                    "HTTP error fetching remote media %s/%s: %s",
+                    server_name,
+                    media_id,
+                    e.response,
+                )
                 if e.code == twisted.web.http.NOT_FOUND:
                     raise e.to_synapse_error()
                 raise SynapseError(502, "Failed to fetch remote media")
 
             except SynapseError:
-                logger.warn(
-                    "Failed to fetch remote media %s/%s",
-                    server_name, media_id,
-                )
+                logger.warn("Failed to fetch remote media %s/%s", server_name, media_id)
                 raise
             except NotRetryingDestination:
                 logger.warn("Not retrying destination %r", server_name)
                 raise SynapseError(502, "Failed to fetch remote media")
             except Exception:
-                logger.exception("Failed to fetch remote media %s/%s",
-                                 server_name, media_id)
+                logger.exception(
+                    "Failed to fetch remote media %s/%s", server_name, media_id
+                )
                 raise SynapseError(502, "Failed to fetch remote media")
 
             yield finish()
 
-        media_type = headers[b"Content-Type"][0].decode('ascii')
+        media_type = headers[b"Content-Type"][0].decode("ascii")
         upload_name = get_filename_from_headers(headers)
         time_now_ms = self.clock.time_msec()
 
@@ -425,24 +419,23 @@ class MediaRepository(object):
             "filesystem_id": file_id,
         }
 
-        yield self._generate_thumbnails(
-            server_name, media_id, file_id, media_type,
-        )
+        yield self._generate_thumbnails(server_name, media_id, file_id, media_type)
 
         defer.returnValue(media_info)
 
     def _get_thumbnail_requirements(self, media_type):
         return self.thumbnail_requirements.get(media_type, ())
 
-    def _generate_thumbnail(self, thumbnailer, t_width, t_height,
-                            t_method, t_type):
+    def _generate_thumbnail(self, thumbnailer, t_width, t_height, t_method, t_type):
         m_width = thumbnailer.width
         m_height = thumbnailer.height
 
         if m_width * m_height >= self.max_image_pixels:
             logger.info(
                 "Image too large to thumbnail %r x %r > %r",
-                m_width, m_height, self.max_image_pixels
+                m_width,
+                m_height,
+                self.max_image_pixels,
             )
             return
 
@@ -462,17 +455,22 @@ class MediaRepository(object):
         return t_byte_source
 
     @defer.inlineCallbacks
-    def generate_local_exact_thumbnail(self, media_id, t_width, t_height,
-                                       t_method, t_type, url_cache):
-        input_path = yield self.media_storage.ensure_media_is_in_local_cache(FileInfo(
-            None, media_id, url_cache=url_cache,
-        ))
+    def generate_local_exact_thumbnail(
+        self, media_id, t_width, t_height, t_method, t_type, url_cache
+    ):
+        input_path = yield self.media_storage.ensure_media_is_in_local_cache(
+            FileInfo(None, media_id, url_cache=url_cache)
+        )
 
         thumbnailer = Thumbnailer(input_path)
         t_byte_source = yield logcontext.defer_to_thread(
             self.hs.get_reactor(),
             self._generate_thumbnail,
-            thumbnailer, t_width, t_height, t_method, t_type
+            thumbnailer,
+            t_width,
+            t_height,
+            t_method,
+            t_type,
         )
 
         if t_byte_source:
@@ -489,7 +487,7 @@ class MediaRepository(object):
                 )
 
                 output_path = yield self.media_storage.store_file(
-                    t_byte_source, file_info,
+                    t_byte_source, file_info
                 )
             finally:
                 t_byte_source.close()
@@ -505,17 +503,22 @@ class MediaRepository(object):
             defer.returnValue(output_path)
 
     @defer.inlineCallbacks
-    def generate_remote_exact_thumbnail(self, server_name, file_id, media_id,
-                                        t_width, t_height, t_method, t_type):
-        input_path = yield self.media_storage.ensure_media_is_in_local_cache(FileInfo(
-            server_name, file_id, url_cache=False,
-        ))
+    def generate_remote_exact_thumbnail(
+        self, server_name, file_id, media_id, t_width, t_height, t_method, t_type
+    ):
+        input_path = yield self.media_storage.ensure_media_is_in_local_cache(
+            FileInfo(server_name, file_id, url_cache=False)
+        )
 
         thumbnailer = Thumbnailer(input_path)
         t_byte_source = yield logcontext.defer_to_thread(
             self.hs.get_reactor(),
             self._generate_thumbnail,
-            thumbnailer, t_width, t_height, t_method, t_type
+            thumbnailer,
+            t_width,
+            t_height,
+            t_method,
+            t_type,
         )
 
         if t_byte_source:
@@ -531,7 +534,7 @@ class MediaRepository(object):
                 )
 
                 output_path = yield self.media_storage.store_file(
-                    t_byte_source, file_info,
+                    t_byte_source, file_info
                 )
             finally:
                 t_byte_source.close()
@@ -541,15 +544,22 @@ class MediaRepository(object):
             t_len = os.path.getsize(output_path)
 
             yield self.store.store_remote_media_thumbnail(
-                server_name, media_id, file_id,
-                t_width, t_height, t_type, t_method, t_len
+                server_name,
+                media_id,
+                file_id,
+                t_width,
+                t_height,
+                t_type,
+                t_method,
+                t_len,
             )
 
             defer.returnValue(output_path)
 
     @defer.inlineCallbacks
-    def _generate_thumbnails(self, server_name, media_id, file_id, media_type,
-                             url_cache=False):
+    def _generate_thumbnails(
+        self, server_name, media_id, file_id, media_type, url_cache=False
+    ):
         """Generate and store thumbnails for an image.
 
         Args:
@@ -568,9 +578,9 @@ class MediaRepository(object):
         if not requirements:
             return
 
-        input_path = yield self.media_storage.ensure_media_is_in_local_cache(FileInfo(
-            server_name, file_id, url_cache=url_cache,
-        ))
+        input_path = yield self.media_storage.ensure_media_is_in_local_cache(
+            FileInfo(server_name, file_id, url_cache=url_cache)
+        )
 
         thumbnailer = Thumbnailer(input_path)
         m_width = thumbnailer.width
@@ -579,14 +589,15 @@ class MediaRepository(object):
         if m_width * m_height >= self.max_image_pixels:
             logger.info(
                 "Image too large to thumbnail %r x %r > %r",
-                m_width, m_height, self.max_image_pixels
+                m_width,
+                m_height,
+                self.max_image_pixels,
             )
             return
 
         if thumbnailer.transpose_method is not None:
             m_width, m_height = yield logcontext.defer_to_thread(
-                self.hs.get_reactor(),
-                thumbnailer.transpose
+                self.hs.get_reactor(), thumbnailer.transpose
             )
 
         # We deduplicate the thumbnail sizes by ignoring the cropped versions if
@@ -606,15 +617,11 @@ class MediaRepository(object):
             # Generate the thumbnail
             if t_method == "crop":
                 t_byte_source = yield logcontext.defer_to_thread(
-                    self.hs.get_reactor(),
-                    thumbnailer.crop,
-                    t_width, t_height, t_type,
+                    self.hs.get_reactor(), thumbnailer.crop, t_width, t_height, t_type
                 )
             elif t_method == "scale":
                 t_byte_source = yield logcontext.defer_to_thread(
-                    self.hs.get_reactor(),
-                    thumbnailer.scale,
-                    t_width, t_height, t_type,
+                    self.hs.get_reactor(), thumbnailer.scale, t_width, t_height, t_type
                 )
             else:
                 logger.error("Unrecognized method: %r", t_method)
@@ -636,7 +643,7 @@ class MediaRepository(object):
                 )
 
                 output_path = yield self.media_storage.store_file(
-                    t_byte_source, file_info,
+                    t_byte_source, file_info
                 )
             finally:
                 t_byte_source.close()
@@ -646,18 +653,21 @@ class MediaRepository(object):
             # Write to database
             if server_name:
                 yield self.store.store_remote_media_thumbnail(
-                    server_name, media_id, file_id,
-                    t_width, t_height, t_type, t_method, t_len
+                    server_name,
+                    media_id,
+                    file_id,
+                    t_width,
+                    t_height,
+                    t_type,
+                    t_method,
+                    t_len,
                 )
             else:
                 yield self.store.store_local_thumbnail(
                     media_id, t_width, t_height, t_type, t_method, t_len
                 )
 
-        defer.returnValue({
-            "width": m_width,
-            "height": m_height,
-        })
+        defer.returnValue({"width": m_width, "height": m_height})
 
     @defer.inlineCallbacks
     def delete_old_remote_media(self, before_ts):
@@ -749,11 +759,12 @@ class MediaRepositoryResource(Resource):
 
         self.putChild(b"upload", UploadResource(hs, media_repo))
         self.putChild(b"download", DownloadResource(hs, media_repo))
-        self.putChild(b"thumbnail", ThumbnailResource(
-            hs, media_repo, media_repo.media_storage,
-        ))
+        self.putChild(
+            b"thumbnail", ThumbnailResource(hs, media_repo, media_repo.media_storage)
+        )
         if hs.config.url_preview_enabled:
-            self.putChild(b"preview_url", PreviewUrlResource(
-                hs, media_repo, media_repo.media_storage,
-            ))
+            self.putChild(
+                b"preview_url",
+                PreviewUrlResource(hs, media_repo, media_repo.media_storage),
+            )
         self.putChild(b"config", MediaConfigResource(hs))
diff --git a/synapse/rest/media/v1/media_storage.py b/synapse/rest/media/v1/media_storage.py
index 896078fe76..eff86836fb 100644
--- a/synapse/rest/media/v1/media_storage.py
+++ b/synapse/rest/media/v1/media_storage.py
@@ -66,8 +66,7 @@ class MediaStorage(object):
         with self.store_into_file(file_info) as (f, fname, finish_cb):
             # Write to the main repository
             yield logcontext.defer_to_thread(
-                self.hs.get_reactor(),
-                _write_file_synchronously, source, f,
+                self.hs.get_reactor(), _write_file_synchronously, source, f
             )
             yield finish_cb()
 
@@ -179,7 +178,8 @@ class MediaStorage(object):
             if res:
                 with res:
                     consumer = BackgroundFileConsumer(
-                        open(local_path, "wb"), self.hs.get_reactor())
+                        open(local_path, "wb"), self.hs.get_reactor()
+                    )
                     yield res.write_to_consumer(consumer)
                     yield consumer.wait()
                 defer.returnValue(local_path)
@@ -217,10 +217,10 @@ class MediaStorage(object):
                     width=file_info.thumbnail_width,
                     height=file_info.thumbnail_height,
                     content_type=file_info.thumbnail_type,
-                    method=file_info.thumbnail_method
+                    method=file_info.thumbnail_method,
                 )
             return self.filepaths.remote_media_filepath_rel(
-                file_info.server_name, file_info.file_id,
+                file_info.server_name, file_info.file_id
             )
 
         if file_info.thumbnail:
@@ -229,11 +229,9 @@ class MediaStorage(object):
                 width=file_info.thumbnail_width,
                 height=file_info.thumbnail_height,
                 content_type=file_info.thumbnail_type,
-                method=file_info.thumbnail_method
+                method=file_info.thumbnail_method,
             )
-        return self.filepaths.local_media_filepath_rel(
-            file_info.file_id,
-        )
+        return self.filepaths.local_media_filepath_rel(file_info.file_id)
 
 
 def _write_file_synchronously(source, dest):
@@ -255,6 +253,7 @@ class FileResponder(Responder):
         open_file (file): A file like object to be streamed ot the client,
             is closed when finished streaming.
     """
+
     def __init__(self, open_file):
         self.open_file = open_file
 
diff --git a/synapse/rest/media/v1/preview_url_resource.py b/synapse/rest/media/v1/preview_url_resource.py
index acf87709f2..de6f292ffb 100644
--- a/synapse/rest/media/v1/preview_url_resource.py
+++ b/synapse/rest/media/v1/preview_url_resource.py
@@ -92,7 +92,7 @@ class PreviewUrlResource(Resource):
         )
 
         self._cleaner_loop = self.clock.looping_call(
-            self._start_expire_url_cache_data, 10 * 1000,
+            self._start_expire_url_cache_data, 10 * 1000
         )
 
     def render_OPTIONS(self, request):
@@ -121,16 +121,16 @@ class PreviewUrlResource(Resource):
             for attrib in entry:
                 pattern = entry[attrib]
                 value = getattr(url_tuple, attrib)
-                logger.debug((
-                    "Matching attrib '%s' with value '%s' against"
-                    " pattern '%s'"
-                ) % (attrib, value, pattern))
+                logger.debug(
+                    ("Matching attrib '%s' with value '%s' against" " pattern '%s'")
+                    % (attrib, value, pattern)
+                )
 
                 if value is None:
                     match = False
                     continue
 
-                if pattern.startswith('^'):
+                if pattern.startswith("^"):
                     if not re.match(pattern, getattr(url_tuple, attrib)):
                         match = False
                         continue
@@ -139,12 +139,9 @@ class PreviewUrlResource(Resource):
                         match = False
                         continue
             if match:
-                logger.warn(
-                    "URL %s blocked by url_blacklist entry %s", url, entry
-                )
+                logger.warn("URL %s blocked by url_blacklist entry %s", url, entry)
                 raise SynapseError(
-                    403, "URL blocked by url pattern blacklist entry",
-                    Codes.UNKNOWN
+                    403, "URL blocked by url pattern blacklist entry", Codes.UNKNOWN
                 )
 
         # the in-memory cache:
@@ -156,14 +153,8 @@ class PreviewUrlResource(Resource):
         observable = self._cache.get(url)
 
         if not observable:
-            download = run_in_background(
-                self._do_preview,
-                url, requester.user, ts,
-            )
-            observable = ObservableDeferred(
-                download,
-                consumeErrors=True
-            )
+            download = run_in_background(self._do_preview, url, requester.user, ts)
+            observable = ObservableDeferred(download, consumeErrors=True)
             self._cache[url] = observable
         else:
             logger.info("Returning cached response")
@@ -187,15 +178,15 @@ class PreviewUrlResource(Resource):
         # historical previews, if we have any)
         cache_result = yield self.store.get_url_cache(url, ts)
         if (
-            cache_result and
-            cache_result["expires_ts"] > ts and
-            cache_result["response_code"] / 100 == 2
+            cache_result
+            and cache_result["expires_ts"] > ts
+            and cache_result["response_code"] / 100 == 2
         ):
             # It may be stored as text in the database, not as bytes (such as
             # PostgreSQL). If so, encode it back before handing it on.
             og = cache_result["og"]
             if isinstance(og, six.text_type):
-                og = og.encode('utf8')
+                og = og.encode("utf8")
             defer.returnValue(og)
             return
 
@@ -203,33 +194,31 @@ class PreviewUrlResource(Resource):
 
         logger.debug("got media_info of '%s'" % media_info)
 
-        if _is_media(media_info['media_type']):
-            file_id = media_info['filesystem_id']
+        if _is_media(media_info["media_type"]):
+            file_id = media_info["filesystem_id"]
             dims = yield self.media_repo._generate_thumbnails(
-                None, file_id, file_id, media_info["media_type"],
-                url_cache=True,
+                None, file_id, file_id, media_info["media_type"], url_cache=True
             )
 
             og = {
-                "og:description": media_info['download_name'],
-                "og:image": "mxc://%s/%s" % (
-                    self.server_name, media_info['filesystem_id']
-                ),
-                "og:image:type": media_info['media_type'],
-                "matrix:image:size": media_info['media_length'],
+                "og:description": media_info["download_name"],
+                "og:image": "mxc://%s/%s"
+                % (self.server_name, media_info["filesystem_id"]),
+                "og:image:type": media_info["media_type"],
+                "matrix:image:size": media_info["media_length"],
             }
 
             if dims:
-                og["og:image:width"] = dims['width']
-                og["og:image:height"] = dims['height']
+                og["og:image:width"] = dims["width"]
+                og["og:image:height"] = dims["height"]
             else:
                 logger.warn("Couldn't get dims for %s" % url)
 
             # define our OG response for this media
-        elif _is_html(media_info['media_type']):
+        elif _is_html(media_info["media_type"]):
             # TODO: somehow stop a big HTML tree from exploding synapse's RAM
 
-            with open(media_info['filename'], 'rb') as file:
+            with open(media_info["filename"], "rb") as file:
                 body = file.read()
 
             encoding = None
@@ -242,45 +231,43 @@ class PreviewUrlResource(Resource):
             # If we find a match, it should take precedence over the
             # Content-Type header, so set it here.
             if match:
-                encoding = match.group(1).decode('ascii')
+                encoding = match.group(1).decode("ascii")
 
             # If we don't find a match, we'll look at the HTTP Content-Type, and
             # if that doesn't exist, we'll fall back to UTF-8.
             if not encoding:
-                match = _content_type_match.match(
-                    media_info['media_type']
-                )
+                match = _content_type_match.match(media_info["media_type"])
                 encoding = match.group(1) if match else "utf-8"
 
-            og = decode_and_calc_og(body, media_info['uri'], encoding)
+            og = decode_and_calc_og(body, media_info["uri"], encoding)
 
             # pre-cache the image for posterity
             # FIXME: it might be cleaner to use the same flow as the main /preview_url
             # request itself and benefit from the same caching etc.  But for now we
             # just rely on the caching on the master request to speed things up.
-            if 'og:image' in og and og['og:image']:
+            if "og:image" in og and og["og:image"]:
                 image_info = yield self._download_url(
-                    _rebase_url(og['og:image'], media_info['uri']), user
+                    _rebase_url(og["og:image"], media_info["uri"]), user
                 )
 
-                if _is_media(image_info['media_type']):
+                if _is_media(image_info["media_type"]):
                     # TODO: make sure we don't choke on white-on-transparent images
-                    file_id = image_info['filesystem_id']
+                    file_id = image_info["filesystem_id"]
                     dims = yield self.media_repo._generate_thumbnails(
-                        None, file_id, file_id, image_info["media_type"],
-                        url_cache=True,
+                        None, file_id, file_id, image_info["media_type"], url_cache=True
                     )
                     if dims:
-                        og["og:image:width"] = dims['width']
-                        og["og:image:height"] = dims['height']
+                        og["og:image:width"] = dims["width"]
+                        og["og:image:height"] = dims["height"]
                     else:
                         logger.warn("Couldn't get dims for %s" % og["og:image"])
 
                     og["og:image"] = "mxc://%s/%s" % (
-                        self.server_name, image_info['filesystem_id']
+                        self.server_name,
+                        image_info["filesystem_id"],
                     )
-                    og["og:image:type"] = image_info['media_type']
-                    og["matrix:image:size"] = image_info['media_length']
+                    og["og:image:type"] = image_info["media_type"]
+                    og["matrix:image:size"] = image_info["media_length"]
                 else:
                     del og["og:image"]
         else:
@@ -289,7 +276,7 @@ class PreviewUrlResource(Resource):
 
         logger.debug("Calculated OG for %s as %s" % (url, og))
 
-        jsonog = json.dumps(og).encode('utf8')
+        jsonog = json.dumps(og).encode("utf8")
 
         # store OG in history-aware DB cache
         yield self.store.store_url_cache(
@@ -310,19 +297,15 @@ class PreviewUrlResource(Resource):
         # we're most likely being explicitly triggered by a human rather than a
         # bot, so are we really a robot?
 
-        file_id = datetime.date.today().isoformat() + '_' + random_string(16)
+        file_id = datetime.date.today().isoformat() + "_" + random_string(16)
 
-        file_info = FileInfo(
-            server_name=None,
-            file_id=file_id,
-            url_cache=True,
-        )
+        file_info = FileInfo(server_name=None, file_id=file_id, url_cache=True)
 
         with self.media_storage.store_into_file(file_info) as (f, fname, finish):
             try:
                 logger.debug("Trying to get url '%s'" % url)
                 length, headers, uri, code = yield self.client.get_file(
-                    url, output_stream=f, max_size=self.max_spider_size,
+                    url, output_stream=f, max_size=self.max_spider_size
                 )
             except SynapseError:
                 # Pass SynapseErrors through directly, so that the servlet
@@ -334,24 +317,25 @@ class PreviewUrlResource(Resource):
                 # Note: This will also be the case if one of the resolved IP
                 # addresses is blacklisted
                 raise SynapseError(
-                    502, "DNS resolution failure during URL preview generation",
-                    Codes.UNKNOWN
+                    502,
+                    "DNS resolution failure during URL preview generation",
+                    Codes.UNKNOWN,
                 )
             except Exception as e:
                 # FIXME: pass through 404s and other error messages nicely
                 logger.warn("Error downloading %s: %r", url, e)
 
                 raise SynapseError(
-                    500, "Failed to download content: %s" % (
-                        traceback.format_exception_only(sys.exc_info()[0], e),
-                    ),
+                    500,
+                    "Failed to download content: %s"
+                    % (traceback.format_exception_only(sys.exc_info()[0], e),),
                     Codes.UNKNOWN,
                 )
             yield finish()
 
         try:
             if b"Content-Type" in headers:
-                media_type = headers[b"Content-Type"][0].decode('ascii')
+                media_type = headers[b"Content-Type"][0].decode("ascii")
             else:
                 media_type = "application/octet-stream"
             time_now_ms = self.clock.time_msec()
@@ -375,24 +359,26 @@ class PreviewUrlResource(Resource):
             # therefore not expire it.
             raise
 
-        defer.returnValue({
-            "media_type": media_type,
-            "media_length": length,
-            "download_name": download_name,
-            "created_ts": time_now_ms,
-            "filesystem_id": file_id,
-            "filename": fname,
-            "uri": uri,
-            "response_code": code,
-            # FIXME: we should calculate a proper expiration based on the
-            # Cache-Control and Expire headers.  But for now, assume 1 hour.
-            "expires": 60 * 60 * 1000,
-            "etag": headers["ETag"][0] if "ETag" in headers else None,
-        })
+        defer.returnValue(
+            {
+                "media_type": media_type,
+                "media_length": length,
+                "download_name": download_name,
+                "created_ts": time_now_ms,
+                "filesystem_id": file_id,
+                "filename": fname,
+                "uri": uri,
+                "response_code": code,
+                # FIXME: we should calculate a proper expiration based on the
+                # Cache-Control and Expire headers.  But for now, assume 1 hour.
+                "expires": 60 * 60 * 1000,
+                "etag": headers["ETag"][0] if "ETag" in headers else None,
+            }
+        )
 
     def _start_expire_url_cache_data(self):
         return run_as_background_process(
-            "expire_url_cache_data", self._expire_url_cache_data,
+            "expire_url_cache_data", self._expire_url_cache_data
         )
 
     @defer.inlineCallbacks
@@ -496,7 +482,7 @@ def decode_and_calc_og(body, media_uri, request_encoding=None):
         # blindly try decoding the body as utf-8, which seems to fix
         # the charset mismatches on https://google.com
         parser = etree.HTMLParser(recover=True, encoding=request_encoding)
-        tree = etree.fromstring(body.decode('utf-8', 'ignore'), parser)
+        tree = etree.fromstring(body.decode("utf-8", "ignore"), parser)
         og = _calc_og(tree, media_uri)
 
     return og
@@ -523,8 +509,8 @@ def _calc_og(tree, media_uri):
 
     og = {}
     for tag in tree.xpath("//*/meta[starts-with(@property, 'og:')]"):
-        if 'content' in tag.attrib:
-            og[tag.attrib['property']] = tag.attrib['content']
+        if "content" in tag.attrib:
+            og[tag.attrib["property"]] = tag.attrib["content"]
 
     # TODO: grab article: meta tags too, e.g.:
 
@@ -535,39 +521,43 @@ def _calc_og(tree, media_uri):
     # "article:published_time" content="2016-03-31T19:58:24+00:00" />
     # "article:modified_time" content="2016-04-01T18:31:53+00:00" />
 
-    if 'og:title' not in og:
+    if "og:title" not in og:
         # do some basic spidering of the HTML
         title = tree.xpath("(//title)[1] | (//h1)[1] | (//h2)[1] | (//h3)[1]")
         if title and title[0].text is not None:
-            og['og:title'] = title[0].text.strip()
+            og["og:title"] = title[0].text.strip()
         else:
-            og['og:title'] = None
+            og["og:title"] = None
 
-    if 'og:image' not in og:
+    if "og:image" not in og:
         # TODO: extract a favicon failing all else
         meta_image = tree.xpath(
             "//*/meta[translate(@itemprop, 'IMAGE', 'image')='image']/@content"
         )
         if meta_image:
-            og['og:image'] = _rebase_url(meta_image[0], media_uri)
+            og["og:image"] = _rebase_url(meta_image[0], media_uri)
         else:
             # TODO: consider inlined CSS styles as well as width & height attribs
             images = tree.xpath("//img[@src][number(@width)>10][number(@height)>10]")
-            images = sorted(images, key=lambda i: (
-                -1 * float(i.attrib['width']) * float(i.attrib['height'])
-            ))
+            images = sorted(
+                images,
+                key=lambda i: (
+                    -1 * float(i.attrib["width"]) * float(i.attrib["height"])
+                ),
+            )
             if not images:
                 images = tree.xpath("//img[@src]")
             if images:
-                og['og:image'] = images[0].attrib['src']
+                og["og:image"] = images[0].attrib["src"]
 
-    if 'og:description' not in og:
+    if "og:description" not in og:
         meta_description = tree.xpath(
             "//*/meta"
             "[translate(@name, 'DESCRIPTION', 'description')='description']"
-            "/@content")
+            "/@content"
+        )
         if meta_description:
-            og['og:description'] = meta_description[0]
+            og["og:description"] = meta_description[0]
         else:
             # grab any text nodes which are inside the <body/> tag...
             # unless they are within an HTML5 semantic markup tag...
@@ -588,18 +578,18 @@ def _calc_og(tree, media_uri):
                 "script",
                 "noscript",
                 "style",
-                etree.Comment
+                etree.Comment,
             )
 
             # Split all the text nodes into paragraphs (by splitting on new
             # lines)
             text_nodes = (
-                re.sub(r'\s+', '\n', el).strip()
+                re.sub(r"\s+", "\n", el).strip()
                 for el in _iterate_over_text(tree.find("body"), *TAGS_TO_REMOVE)
             )
-            og['og:description'] = summarize_paragraphs(text_nodes)
+            og["og:description"] = summarize_paragraphs(text_nodes)
     else:
-        og['og:description'] = summarize_paragraphs([og['og:description']])
+        og["og:description"] = summarize_paragraphs([og["og:description"]])
 
     # TODO: delete the url downloads to stop diskfilling,
     # as we only ever cared about its OG
@@ -636,7 +626,7 @@ def _iterate_over_text(tree, *tags_to_ignore):
                     [child, child.tail] if child.tail else [child]
                     for child in el.iterchildren()
                 ),
-                elements
+                elements,
             )
 
 
@@ -647,8 +637,8 @@ def _rebase_url(url, base):
         url[0] = base[0] or "http"
     if not url[1]:  # fix up hostname
         url[1] = base[1]
-        if not url[2].startswith('/'):
-            url[2] = re.sub(r'/[^/]+$', '/', base[2]) + url[2]
+        if not url[2].startswith("/"):
+            url[2] = re.sub(r"/[^/]+$", "/", base[2]) + url[2]
     return urlparse.urlunparse(url)
 
 
@@ -659,9 +649,8 @@ def _is_media(content_type):
 
 def _is_html(content_type):
     content_type = content_type.lower()
-    if (
-        content_type.startswith("text/html") or
-        content_type.startswith("application/xhtml")
+    if content_type.startswith("text/html") or content_type.startswith(
+        "application/xhtml"
     ):
         return True
 
@@ -671,19 +660,19 @@ def summarize_paragraphs(text_nodes, min_size=200, max_size=500):
     # first paragraph and then word boundaries.
     # TODO: Respect sentences?
 
-    description = ''
+    description = ""
 
     # Keep adding paragraphs until we get to the MIN_SIZE.
     for text_node in text_nodes:
         if len(description) < min_size:
-            text_node = re.sub(r'[\t \r\n]+', ' ', text_node)
-            description += text_node + '\n\n'
+            text_node = re.sub(r"[\t \r\n]+", " ", text_node)
+            description += text_node + "\n\n"
         else:
             break
 
     description = description.strip()
-    description = re.sub(r'[\t ]+', ' ', description)
-    description = re.sub(r'[\t \r\n]*[\r\n]+', '\n\n', description)
+    description = re.sub(r"[\t ]+", " ", description)
+    description = re.sub(r"[\t \r\n]*[\r\n]+", "\n\n", description)
 
     # If the concatenation of paragraphs to get above MIN_SIZE
     # took us over MAX_SIZE, then we need to truncate mid paragraph
@@ -715,5 +704,5 @@ def summarize_paragraphs(text_nodes, min_size=200, max_size=500):
 
         # We always add an ellipsis because at the very least
         # we chopped mid paragraph.
-        description = new_desc.strip() + u"…"
+        description = new_desc.strip() + "…"
     return description if description else None
diff --git a/synapse/rest/media/v1/storage_provider.py b/synapse/rest/media/v1/storage_provider.py
index d90cbfb56a..359b45ebfc 100644
--- a/synapse/rest/media/v1/storage_provider.py
+++ b/synapse/rest/media/v1/storage_provider.py
@@ -32,6 +32,7 @@ class StorageProvider(object):
     """A storage provider is a service that can store uploaded media and
     retrieve them.
     """
+
     def store_file(self, path, file_info):
         """Store the file described by file_info. The actual contents can be
         retrieved by reading the file in file_info.upload_path.
@@ -70,6 +71,7 @@ class StorageProviderWrapper(StorageProvider):
             uploaded, or todo the upload in the backgroud.
         store_remote (bool): Whether remote media should be uploaded
     """
+
     def __init__(self, backend, store_local, store_synchronous, store_remote):
         self.backend = backend
         self.store_local = store_local
@@ -92,6 +94,7 @@ class StorageProviderWrapper(StorageProvider):
                     return self.backend.store_file(path, file_info)
                 except Exception:
                     logger.exception("Error storing file")
+
             run_in_background(store)
             return defer.succeed(None)
 
@@ -123,8 +126,7 @@ class FileStorageProviderBackend(StorageProvider):
             os.makedirs(dirname)
 
         return logcontext.defer_to_thread(
-            self.hs.get_reactor(),
-            shutil.copyfile, primary_fname, backup_fname,
+            self.hs.get_reactor(), shutil.copyfile, primary_fname, backup_fname
         )
 
     def fetch(self, path, file_info):
diff --git a/synapse/rest/media/v1/thumbnail_resource.py b/synapse/rest/media/v1/thumbnail_resource.py
index 35a750923b..ca84c9f139 100644
--- a/synapse/rest/media/v1/thumbnail_resource.py
+++ b/synapse/rest/media/v1/thumbnail_resource.py
@@ -74,19 +74,18 @@ class ThumbnailResource(Resource):
         else:
             if self.dynamic_thumbnails:
                 yield self._select_or_generate_remote_thumbnail(
-                    request, server_name, media_id,
-                    width, height, method, m_type
+                    request, server_name, media_id, width, height, method, m_type
                 )
             else:
                 yield self._respond_remote_thumbnail(
-                    request, server_name, media_id,
-                    width, height, method, m_type
+                    request, server_name, media_id, width, height, method, m_type
                 )
             self.media_repo.mark_recently_accessed(server_name, media_id)
 
     @defer.inlineCallbacks
-    def _respond_local_thumbnail(self, request, media_id, width, height,
-                                 method, m_type):
+    def _respond_local_thumbnail(
+        self, request, media_id, width, height, method, m_type
+    ):
         media_info = yield self.store.get_local_media(media_id)
 
         if not media_info:
@@ -105,7 +104,8 @@ class ThumbnailResource(Resource):
             )
 
             file_info = FileInfo(
-                server_name=None, file_id=media_id,
+                server_name=None,
+                file_id=media_id,
                 url_cache=media_info["url_cache"],
                 thumbnail=True,
                 thumbnail_width=thumbnail_info["thumbnail_width"],
@@ -124,9 +124,15 @@ class ThumbnailResource(Resource):
             respond_404(request)
 
     @defer.inlineCallbacks
-    def _select_or_generate_local_thumbnail(self, request, media_id, desired_width,
-                                            desired_height, desired_method,
-                                            desired_type):
+    def _select_or_generate_local_thumbnail(
+        self,
+        request,
+        media_id,
+        desired_width,
+        desired_height,
+        desired_method,
+        desired_type,
+    ):
         media_info = yield self.store.get_local_media(media_id)
 
         if not media_info:
@@ -146,7 +152,8 @@ class ThumbnailResource(Resource):
 
             if t_w and t_h and t_method and t_type:
                 file_info = FileInfo(
-                    server_name=None, file_id=media_id,
+                    server_name=None,
+                    file_id=media_id,
                     url_cache=media_info["url_cache"],
                     thumbnail=True,
                     thumbnail_width=info["thumbnail_width"],
@@ -167,7 +174,11 @@ class ThumbnailResource(Resource):
 
         # Okay, so we generate one.
         file_path = yield self.media_repo.generate_local_exact_thumbnail(
-            media_id, desired_width, desired_height, desired_method, desired_type,
+            media_id,
+            desired_width,
+            desired_height,
+            desired_method,
+            desired_type,
             url_cache=media_info["url_cache"],
         )
 
@@ -178,13 +189,20 @@ class ThumbnailResource(Resource):
             respond_404(request)
 
     @defer.inlineCallbacks
-    def _select_or_generate_remote_thumbnail(self, request, server_name, media_id,
-                                             desired_width, desired_height,
-                                             desired_method, desired_type):
+    def _select_or_generate_remote_thumbnail(
+        self,
+        request,
+        server_name,
+        media_id,
+        desired_width,
+        desired_height,
+        desired_method,
+        desired_type,
+    ):
         media_info = yield self.media_repo.get_remote_media_info(server_name, media_id)
 
         thumbnail_infos = yield self.store.get_remote_media_thumbnails(
-            server_name, media_id,
+            server_name, media_id
         )
 
         file_id = media_info["filesystem_id"]
@@ -197,7 +215,8 @@ class ThumbnailResource(Resource):
 
             if t_w and t_h and t_method and t_type:
                 file_info = FileInfo(
-                    server_name=server_name, file_id=media_info["filesystem_id"],
+                    server_name=server_name,
+                    file_id=media_info["filesystem_id"],
                     thumbnail=True,
                     thumbnail_width=info["thumbnail_width"],
                     thumbnail_height=info["thumbnail_height"],
@@ -217,8 +236,13 @@ class ThumbnailResource(Resource):
 
         # Okay, so we generate one.
         file_path = yield self.media_repo.generate_remote_exact_thumbnail(
-            server_name, file_id, media_id, desired_width,
-            desired_height, desired_method, desired_type
+            server_name,
+            file_id,
+            media_id,
+            desired_width,
+            desired_height,
+            desired_method,
+            desired_type,
         )
 
         if file_path:
@@ -228,15 +252,16 @@ class ThumbnailResource(Resource):
             respond_404(request)
 
     @defer.inlineCallbacks
-    def _respond_remote_thumbnail(self, request, server_name, media_id, width,
-                                  height, method, m_type):
+    def _respond_remote_thumbnail(
+        self, request, server_name, media_id, width, height, method, m_type
+    ):
         # TODO: Don't download the whole remote file
         # We should proxy the thumbnail from the remote server instead of
         # downloading the remote file and generating our own thumbnails.
         media_info = yield self.media_repo.get_remote_media_info(server_name, media_id)
 
         thumbnail_infos = yield self.store.get_remote_media_thumbnails(
-            server_name, media_id,
+            server_name, media_id
         )
 
         if thumbnail_infos:
@@ -244,7 +269,8 @@ class ThumbnailResource(Resource):
                 width, height, method, m_type, thumbnail_infos
             )
             file_info = FileInfo(
-                server_name=server_name, file_id=media_info["filesystem_id"],
+                server_name=server_name,
+                file_id=media_info["filesystem_id"],
                 thumbnail=True,
                 thumbnail_width=thumbnail_info["thumbnail_width"],
                 thumbnail_height=thumbnail_info["thumbnail_height"],
@@ -261,8 +287,14 @@ class ThumbnailResource(Resource):
             logger.info("Failed to find any generated thumbnails")
             respond_404(request)
 
-    def _select_thumbnail(self, desired_width, desired_height, desired_method,
-                          desired_type, thumbnail_infos):
+    def _select_thumbnail(
+        self,
+        desired_width,
+        desired_height,
+        desired_method,
+        desired_type,
+        thumbnail_infos,
+    ):
         d_w = desired_width
         d_h = desired_height
 
@@ -280,15 +312,27 @@ class ThumbnailResource(Resource):
                     type_quality = desired_type != info["thumbnail_type"]
                     length_quality = info["thumbnail_length"]
                     if t_w >= d_w or t_h >= d_h:
-                        info_list.append((
-                            aspect_quality, min_quality, size_quality, type_quality,
-                            length_quality, info
-                        ))
+                        info_list.append(
+                            (
+                                aspect_quality,
+                                min_quality,
+                                size_quality,
+                                type_quality,
+                                length_quality,
+                                info,
+                            )
+                        )
                     else:
-                        info_list2.append((
-                            aspect_quality, min_quality, size_quality, type_quality,
-                            length_quality, info
-                        ))
+                        info_list2.append(
+                            (
+                                aspect_quality,
+                                min_quality,
+                                size_quality,
+                                type_quality,
+                                length_quality,
+                                info,
+                            )
+                        )
             if info_list:
                 return min(info_list)[-1]
             else:
@@ -304,13 +348,11 @@ class ThumbnailResource(Resource):
                 type_quality = desired_type != info["thumbnail_type"]
                 length_quality = info["thumbnail_length"]
                 if t_method == "scale" and (t_w >= d_w or t_h >= d_h):
-                    info_list.append((
-                        size_quality, type_quality, length_quality, info
-                    ))
+                    info_list.append((size_quality, type_quality, length_quality, info))
                 elif t_method == "scale":
-                    info_list2.append((
-                        size_quality, type_quality, length_quality, info
-                    ))
+                    info_list2.append(
+                        (size_quality, type_quality, length_quality, info)
+                    )
             if info_list:
                 return min(info_list)[-1]
             else:
diff --git a/synapse/rest/media/v1/thumbnailer.py b/synapse/rest/media/v1/thumbnailer.py
index 3efd0d80fc..90d8e6bffe 100644
--- a/synapse/rest/media/v1/thumbnailer.py
+++ b/synapse/rest/media/v1/thumbnailer.py
@@ -28,16 +28,13 @@ EXIF_TRANSPOSE_MAPPINGS = {
     5: Image.TRANSPOSE,
     6: Image.ROTATE_270,
     7: Image.TRANSVERSE,
-    8: Image.ROTATE_90
+    8: Image.ROTATE_90,
 }
 
 
 class Thumbnailer(object):
 
-    FORMATS = {
-        "image/jpeg": "JPEG",
-        "image/png": "PNG",
-    }
+    FORMATS = {"image/jpeg": "JPEG", "image/png": "PNG"}
 
     def __init__(self, input_path):
         self.image = Image.open(input_path)
@@ -110,17 +107,13 @@ class Thumbnailer(object):
         """
         if width * self.height > height * self.width:
             scaled_height = (width * self.height) // self.width
-            scaled_image = self.image.resize(
-                (width, scaled_height), Image.ANTIALIAS
-            )
+            scaled_image = self.image.resize((width, scaled_height), Image.ANTIALIAS)
             crop_top = (scaled_height - height) // 2
             crop_bottom = height + crop_top
             cropped = scaled_image.crop((0, crop_top, width, crop_bottom))
         else:
             scaled_width = (height * self.width) // self.height
-            scaled_image = self.image.resize(
-                (scaled_width, height), Image.ANTIALIAS
-            )
+            scaled_image = self.image.resize((scaled_width, height), Image.ANTIALIAS)
             crop_left = (scaled_width - width) // 2
             crop_right = width + crop_left
             cropped = scaled_image.crop((crop_left, 0, crop_right, height))
diff --git a/synapse/rest/media/v1/upload_resource.py b/synapse/rest/media/v1/upload_resource.py
index c1240e1963..d1d7e959f0 100644
--- a/synapse/rest/media/v1/upload_resource.py
+++ b/synapse/rest/media/v1/upload_resource.py
@@ -55,48 +55,36 @@ class UploadResource(Resource):
         requester = yield self.auth.get_user_by_req(request)
         # TODO: The checks here are a bit late. The content will have
         # already been uploaded to a tmp file at this point
-        content_length = request.getHeader(b"Content-Length").decode('ascii')
+        content_length = request.getHeader(b"Content-Length").decode("ascii")
         if content_length is None:
-            raise SynapseError(
-                msg="Request must specify a Content-Length", code=400
-            )
+            raise SynapseError(msg="Request must specify a Content-Length", code=400)
         if int(content_length) > self.max_upload_size:
-            raise SynapseError(
-                msg="Upload request body is too large",
-                code=413,
-            )
+            raise SynapseError(msg="Upload request body is too large", code=413)
 
         upload_name = parse_string(request, b"filename", encoding=None)
         if upload_name:
             try:
-                upload_name = upload_name.decode('utf8')
+                upload_name = upload_name.decode("utf8")
             except UnicodeDecodeError:
                 raise SynapseError(
-                    msg="Invalid UTF-8 filename parameter: %r" % (upload_name),
-                    code=400,
+                    msg="Invalid UTF-8 filename parameter: %r" % (upload_name), code=400
                 )
 
         headers = request.requestHeaders
 
         if headers.hasHeader(b"Content-Type"):
-            media_type = headers.getRawHeaders(b"Content-Type")[0].decode('ascii')
+            media_type = headers.getRawHeaders(b"Content-Type")[0].decode("ascii")
         else:
-            raise SynapseError(
-                msg="Upload request missing 'Content-Type'",
-                code=400,
-            )
+            raise SynapseError(msg="Upload request missing 'Content-Type'", code=400)
 
         # if headers.hasHeader(b"Content-Disposition"):
         #     disposition = headers.getRawHeaders(b"Content-Disposition")[0]
         # TODO(markjh): parse content-dispostion
 
         content_uri = yield self.media_repo.create_content(
-            media_type, upload_name, request.content,
-            content_length, requester.user
+            media_type, upload_name, request.content, content_length, requester.user
         )
 
         logger.info("Uploaded content with URI %r", content_uri)
 
-        respond_with_json(
-            request, 200, {"content_uri": content_uri}, send_cors=True
-        )
+        respond_with_json(request, 200, {"content_uri": content_uri}, send_cors=True)