diff --git a/synapse/rest/media/v1/_base.py b/synapse/rest/media/v1/_base.py
index 2dcc8f74d6..3318638d3e 100644
--- a/synapse/rest/media/v1/_base.py
+++ b/synapse/rest/media/v1/_base.py
@@ -38,8 +38,8 @@ def parse_media_id(request):
server_name, media_id = request.postpath[:2]
if isinstance(server_name, bytes):
- server_name = server_name.decode('utf-8')
- media_id = media_id.decode('utf8')
+ server_name = server_name.decode("utf-8")
+ media_id = media_id.decode("utf8")
file_name = None
if len(request.postpath) > 2:
@@ -120,11 +120,11 @@ def add_file_headers(request, media_type, file_size, upload_name):
# correctly interpret those as of 0.99.2 and (b) they are a bit of a pain and we
# may as well just do the filename* version.
if _can_encode_filename_as_token(upload_name):
- disposition = 'inline; filename=%s' % (upload_name, )
+ disposition = "inline; filename=%s" % (upload_name,)
else:
- disposition = "inline; filename*=utf-8''%s" % (_quote(upload_name), )
+ disposition = "inline; filename*=utf-8''%s" % (_quote(upload_name),)
- request.setHeader(b"Content-Disposition", disposition.encode('ascii'))
+ request.setHeader(b"Content-Disposition", disposition.encode("ascii"))
# cache for at least a day.
# XXX: we might want to turn this off for data we don't want to
@@ -137,10 +137,27 @@ def add_file_headers(request, media_type, file_size, upload_name):
# separators as defined in RFC2616. SP and HT are handled separately.
# see _can_encode_filename_as_token.
-_FILENAME_SEPARATOR_CHARS = set((
- "(", ")", "<", ">", "@", ",", ";", ":", "\\", '"',
- "/", "[", "]", "?", "=", "{", "}",
-))
+_FILENAME_SEPARATOR_CHARS = set(
+ (
+ "(",
+ ")",
+ "<",
+ ">",
+ "@",
+ ",",
+ ";",
+ ":",
+ "\\",
+ '"',
+ "/",
+ "[",
+ "]",
+ "?",
+ "=",
+ "{",
+ "}",
+ )
+)
def _can_encode_filename_as_token(x):
@@ -271,7 +288,7 @@ def get_filename_from_headers(headers):
Returns:
A Unicode string of the filename, or None.
"""
- content_disposition = headers.get(b"Content-Disposition", [b''])
+ content_disposition = headers.get(b"Content-Disposition", [b""])
# No header, bail out.
if not content_disposition[0]:
@@ -293,7 +310,7 @@ def get_filename_from_headers(headers):
# Once it is decoded, we can then unquote the %-encoded
# parts strictly into a unicode string.
upload_name = urllib.parse.unquote(
- upload_name_utf8.decode('ascii'), errors="strict"
+ upload_name_utf8.decode("ascii"), errors="strict"
)
except UnicodeDecodeError:
# Incorrect UTF-8.
@@ -302,7 +319,7 @@ def get_filename_from_headers(headers):
# On Python 2, we first unquote the %-encoded parts and then
# decode it strictly using UTF-8.
try:
- upload_name = urllib.parse.unquote(upload_name_utf8).decode('utf8')
+ upload_name = urllib.parse.unquote(upload_name_utf8).decode("utf8")
except UnicodeDecodeError:
pass
@@ -310,7 +327,7 @@ def get_filename_from_headers(headers):
if not upload_name:
upload_name_ascii = params.get(b"filename", None)
if upload_name_ascii and is_ascii(upload_name_ascii):
- upload_name = upload_name_ascii.decode('ascii')
+ upload_name = upload_name_ascii.decode("ascii")
# This may be None here, indicating we did not find a matching name.
return upload_name
@@ -328,19 +345,19 @@ def _parse_header(line):
Tuple[bytes, dict[bytes, bytes]]:
the main content-type, followed by the parameter dictionary
"""
- parts = _parseparam(b';' + line)
+ parts = _parseparam(b";" + line)
key = next(parts)
pdict = {}
for p in parts:
- i = p.find(b'=')
+ i = p.find(b"=")
if i >= 0:
name = p[:i].strip().lower()
- value = p[i + 1:].strip()
+ value = p[i + 1 :].strip()
# strip double-quotes
if len(value) >= 2 and value[0:1] == value[-1:] == b'"':
value = value[1:-1]
- value = value.replace(b'\\\\', b'\\').replace(b'\\"', b'"')
+ value = value.replace(b"\\\\", b"\\").replace(b'\\"', b'"')
pdict[name] = value
return key, pdict
@@ -357,16 +374,16 @@ def _parseparam(s):
Returns:
Iterable[bytes]: the split input
"""
- while s[:1] == b';':
+ while s[:1] == b";":
s = s[1:]
# look for the next ;
- end = s.find(b';')
+ end = s.find(b";")
# if there is an odd number of " marks between here and the next ;, skip to the
# next ; instead
while end > 0 and (s.count(b'"', 0, end) - s.count(b'\\"', 0, end)) % 2:
- end = s.find(b';', end + 1)
+ end = s.find(b";", end + 1)
if end < 0:
end = len(s)
diff --git a/synapse/rest/media/v1/config_resource.py b/synapse/rest/media/v1/config_resource.py
index 77316033f7..fa3d6680fc 100644
--- a/synapse/rest/media/v1/config_resource.py
+++ b/synapse/rest/media/v1/config_resource.py
@@ -29,9 +29,7 @@ class MediaConfigResource(Resource):
config = hs.get_config()
self.clock = hs.get_clock()
self.auth = hs.get_auth()
- self.limits_dict = {
- "m.upload.size": config.max_upload_size,
- }
+ self.limits_dict = {"m.upload.size": config.max_upload_size}
def render_GET(self, request):
self._async_render_GET(request)
diff --git a/synapse/rest/media/v1/download_resource.py b/synapse/rest/media/v1/download_resource.py
index bdc5daecc1..a21a35f843 100644
--- a/synapse/rest/media/v1/download_resource.py
+++ b/synapse/rest/media/v1/download_resource.py
@@ -54,18 +54,20 @@ class DownloadResource(Resource):
b" plugin-types application/pdf;"
b" style-src 'unsafe-inline';"
b" media-src 'self';"
- b" object-src 'self';"
+ b" object-src 'self';",
)
server_name, media_id, name = parse_media_id(request)
if server_name == self.server_name:
yield self.media_repo.get_local_media(request, media_id, name)
else:
allow_remote = synapse.http.servlet.parse_boolean(
- request, "allow_remote", default=True)
+ request, "allow_remote", default=True
+ )
if not allow_remote:
logger.info(
"Rejecting request for remote media %s/%s due to allow_remote",
- server_name, media_id,
+ server_name,
+ media_id,
)
respond_404(request)
return
diff --git a/synapse/rest/media/v1/filepath.py b/synapse/rest/media/v1/filepath.py
index c8586fa280..e25c382c9c 100644
--- a/synapse/rest/media/v1/filepath.py
+++ b/synapse/rest/media/v1/filepath.py
@@ -24,6 +24,7 @@ def _wrap_in_base_path(func):
"""Takes a function that returns a relative path and turns it into an
absolute path based on the location of the primary media store
"""
+
@functools.wraps(func)
def _wrapped(self, *args, **kwargs):
path = func(self, *args, **kwargs)
@@ -43,125 +44,102 @@ class MediaFilePaths(object):
def __init__(self, primary_base_path):
self.base_path = primary_base_path
- def default_thumbnail_rel(self, default_top_level, default_sub_type, width,
- height, content_type, method):
+ def default_thumbnail_rel(
+ self, default_top_level, default_sub_type, width, height, content_type, method
+ ):
top_level_type, sub_type = content_type.split("/")
- file_name = "%i-%i-%s-%s-%s" % (
- width, height, top_level_type, sub_type, method
- )
+ file_name = "%i-%i-%s-%s-%s" % (width, height, top_level_type, sub_type, method)
return os.path.join(
- "default_thumbnails", default_top_level,
- default_sub_type, file_name
+ "default_thumbnails", default_top_level, default_sub_type, file_name
)
default_thumbnail = _wrap_in_base_path(default_thumbnail_rel)
def local_media_filepath_rel(self, media_id):
- return os.path.join(
- "local_content",
- media_id[0:2], media_id[2:4], media_id[4:]
- )
+ return os.path.join("local_content", media_id[0:2], media_id[2:4], media_id[4:])
local_media_filepath = _wrap_in_base_path(local_media_filepath_rel)
- def local_media_thumbnail_rel(self, media_id, width, height, content_type,
- method):
+ def local_media_thumbnail_rel(self, media_id, width, height, content_type, method):
top_level_type, sub_type = content_type.split("/")
- file_name = "%i-%i-%s-%s-%s" % (
- width, height, top_level_type, sub_type, method
- )
+ file_name = "%i-%i-%s-%s-%s" % (width, height, top_level_type, sub_type, method)
return os.path.join(
- "local_thumbnails",
- media_id[0:2], media_id[2:4], media_id[4:],
- file_name
+ "local_thumbnails", media_id[0:2], media_id[2:4], media_id[4:], file_name
)
local_media_thumbnail = _wrap_in_base_path(local_media_thumbnail_rel)
def remote_media_filepath_rel(self, server_name, file_id):
return os.path.join(
- "remote_content", server_name,
- file_id[0:2], file_id[2:4], file_id[4:]
+ "remote_content", server_name, file_id[0:2], file_id[2:4], file_id[4:]
)
remote_media_filepath = _wrap_in_base_path(remote_media_filepath_rel)
- def remote_media_thumbnail_rel(self, server_name, file_id, width, height,
- content_type, method):
+ def remote_media_thumbnail_rel(
+ self, server_name, file_id, width, height, content_type, method
+ ):
top_level_type, sub_type = content_type.split("/")
file_name = "%i-%i-%s-%s" % (width, height, top_level_type, sub_type)
return os.path.join(
- "remote_thumbnail", server_name,
- file_id[0:2], file_id[2:4], file_id[4:],
- file_name
+ "remote_thumbnail",
+ server_name,
+ file_id[0:2],
+ file_id[2:4],
+ file_id[4:],
+ file_name,
)
remote_media_thumbnail = _wrap_in_base_path(remote_media_thumbnail_rel)
def remote_media_thumbnail_dir(self, server_name, file_id):
return os.path.join(
- self.base_path, "remote_thumbnail", server_name,
- file_id[0:2], file_id[2:4], file_id[4:],
+ self.base_path,
+ "remote_thumbnail",
+ server_name,
+ file_id[0:2],
+ file_id[2:4],
+ file_id[4:],
)
def url_cache_filepath_rel(self, media_id):
if NEW_FORMAT_ID_RE.match(media_id):
# Media id is of the form <DATE><RANDOM_STRING>
# E.g.: 2017-09-28-fsdRDt24DS234dsf
- return os.path.join(
- "url_cache",
- media_id[:10], media_id[11:]
- )
+ return os.path.join("url_cache", media_id[:10], media_id[11:])
else:
- return os.path.join(
- "url_cache",
- media_id[0:2], media_id[2:4], media_id[4:],
- )
+ return os.path.join("url_cache", media_id[0:2], media_id[2:4], media_id[4:])
url_cache_filepath = _wrap_in_base_path(url_cache_filepath_rel)
def url_cache_filepath_dirs_to_delete(self, media_id):
"The dirs to try and remove if we delete the media_id file"
if NEW_FORMAT_ID_RE.match(media_id):
- return [
- os.path.join(
- self.base_path, "url_cache",
- media_id[:10],
- ),
- ]
+ return [os.path.join(self.base_path, "url_cache", media_id[:10])]
else:
return [
- os.path.join(
- self.base_path, "url_cache",
- media_id[0:2], media_id[2:4],
- ),
- os.path.join(
- self.base_path, "url_cache",
- media_id[0:2],
- ),
+ os.path.join(self.base_path, "url_cache", media_id[0:2], media_id[2:4]),
+ os.path.join(self.base_path, "url_cache", media_id[0:2]),
]
- def url_cache_thumbnail_rel(self, media_id, width, height, content_type,
- method):
+ def url_cache_thumbnail_rel(self, media_id, width, height, content_type, method):
# Media id is of the form <DATE><RANDOM_STRING>
# E.g.: 2017-09-28-fsdRDt24DS234dsf
top_level_type, sub_type = content_type.split("/")
- file_name = "%i-%i-%s-%s-%s" % (
- width, height, top_level_type, sub_type, method
- )
+ file_name = "%i-%i-%s-%s-%s" % (width, height, top_level_type, sub_type, method)
if NEW_FORMAT_ID_RE.match(media_id):
return os.path.join(
- "url_cache_thumbnails",
- media_id[:10], media_id[11:],
- file_name
+ "url_cache_thumbnails", media_id[:10], media_id[11:], file_name
)
else:
return os.path.join(
"url_cache_thumbnails",
- media_id[0:2], media_id[2:4], media_id[4:],
- file_name
+ media_id[0:2],
+ media_id[2:4],
+ media_id[4:],
+ file_name,
)
url_cache_thumbnail = _wrap_in_base_path(url_cache_thumbnail_rel)
@@ -172,13 +150,15 @@ class MediaFilePaths(object):
if NEW_FORMAT_ID_RE.match(media_id):
return os.path.join(
- self.base_path, "url_cache_thumbnails",
- media_id[:10], media_id[11:],
+ self.base_path, "url_cache_thumbnails", media_id[:10], media_id[11:]
)
else:
return os.path.join(
- self.base_path, "url_cache_thumbnails",
- media_id[0:2], media_id[2:4], media_id[4:],
+ self.base_path,
+ "url_cache_thumbnails",
+ media_id[0:2],
+ media_id[2:4],
+ media_id[4:],
)
def url_cache_thumbnail_dirs_to_delete(self, media_id):
@@ -188,26 +168,21 @@ class MediaFilePaths(object):
if NEW_FORMAT_ID_RE.match(media_id):
return [
os.path.join(
- self.base_path, "url_cache_thumbnails",
- media_id[:10], media_id[11:],
- ),
- os.path.join(
- self.base_path, "url_cache_thumbnails",
- media_id[:10],
+ self.base_path, "url_cache_thumbnails", media_id[:10], media_id[11:]
),
+ os.path.join(self.base_path, "url_cache_thumbnails", media_id[:10]),
]
else:
return [
os.path.join(
- self.base_path, "url_cache_thumbnails",
- media_id[0:2], media_id[2:4], media_id[4:],
- ),
- os.path.join(
- self.base_path, "url_cache_thumbnails",
- media_id[0:2], media_id[2:4],
+ self.base_path,
+ "url_cache_thumbnails",
+ media_id[0:2],
+ media_id[2:4],
+ media_id[4:],
),
os.path.join(
- self.base_path, "url_cache_thumbnails",
- media_id[0:2],
+ self.base_path, "url_cache_thumbnails", media_id[0:2], media_id[2:4]
),
+ os.path.join(self.base_path, "url_cache_thumbnails", media_id[0:2]),
]
diff --git a/synapse/rest/media/v1/media_repository.py b/synapse/rest/media/v1/media_repository.py
index a4929dd5db..df3d985a38 100644
--- a/synapse/rest/media/v1/media_repository.py
+++ b/synapse/rest/media/v1/media_repository.py
@@ -100,17 +100,16 @@ class MediaRepository(object):
storage_providers.append(provider)
self.media_storage = MediaStorage(
- self.hs, self.primary_base_path, self.filepaths, storage_providers,
+ self.hs, self.primary_base_path, self.filepaths, storage_providers
)
self.clock.looping_call(
- self._start_update_recently_accessed,
- UPDATE_RECENTLY_ACCESSED_TS,
+ self._start_update_recently_accessed, UPDATE_RECENTLY_ACCESSED_TS
)
def _start_update_recently_accessed(self):
return run_as_background_process(
- "update_recently_accessed_media", self._update_recently_accessed,
+ "update_recently_accessed_media", self._update_recently_accessed
)
@defer.inlineCallbacks
@@ -138,8 +137,9 @@ class MediaRepository(object):
self.recently_accessed_locals.add(media_id)
@defer.inlineCallbacks
- def create_content(self, media_type, upload_name, content, content_length,
- auth_user):
+ def create_content(
+ self, media_type, upload_name, content, content_length, auth_user
+ ):
"""Store uploaded content for a local user and return the mxc URL
Args:
@@ -154,10 +154,7 @@ class MediaRepository(object):
"""
media_id = random_string(24)
- file_info = FileInfo(
- server_name=None,
- file_id=media_id,
- )
+ file_info = FileInfo(server_name=None, file_id=media_id)
fname = yield self.media_storage.store_file(content, file_info)
@@ -172,9 +169,7 @@ class MediaRepository(object):
user_id=auth_user,
)
- yield self._generate_thumbnails(
- None, media_id, media_id, media_type,
- )
+ yield self._generate_thumbnails(None, media_id, media_id, media_type)
defer.returnValue("mxc://%s/%s" % (self.server_name, media_id))
@@ -205,14 +200,11 @@ class MediaRepository(object):
upload_name = name if name else media_info["upload_name"]
url_cache = media_info["url_cache"]
- file_info = FileInfo(
- None, media_id,
- url_cache=url_cache,
- )
+ file_info = FileInfo(None, media_id, url_cache=url_cache)
responder = yield self.media_storage.fetch_media(file_info)
yield respond_with_responder(
- request, responder, media_type, media_length, upload_name,
+ request, responder, media_type, media_length, upload_name
)
@defer.inlineCallbacks
@@ -232,8 +224,8 @@ class MediaRepository(object):
to request
"""
if (
- self.federation_domain_whitelist is not None and
- server_name not in self.federation_domain_whitelist
+ self.federation_domain_whitelist is not None
+ and server_name not in self.federation_domain_whitelist
):
raise FederationDeniedError(server_name)
@@ -244,7 +236,7 @@ class MediaRepository(object):
key = (server_name, media_id)
with (yield self.remote_media_linearizer.queue(key)):
responder, media_info = yield self._get_remote_media_impl(
- server_name, media_id,
+ server_name, media_id
)
# We deliberately stream the file outside the lock
@@ -253,7 +245,7 @@ class MediaRepository(object):
media_length = media_info["media_length"]
upload_name = name if name else media_info["upload_name"]
yield respond_with_responder(
- request, responder, media_type, media_length, upload_name,
+ request, responder, media_type, media_length, upload_name
)
else:
respond_404(request)
@@ -272,8 +264,8 @@ class MediaRepository(object):
Deferred[dict]: The media_info of the file
"""
if (
- self.federation_domain_whitelist is not None and
- server_name not in self.federation_domain_whitelist
+ self.federation_domain_whitelist is not None
+ and server_name not in self.federation_domain_whitelist
):
raise FederationDeniedError(server_name)
@@ -282,7 +274,7 @@ class MediaRepository(object):
key = (server_name, media_id)
with (yield self.remote_media_linearizer.queue(key)):
responder, media_info = yield self._get_remote_media_impl(
- server_name, media_id,
+ server_name, media_id
)
# Ensure we actually use the responder so that it releases resources
@@ -305,9 +297,7 @@ class MediaRepository(object):
Returns:
Deferred[(Responder, media_info)]
"""
- media_info = yield self.store.get_cached_remote_media(
- server_name, media_id
- )
+ media_info = yield self.store.get_cached_remote_media(server_name, media_id)
# file_id is the ID we use to track the file locally. If we've already
# seen the file then reuse the existing ID, otherwise genereate a new
@@ -331,9 +321,7 @@ class MediaRepository(object):
# Failed to find the file anywhere, lets download it.
- media_info = yield self._download_remote_file(
- server_name, media_id, file_id
- )
+ media_info = yield self._download_remote_file(server_name, media_id, file_id)
responder = yield self.media_storage.fetch_media(file_info)
defer.returnValue((responder, media_info))
@@ -354,54 +342,60 @@ class MediaRepository(object):
Deferred[MediaInfo]
"""
- file_info = FileInfo(
- server_name=server_name,
- file_id=file_id,
- )
+ file_info = FileInfo(server_name=server_name, file_id=file_id)
with self.media_storage.store_into_file(file_info) as (f, fname, finish):
- request_path = "/".join((
- "/_matrix/media/v1/download", server_name, media_id,
- ))
+ request_path = "/".join(
+ ("/_matrix/media/v1/download", server_name, media_id)
+ )
try:
length, headers = yield self.client.get_file(
- server_name, request_path, output_stream=f,
- max_size=self.max_upload_size, args={
+ server_name,
+ request_path,
+ output_stream=f,
+ max_size=self.max_upload_size,
+ args={
# tell the remote server to 404 if it doesn't
# recognise the server_name, to make sure we don't
# end up with a routing loop.
- "allow_remote": "false",
- }
+ "allow_remote": "false"
+ },
)
except RequestSendFailed as e:
- logger.warn("Request failed fetching remote media %s/%s: %r",
- server_name, media_id, e)
+ logger.warn(
+ "Request failed fetching remote media %s/%s: %r",
+ server_name,
+ media_id,
+ e,
+ )
raise SynapseError(502, "Failed to fetch remote media")
except HttpResponseException as e:
- logger.warn("HTTP error fetching remote media %s/%s: %s",
- server_name, media_id, e.response)
+ logger.warn(
+ "HTTP error fetching remote media %s/%s: %s",
+ server_name,
+ media_id,
+ e.response,
+ )
if e.code == twisted.web.http.NOT_FOUND:
raise e.to_synapse_error()
raise SynapseError(502, "Failed to fetch remote media")
except SynapseError:
- logger.warn(
- "Failed to fetch remote media %s/%s",
- server_name, media_id,
- )
+ logger.warn("Failed to fetch remote media %s/%s", server_name, media_id)
raise
except NotRetryingDestination:
logger.warn("Not retrying destination %r", server_name)
raise SynapseError(502, "Failed to fetch remote media")
except Exception:
- logger.exception("Failed to fetch remote media %s/%s",
- server_name, media_id)
+ logger.exception(
+ "Failed to fetch remote media %s/%s", server_name, media_id
+ )
raise SynapseError(502, "Failed to fetch remote media")
yield finish()
- media_type = headers[b"Content-Type"][0].decode('ascii')
+ media_type = headers[b"Content-Type"][0].decode("ascii")
upload_name = get_filename_from_headers(headers)
time_now_ms = self.clock.time_msec()
@@ -425,24 +419,23 @@ class MediaRepository(object):
"filesystem_id": file_id,
}
- yield self._generate_thumbnails(
- server_name, media_id, file_id, media_type,
- )
+ yield self._generate_thumbnails(server_name, media_id, file_id, media_type)
defer.returnValue(media_info)
def _get_thumbnail_requirements(self, media_type):
return self.thumbnail_requirements.get(media_type, ())
- def _generate_thumbnail(self, thumbnailer, t_width, t_height,
- t_method, t_type):
+ def _generate_thumbnail(self, thumbnailer, t_width, t_height, t_method, t_type):
m_width = thumbnailer.width
m_height = thumbnailer.height
if m_width * m_height >= self.max_image_pixels:
logger.info(
"Image too large to thumbnail %r x %r > %r",
- m_width, m_height, self.max_image_pixels
+ m_width,
+ m_height,
+ self.max_image_pixels,
)
return
@@ -462,17 +455,22 @@ class MediaRepository(object):
return t_byte_source
@defer.inlineCallbacks
- def generate_local_exact_thumbnail(self, media_id, t_width, t_height,
- t_method, t_type, url_cache):
- input_path = yield self.media_storage.ensure_media_is_in_local_cache(FileInfo(
- None, media_id, url_cache=url_cache,
- ))
+ def generate_local_exact_thumbnail(
+ self, media_id, t_width, t_height, t_method, t_type, url_cache
+ ):
+ input_path = yield self.media_storage.ensure_media_is_in_local_cache(
+ FileInfo(None, media_id, url_cache=url_cache)
+ )
thumbnailer = Thumbnailer(input_path)
t_byte_source = yield logcontext.defer_to_thread(
self.hs.get_reactor(),
self._generate_thumbnail,
- thumbnailer, t_width, t_height, t_method, t_type
+ thumbnailer,
+ t_width,
+ t_height,
+ t_method,
+ t_type,
)
if t_byte_source:
@@ -489,7 +487,7 @@ class MediaRepository(object):
)
output_path = yield self.media_storage.store_file(
- t_byte_source, file_info,
+ t_byte_source, file_info
)
finally:
t_byte_source.close()
@@ -505,17 +503,22 @@ class MediaRepository(object):
defer.returnValue(output_path)
@defer.inlineCallbacks
- def generate_remote_exact_thumbnail(self, server_name, file_id, media_id,
- t_width, t_height, t_method, t_type):
- input_path = yield self.media_storage.ensure_media_is_in_local_cache(FileInfo(
- server_name, file_id, url_cache=False,
- ))
+ def generate_remote_exact_thumbnail(
+ self, server_name, file_id, media_id, t_width, t_height, t_method, t_type
+ ):
+ input_path = yield self.media_storage.ensure_media_is_in_local_cache(
+ FileInfo(server_name, file_id, url_cache=False)
+ )
thumbnailer = Thumbnailer(input_path)
t_byte_source = yield logcontext.defer_to_thread(
self.hs.get_reactor(),
self._generate_thumbnail,
- thumbnailer, t_width, t_height, t_method, t_type
+ thumbnailer,
+ t_width,
+ t_height,
+ t_method,
+ t_type,
)
if t_byte_source:
@@ -531,7 +534,7 @@ class MediaRepository(object):
)
output_path = yield self.media_storage.store_file(
- t_byte_source, file_info,
+ t_byte_source, file_info
)
finally:
t_byte_source.close()
@@ -541,15 +544,22 @@ class MediaRepository(object):
t_len = os.path.getsize(output_path)
yield self.store.store_remote_media_thumbnail(
- server_name, media_id, file_id,
- t_width, t_height, t_type, t_method, t_len
+ server_name,
+ media_id,
+ file_id,
+ t_width,
+ t_height,
+ t_type,
+ t_method,
+ t_len,
)
defer.returnValue(output_path)
@defer.inlineCallbacks
- def _generate_thumbnails(self, server_name, media_id, file_id, media_type,
- url_cache=False):
+ def _generate_thumbnails(
+ self, server_name, media_id, file_id, media_type, url_cache=False
+ ):
"""Generate and store thumbnails for an image.
Args:
@@ -568,9 +578,9 @@ class MediaRepository(object):
if not requirements:
return
- input_path = yield self.media_storage.ensure_media_is_in_local_cache(FileInfo(
- server_name, file_id, url_cache=url_cache,
- ))
+ input_path = yield self.media_storage.ensure_media_is_in_local_cache(
+ FileInfo(server_name, file_id, url_cache=url_cache)
+ )
thumbnailer = Thumbnailer(input_path)
m_width = thumbnailer.width
@@ -579,14 +589,15 @@ class MediaRepository(object):
if m_width * m_height >= self.max_image_pixels:
logger.info(
"Image too large to thumbnail %r x %r > %r",
- m_width, m_height, self.max_image_pixels
+ m_width,
+ m_height,
+ self.max_image_pixels,
)
return
if thumbnailer.transpose_method is not None:
m_width, m_height = yield logcontext.defer_to_thread(
- self.hs.get_reactor(),
- thumbnailer.transpose
+ self.hs.get_reactor(), thumbnailer.transpose
)
# We deduplicate the thumbnail sizes by ignoring the cropped versions if
@@ -606,15 +617,11 @@ class MediaRepository(object):
# Generate the thumbnail
if t_method == "crop":
t_byte_source = yield logcontext.defer_to_thread(
- self.hs.get_reactor(),
- thumbnailer.crop,
- t_width, t_height, t_type,
+ self.hs.get_reactor(), thumbnailer.crop, t_width, t_height, t_type
)
elif t_method == "scale":
t_byte_source = yield logcontext.defer_to_thread(
- self.hs.get_reactor(),
- thumbnailer.scale,
- t_width, t_height, t_type,
+ self.hs.get_reactor(), thumbnailer.scale, t_width, t_height, t_type
)
else:
logger.error("Unrecognized method: %r", t_method)
@@ -636,7 +643,7 @@ class MediaRepository(object):
)
output_path = yield self.media_storage.store_file(
- t_byte_source, file_info,
+ t_byte_source, file_info
)
finally:
t_byte_source.close()
@@ -646,18 +653,21 @@ class MediaRepository(object):
# Write to database
if server_name:
yield self.store.store_remote_media_thumbnail(
- server_name, media_id, file_id,
- t_width, t_height, t_type, t_method, t_len
+ server_name,
+ media_id,
+ file_id,
+ t_width,
+ t_height,
+ t_type,
+ t_method,
+ t_len,
)
else:
yield self.store.store_local_thumbnail(
media_id, t_width, t_height, t_type, t_method, t_len
)
- defer.returnValue({
- "width": m_width,
- "height": m_height,
- })
+ defer.returnValue({"width": m_width, "height": m_height})
@defer.inlineCallbacks
def delete_old_remote_media(self, before_ts):
@@ -749,11 +759,12 @@ class MediaRepositoryResource(Resource):
self.putChild(b"upload", UploadResource(hs, media_repo))
self.putChild(b"download", DownloadResource(hs, media_repo))
- self.putChild(b"thumbnail", ThumbnailResource(
- hs, media_repo, media_repo.media_storage,
- ))
+ self.putChild(
+ b"thumbnail", ThumbnailResource(hs, media_repo, media_repo.media_storage)
+ )
if hs.config.url_preview_enabled:
- self.putChild(b"preview_url", PreviewUrlResource(
- hs, media_repo, media_repo.media_storage,
- ))
+ self.putChild(
+ b"preview_url",
+ PreviewUrlResource(hs, media_repo, media_repo.media_storage),
+ )
self.putChild(b"config", MediaConfigResource(hs))
diff --git a/synapse/rest/media/v1/media_storage.py b/synapse/rest/media/v1/media_storage.py
index 896078fe76..eff86836fb 100644
--- a/synapse/rest/media/v1/media_storage.py
+++ b/synapse/rest/media/v1/media_storage.py
@@ -66,8 +66,7 @@ class MediaStorage(object):
with self.store_into_file(file_info) as (f, fname, finish_cb):
# Write to the main repository
yield logcontext.defer_to_thread(
- self.hs.get_reactor(),
- _write_file_synchronously, source, f,
+ self.hs.get_reactor(), _write_file_synchronously, source, f
)
yield finish_cb()
@@ -179,7 +178,8 @@ class MediaStorage(object):
if res:
with res:
consumer = BackgroundFileConsumer(
- open(local_path, "wb"), self.hs.get_reactor())
+ open(local_path, "wb"), self.hs.get_reactor()
+ )
yield res.write_to_consumer(consumer)
yield consumer.wait()
defer.returnValue(local_path)
@@ -217,10 +217,10 @@ class MediaStorage(object):
width=file_info.thumbnail_width,
height=file_info.thumbnail_height,
content_type=file_info.thumbnail_type,
- method=file_info.thumbnail_method
+ method=file_info.thumbnail_method,
)
return self.filepaths.remote_media_filepath_rel(
- file_info.server_name, file_info.file_id,
+ file_info.server_name, file_info.file_id
)
if file_info.thumbnail:
@@ -229,11 +229,9 @@ class MediaStorage(object):
width=file_info.thumbnail_width,
height=file_info.thumbnail_height,
content_type=file_info.thumbnail_type,
- method=file_info.thumbnail_method
+ method=file_info.thumbnail_method,
)
- return self.filepaths.local_media_filepath_rel(
- file_info.file_id,
- )
+ return self.filepaths.local_media_filepath_rel(file_info.file_id)
def _write_file_synchronously(source, dest):
@@ -255,6 +253,7 @@ class FileResponder(Responder):
open_file (file): A file like object to be streamed ot the client,
is closed when finished streaming.
"""
+
def __init__(self, open_file):
self.open_file = open_file
diff --git a/synapse/rest/media/v1/preview_url_resource.py b/synapse/rest/media/v1/preview_url_resource.py
index acf87709f2..de6f292ffb 100644
--- a/synapse/rest/media/v1/preview_url_resource.py
+++ b/synapse/rest/media/v1/preview_url_resource.py
@@ -92,7 +92,7 @@ class PreviewUrlResource(Resource):
)
self._cleaner_loop = self.clock.looping_call(
- self._start_expire_url_cache_data, 10 * 1000,
+ self._start_expire_url_cache_data, 10 * 1000
)
def render_OPTIONS(self, request):
@@ -121,16 +121,16 @@ class PreviewUrlResource(Resource):
for attrib in entry:
pattern = entry[attrib]
value = getattr(url_tuple, attrib)
- logger.debug((
- "Matching attrib '%s' with value '%s' against"
- " pattern '%s'"
- ) % (attrib, value, pattern))
+ logger.debug(
+ ("Matching attrib '%s' with value '%s' against" " pattern '%s'")
+ % (attrib, value, pattern)
+ )
if value is None:
match = False
continue
- if pattern.startswith('^'):
+ if pattern.startswith("^"):
if not re.match(pattern, getattr(url_tuple, attrib)):
match = False
continue
@@ -139,12 +139,9 @@ class PreviewUrlResource(Resource):
match = False
continue
if match:
- logger.warn(
- "URL %s blocked by url_blacklist entry %s", url, entry
- )
+ logger.warn("URL %s blocked by url_blacklist entry %s", url, entry)
raise SynapseError(
- 403, "URL blocked by url pattern blacklist entry",
- Codes.UNKNOWN
+ 403, "URL blocked by url pattern blacklist entry", Codes.UNKNOWN
)
# the in-memory cache:
@@ -156,14 +153,8 @@ class PreviewUrlResource(Resource):
observable = self._cache.get(url)
if not observable:
- download = run_in_background(
- self._do_preview,
- url, requester.user, ts,
- )
- observable = ObservableDeferred(
- download,
- consumeErrors=True
- )
+ download = run_in_background(self._do_preview, url, requester.user, ts)
+ observable = ObservableDeferred(download, consumeErrors=True)
self._cache[url] = observable
else:
logger.info("Returning cached response")
@@ -187,15 +178,15 @@ class PreviewUrlResource(Resource):
# historical previews, if we have any)
cache_result = yield self.store.get_url_cache(url, ts)
if (
- cache_result and
- cache_result["expires_ts"] > ts and
- cache_result["response_code"] / 100 == 2
+ cache_result
+ and cache_result["expires_ts"] > ts
+ and cache_result["response_code"] / 100 == 2
):
# It may be stored as text in the database, not as bytes (such as
# PostgreSQL). If so, encode it back before handing it on.
og = cache_result["og"]
if isinstance(og, six.text_type):
- og = og.encode('utf8')
+ og = og.encode("utf8")
defer.returnValue(og)
return
@@ -203,33 +194,31 @@ class PreviewUrlResource(Resource):
logger.debug("got media_info of '%s'" % media_info)
- if _is_media(media_info['media_type']):
- file_id = media_info['filesystem_id']
+ if _is_media(media_info["media_type"]):
+ file_id = media_info["filesystem_id"]
dims = yield self.media_repo._generate_thumbnails(
- None, file_id, file_id, media_info["media_type"],
- url_cache=True,
+ None, file_id, file_id, media_info["media_type"], url_cache=True
)
og = {
- "og:description": media_info['download_name'],
- "og:image": "mxc://%s/%s" % (
- self.server_name, media_info['filesystem_id']
- ),
- "og:image:type": media_info['media_type'],
- "matrix:image:size": media_info['media_length'],
+ "og:description": media_info["download_name"],
+ "og:image": "mxc://%s/%s"
+ % (self.server_name, media_info["filesystem_id"]),
+ "og:image:type": media_info["media_type"],
+ "matrix:image:size": media_info["media_length"],
}
if dims:
- og["og:image:width"] = dims['width']
- og["og:image:height"] = dims['height']
+ og["og:image:width"] = dims["width"]
+ og["og:image:height"] = dims["height"]
else:
logger.warn("Couldn't get dims for %s" % url)
# define our OG response for this media
- elif _is_html(media_info['media_type']):
+ elif _is_html(media_info["media_type"]):
# TODO: somehow stop a big HTML tree from exploding synapse's RAM
- with open(media_info['filename'], 'rb') as file:
+ with open(media_info["filename"], "rb") as file:
body = file.read()
encoding = None
@@ -242,45 +231,43 @@ class PreviewUrlResource(Resource):
# If we find a match, it should take precedence over the
# Content-Type header, so set it here.
if match:
- encoding = match.group(1).decode('ascii')
+ encoding = match.group(1).decode("ascii")
# If we don't find a match, we'll look at the HTTP Content-Type, and
# if that doesn't exist, we'll fall back to UTF-8.
if not encoding:
- match = _content_type_match.match(
- media_info['media_type']
- )
+ match = _content_type_match.match(media_info["media_type"])
encoding = match.group(1) if match else "utf-8"
- og = decode_and_calc_og(body, media_info['uri'], encoding)
+ og = decode_and_calc_og(body, media_info["uri"], encoding)
# pre-cache the image for posterity
# FIXME: it might be cleaner to use the same flow as the main /preview_url
# request itself and benefit from the same caching etc. But for now we
# just rely on the caching on the master request to speed things up.
- if 'og:image' in og and og['og:image']:
+ if "og:image" in og and og["og:image"]:
image_info = yield self._download_url(
- _rebase_url(og['og:image'], media_info['uri']), user
+ _rebase_url(og["og:image"], media_info["uri"]), user
)
- if _is_media(image_info['media_type']):
+ if _is_media(image_info["media_type"]):
# TODO: make sure we don't choke on white-on-transparent images
- file_id = image_info['filesystem_id']
+ file_id = image_info["filesystem_id"]
dims = yield self.media_repo._generate_thumbnails(
- None, file_id, file_id, image_info["media_type"],
- url_cache=True,
+ None, file_id, file_id, image_info["media_type"], url_cache=True
)
if dims:
- og["og:image:width"] = dims['width']
- og["og:image:height"] = dims['height']
+ og["og:image:width"] = dims["width"]
+ og["og:image:height"] = dims["height"]
else:
logger.warn("Couldn't get dims for %s" % og["og:image"])
og["og:image"] = "mxc://%s/%s" % (
- self.server_name, image_info['filesystem_id']
+ self.server_name,
+ image_info["filesystem_id"],
)
- og["og:image:type"] = image_info['media_type']
- og["matrix:image:size"] = image_info['media_length']
+ og["og:image:type"] = image_info["media_type"]
+ og["matrix:image:size"] = image_info["media_length"]
else:
del og["og:image"]
else:
@@ -289,7 +276,7 @@ class PreviewUrlResource(Resource):
logger.debug("Calculated OG for %s as %s" % (url, og))
- jsonog = json.dumps(og).encode('utf8')
+ jsonog = json.dumps(og).encode("utf8")
# store OG in history-aware DB cache
yield self.store.store_url_cache(
@@ -310,19 +297,15 @@ class PreviewUrlResource(Resource):
# we're most likely being explicitly triggered by a human rather than a
# bot, so are we really a robot?
- file_id = datetime.date.today().isoformat() + '_' + random_string(16)
+ file_id = datetime.date.today().isoformat() + "_" + random_string(16)
- file_info = FileInfo(
- server_name=None,
- file_id=file_id,
- url_cache=True,
- )
+ file_info = FileInfo(server_name=None, file_id=file_id, url_cache=True)
with self.media_storage.store_into_file(file_info) as (f, fname, finish):
try:
logger.debug("Trying to get url '%s'" % url)
length, headers, uri, code = yield self.client.get_file(
- url, output_stream=f, max_size=self.max_spider_size,
+ url, output_stream=f, max_size=self.max_spider_size
)
except SynapseError:
# Pass SynapseErrors through directly, so that the servlet
@@ -334,24 +317,25 @@ class PreviewUrlResource(Resource):
# Note: This will also be the case if one of the resolved IP
# addresses is blacklisted
raise SynapseError(
- 502, "DNS resolution failure during URL preview generation",
- Codes.UNKNOWN
+ 502,
+ "DNS resolution failure during URL preview generation",
+ Codes.UNKNOWN,
)
except Exception as e:
# FIXME: pass through 404s and other error messages nicely
logger.warn("Error downloading %s: %r", url, e)
raise SynapseError(
- 500, "Failed to download content: %s" % (
- traceback.format_exception_only(sys.exc_info()[0], e),
- ),
+ 500,
+ "Failed to download content: %s"
+ % (traceback.format_exception_only(sys.exc_info()[0], e),),
Codes.UNKNOWN,
)
yield finish()
try:
if b"Content-Type" in headers:
- media_type = headers[b"Content-Type"][0].decode('ascii')
+ media_type = headers[b"Content-Type"][0].decode("ascii")
else:
media_type = "application/octet-stream"
time_now_ms = self.clock.time_msec()
@@ -375,24 +359,26 @@ class PreviewUrlResource(Resource):
# therefore not expire it.
raise
- defer.returnValue({
- "media_type": media_type,
- "media_length": length,
- "download_name": download_name,
- "created_ts": time_now_ms,
- "filesystem_id": file_id,
- "filename": fname,
- "uri": uri,
- "response_code": code,
- # FIXME: we should calculate a proper expiration based on the
- # Cache-Control and Expire headers. But for now, assume 1 hour.
- "expires": 60 * 60 * 1000,
- "etag": headers["ETag"][0] if "ETag" in headers else None,
- })
+ defer.returnValue(
+ {
+ "media_type": media_type,
+ "media_length": length,
+ "download_name": download_name,
+ "created_ts": time_now_ms,
+ "filesystem_id": file_id,
+ "filename": fname,
+ "uri": uri,
+ "response_code": code,
+ # FIXME: we should calculate a proper expiration based on the
+ # Cache-Control and Expire headers. But for now, assume 1 hour.
+ "expires": 60 * 60 * 1000,
+ "etag": headers["ETag"][0] if "ETag" in headers else None,
+ }
+ )
def _start_expire_url_cache_data(self):
return run_as_background_process(
- "expire_url_cache_data", self._expire_url_cache_data,
+ "expire_url_cache_data", self._expire_url_cache_data
)
@defer.inlineCallbacks
@@ -496,7 +482,7 @@ def decode_and_calc_og(body, media_uri, request_encoding=None):
# blindly try decoding the body as utf-8, which seems to fix
# the charset mismatches on https://google.com
parser = etree.HTMLParser(recover=True, encoding=request_encoding)
- tree = etree.fromstring(body.decode('utf-8', 'ignore'), parser)
+ tree = etree.fromstring(body.decode("utf-8", "ignore"), parser)
og = _calc_og(tree, media_uri)
return og
@@ -523,8 +509,8 @@ def _calc_og(tree, media_uri):
og = {}
for tag in tree.xpath("//*/meta[starts-with(@property, 'og:')]"):
- if 'content' in tag.attrib:
- og[tag.attrib['property']] = tag.attrib['content']
+ if "content" in tag.attrib:
+ og[tag.attrib["property"]] = tag.attrib["content"]
# TODO: grab article: meta tags too, e.g.:
@@ -535,39 +521,43 @@ def _calc_og(tree, media_uri):
# "article:published_time" content="2016-03-31T19:58:24+00:00" />
# "article:modified_time" content="2016-04-01T18:31:53+00:00" />
- if 'og:title' not in og:
+ if "og:title" not in og:
# do some basic spidering of the HTML
title = tree.xpath("(//title)[1] | (//h1)[1] | (//h2)[1] | (//h3)[1]")
if title and title[0].text is not None:
- og['og:title'] = title[0].text.strip()
+ og["og:title"] = title[0].text.strip()
else:
- og['og:title'] = None
+ og["og:title"] = None
- if 'og:image' not in og:
+ if "og:image" not in og:
# TODO: extract a favicon failing all else
meta_image = tree.xpath(
"//*/meta[translate(@itemprop, 'IMAGE', 'image')='image']/@content"
)
if meta_image:
- og['og:image'] = _rebase_url(meta_image[0], media_uri)
+ og["og:image"] = _rebase_url(meta_image[0], media_uri)
else:
# TODO: consider inlined CSS styles as well as width & height attribs
images = tree.xpath("//img[@src][number(@width)>10][number(@height)>10]")
- images = sorted(images, key=lambda i: (
- -1 * float(i.attrib['width']) * float(i.attrib['height'])
- ))
+ images = sorted(
+ images,
+ key=lambda i: (
+ -1 * float(i.attrib["width"]) * float(i.attrib["height"])
+ ),
+ )
if not images:
images = tree.xpath("//img[@src]")
if images:
- og['og:image'] = images[0].attrib['src']
+ og["og:image"] = images[0].attrib["src"]
- if 'og:description' not in og:
+ if "og:description" not in og:
meta_description = tree.xpath(
"//*/meta"
"[translate(@name, 'DESCRIPTION', 'description')='description']"
- "/@content")
+ "/@content"
+ )
if meta_description:
- og['og:description'] = meta_description[0]
+ og["og:description"] = meta_description[0]
else:
# grab any text nodes which are inside the <body/> tag...
# unless they are within an HTML5 semantic markup tag...
@@ -588,18 +578,18 @@ def _calc_og(tree, media_uri):
"script",
"noscript",
"style",
- etree.Comment
+ etree.Comment,
)
# Split all the text nodes into paragraphs (by splitting on new
# lines)
text_nodes = (
- re.sub(r'\s+', '\n', el).strip()
+ re.sub(r"\s+", "\n", el).strip()
for el in _iterate_over_text(tree.find("body"), *TAGS_TO_REMOVE)
)
- og['og:description'] = summarize_paragraphs(text_nodes)
+ og["og:description"] = summarize_paragraphs(text_nodes)
else:
- og['og:description'] = summarize_paragraphs([og['og:description']])
+ og["og:description"] = summarize_paragraphs([og["og:description"]])
# TODO: delete the url downloads to stop diskfilling,
# as we only ever cared about its OG
@@ -636,7 +626,7 @@ def _iterate_over_text(tree, *tags_to_ignore):
[child, child.tail] if child.tail else [child]
for child in el.iterchildren()
),
- elements
+ elements,
)
@@ -647,8 +637,8 @@ def _rebase_url(url, base):
url[0] = base[0] or "http"
if not url[1]: # fix up hostname
url[1] = base[1]
- if not url[2].startswith('/'):
- url[2] = re.sub(r'/[^/]+$', '/', base[2]) + url[2]
+ if not url[2].startswith("/"):
+ url[2] = re.sub(r"/[^/]+$", "/", base[2]) + url[2]
return urlparse.urlunparse(url)
@@ -659,9 +649,8 @@ def _is_media(content_type):
def _is_html(content_type):
content_type = content_type.lower()
- if (
- content_type.startswith("text/html") or
- content_type.startswith("application/xhtml")
+ if content_type.startswith("text/html") or content_type.startswith(
+ "application/xhtml"
):
return True
@@ -671,19 +660,19 @@ def summarize_paragraphs(text_nodes, min_size=200, max_size=500):
# first paragraph and then word boundaries.
# TODO: Respect sentences?
- description = ''
+ description = ""
# Keep adding paragraphs until we get to the MIN_SIZE.
for text_node in text_nodes:
if len(description) < min_size:
- text_node = re.sub(r'[\t \r\n]+', ' ', text_node)
- description += text_node + '\n\n'
+ text_node = re.sub(r"[\t \r\n]+", " ", text_node)
+ description += text_node + "\n\n"
else:
break
description = description.strip()
- description = re.sub(r'[\t ]+', ' ', description)
- description = re.sub(r'[\t \r\n]*[\r\n]+', '\n\n', description)
+ description = re.sub(r"[\t ]+", " ", description)
+ description = re.sub(r"[\t \r\n]*[\r\n]+", "\n\n", description)
# If the concatenation of paragraphs to get above MIN_SIZE
# took us over MAX_SIZE, then we need to truncate mid paragraph
@@ -715,5 +704,5 @@ def summarize_paragraphs(text_nodes, min_size=200, max_size=500):
# We always add an ellipsis because at the very least
# we chopped mid paragraph.
- description = new_desc.strip() + u"…"
+ description = new_desc.strip() + "…"
return description if description else None
diff --git a/synapse/rest/media/v1/storage_provider.py b/synapse/rest/media/v1/storage_provider.py
index d90cbfb56a..359b45ebfc 100644
--- a/synapse/rest/media/v1/storage_provider.py
+++ b/synapse/rest/media/v1/storage_provider.py
@@ -32,6 +32,7 @@ class StorageProvider(object):
"""A storage provider is a service that can store uploaded media and
retrieve them.
"""
+
def store_file(self, path, file_info):
"""Store the file described by file_info. The actual contents can be
retrieved by reading the file in file_info.upload_path.
@@ -70,6 +71,7 @@ class StorageProviderWrapper(StorageProvider):
uploaded, or todo the upload in the backgroud.
store_remote (bool): Whether remote media should be uploaded
"""
+
def __init__(self, backend, store_local, store_synchronous, store_remote):
self.backend = backend
self.store_local = store_local
@@ -92,6 +94,7 @@ class StorageProviderWrapper(StorageProvider):
return self.backend.store_file(path, file_info)
except Exception:
logger.exception("Error storing file")
+
run_in_background(store)
return defer.succeed(None)
@@ -123,8 +126,7 @@ class FileStorageProviderBackend(StorageProvider):
os.makedirs(dirname)
return logcontext.defer_to_thread(
- self.hs.get_reactor(),
- shutil.copyfile, primary_fname, backup_fname,
+ self.hs.get_reactor(), shutil.copyfile, primary_fname, backup_fname
)
def fetch(self, path, file_info):
diff --git a/synapse/rest/media/v1/thumbnail_resource.py b/synapse/rest/media/v1/thumbnail_resource.py
index 35a750923b..ca84c9f139 100644
--- a/synapse/rest/media/v1/thumbnail_resource.py
+++ b/synapse/rest/media/v1/thumbnail_resource.py
@@ -74,19 +74,18 @@ class ThumbnailResource(Resource):
else:
if self.dynamic_thumbnails:
yield self._select_or_generate_remote_thumbnail(
- request, server_name, media_id,
- width, height, method, m_type
+ request, server_name, media_id, width, height, method, m_type
)
else:
yield self._respond_remote_thumbnail(
- request, server_name, media_id,
- width, height, method, m_type
+ request, server_name, media_id, width, height, method, m_type
)
self.media_repo.mark_recently_accessed(server_name, media_id)
@defer.inlineCallbacks
- def _respond_local_thumbnail(self, request, media_id, width, height,
- method, m_type):
+ def _respond_local_thumbnail(
+ self, request, media_id, width, height, method, m_type
+ ):
media_info = yield self.store.get_local_media(media_id)
if not media_info:
@@ -105,7 +104,8 @@ class ThumbnailResource(Resource):
)
file_info = FileInfo(
- server_name=None, file_id=media_id,
+ server_name=None,
+ file_id=media_id,
url_cache=media_info["url_cache"],
thumbnail=True,
thumbnail_width=thumbnail_info["thumbnail_width"],
@@ -124,9 +124,15 @@ class ThumbnailResource(Resource):
respond_404(request)
@defer.inlineCallbacks
- def _select_or_generate_local_thumbnail(self, request, media_id, desired_width,
- desired_height, desired_method,
- desired_type):
+ def _select_or_generate_local_thumbnail(
+ self,
+ request,
+ media_id,
+ desired_width,
+ desired_height,
+ desired_method,
+ desired_type,
+ ):
media_info = yield self.store.get_local_media(media_id)
if not media_info:
@@ -146,7 +152,8 @@ class ThumbnailResource(Resource):
if t_w and t_h and t_method and t_type:
file_info = FileInfo(
- server_name=None, file_id=media_id,
+ server_name=None,
+ file_id=media_id,
url_cache=media_info["url_cache"],
thumbnail=True,
thumbnail_width=info["thumbnail_width"],
@@ -167,7 +174,11 @@ class ThumbnailResource(Resource):
# Okay, so we generate one.
file_path = yield self.media_repo.generate_local_exact_thumbnail(
- media_id, desired_width, desired_height, desired_method, desired_type,
+ media_id,
+ desired_width,
+ desired_height,
+ desired_method,
+ desired_type,
url_cache=media_info["url_cache"],
)
@@ -178,13 +189,20 @@ class ThumbnailResource(Resource):
respond_404(request)
@defer.inlineCallbacks
- def _select_or_generate_remote_thumbnail(self, request, server_name, media_id,
- desired_width, desired_height,
- desired_method, desired_type):
+ def _select_or_generate_remote_thumbnail(
+ self,
+ request,
+ server_name,
+ media_id,
+ desired_width,
+ desired_height,
+ desired_method,
+ desired_type,
+ ):
media_info = yield self.media_repo.get_remote_media_info(server_name, media_id)
thumbnail_infos = yield self.store.get_remote_media_thumbnails(
- server_name, media_id,
+ server_name, media_id
)
file_id = media_info["filesystem_id"]
@@ -197,7 +215,8 @@ class ThumbnailResource(Resource):
if t_w and t_h and t_method and t_type:
file_info = FileInfo(
- server_name=server_name, file_id=media_info["filesystem_id"],
+ server_name=server_name,
+ file_id=media_info["filesystem_id"],
thumbnail=True,
thumbnail_width=info["thumbnail_width"],
thumbnail_height=info["thumbnail_height"],
@@ -217,8 +236,13 @@ class ThumbnailResource(Resource):
# Okay, so we generate one.
file_path = yield self.media_repo.generate_remote_exact_thumbnail(
- server_name, file_id, media_id, desired_width,
- desired_height, desired_method, desired_type
+ server_name,
+ file_id,
+ media_id,
+ desired_width,
+ desired_height,
+ desired_method,
+ desired_type,
)
if file_path:
@@ -228,15 +252,16 @@ class ThumbnailResource(Resource):
respond_404(request)
@defer.inlineCallbacks
- def _respond_remote_thumbnail(self, request, server_name, media_id, width,
- height, method, m_type):
+ def _respond_remote_thumbnail(
+ self, request, server_name, media_id, width, height, method, m_type
+ ):
# TODO: Don't download the whole remote file
# We should proxy the thumbnail from the remote server instead of
# downloading the remote file and generating our own thumbnails.
media_info = yield self.media_repo.get_remote_media_info(server_name, media_id)
thumbnail_infos = yield self.store.get_remote_media_thumbnails(
- server_name, media_id,
+ server_name, media_id
)
if thumbnail_infos:
@@ -244,7 +269,8 @@ class ThumbnailResource(Resource):
width, height, method, m_type, thumbnail_infos
)
file_info = FileInfo(
- server_name=server_name, file_id=media_info["filesystem_id"],
+ server_name=server_name,
+ file_id=media_info["filesystem_id"],
thumbnail=True,
thumbnail_width=thumbnail_info["thumbnail_width"],
thumbnail_height=thumbnail_info["thumbnail_height"],
@@ -261,8 +287,14 @@ class ThumbnailResource(Resource):
logger.info("Failed to find any generated thumbnails")
respond_404(request)
- def _select_thumbnail(self, desired_width, desired_height, desired_method,
- desired_type, thumbnail_infos):
+ def _select_thumbnail(
+ self,
+ desired_width,
+ desired_height,
+ desired_method,
+ desired_type,
+ thumbnail_infos,
+ ):
d_w = desired_width
d_h = desired_height
@@ -280,15 +312,27 @@ class ThumbnailResource(Resource):
type_quality = desired_type != info["thumbnail_type"]
length_quality = info["thumbnail_length"]
if t_w >= d_w or t_h >= d_h:
- info_list.append((
- aspect_quality, min_quality, size_quality, type_quality,
- length_quality, info
- ))
+ info_list.append(
+ (
+ aspect_quality,
+ min_quality,
+ size_quality,
+ type_quality,
+ length_quality,
+ info,
+ )
+ )
else:
- info_list2.append((
- aspect_quality, min_quality, size_quality, type_quality,
- length_quality, info
- ))
+ info_list2.append(
+ (
+ aspect_quality,
+ min_quality,
+ size_quality,
+ type_quality,
+ length_quality,
+ info,
+ )
+ )
if info_list:
return min(info_list)[-1]
else:
@@ -304,13 +348,11 @@ class ThumbnailResource(Resource):
type_quality = desired_type != info["thumbnail_type"]
length_quality = info["thumbnail_length"]
if t_method == "scale" and (t_w >= d_w or t_h >= d_h):
- info_list.append((
- size_quality, type_quality, length_quality, info
- ))
+ info_list.append((size_quality, type_quality, length_quality, info))
elif t_method == "scale":
- info_list2.append((
- size_quality, type_quality, length_quality, info
- ))
+ info_list2.append(
+ (size_quality, type_quality, length_quality, info)
+ )
if info_list:
return min(info_list)[-1]
else:
diff --git a/synapse/rest/media/v1/thumbnailer.py b/synapse/rest/media/v1/thumbnailer.py
index 3efd0d80fc..90d8e6bffe 100644
--- a/synapse/rest/media/v1/thumbnailer.py
+++ b/synapse/rest/media/v1/thumbnailer.py
@@ -28,16 +28,13 @@ EXIF_TRANSPOSE_MAPPINGS = {
5: Image.TRANSPOSE,
6: Image.ROTATE_270,
7: Image.TRANSVERSE,
- 8: Image.ROTATE_90
+ 8: Image.ROTATE_90,
}
class Thumbnailer(object):
- FORMATS = {
- "image/jpeg": "JPEG",
- "image/png": "PNG",
- }
+ FORMATS = {"image/jpeg": "JPEG", "image/png": "PNG"}
def __init__(self, input_path):
self.image = Image.open(input_path)
@@ -110,17 +107,13 @@ class Thumbnailer(object):
"""
if width * self.height > height * self.width:
scaled_height = (width * self.height) // self.width
- scaled_image = self.image.resize(
- (width, scaled_height), Image.ANTIALIAS
- )
+ scaled_image = self.image.resize((width, scaled_height), Image.ANTIALIAS)
crop_top = (scaled_height - height) // 2
crop_bottom = height + crop_top
cropped = scaled_image.crop((0, crop_top, width, crop_bottom))
else:
scaled_width = (height * self.width) // self.height
- scaled_image = self.image.resize(
- (scaled_width, height), Image.ANTIALIAS
- )
+ scaled_image = self.image.resize((scaled_width, height), Image.ANTIALIAS)
crop_left = (scaled_width - width) // 2
crop_right = width + crop_left
cropped = scaled_image.crop((crop_left, 0, crop_right, height))
diff --git a/synapse/rest/media/v1/upload_resource.py b/synapse/rest/media/v1/upload_resource.py
index c1240e1963..d1d7e959f0 100644
--- a/synapse/rest/media/v1/upload_resource.py
+++ b/synapse/rest/media/v1/upload_resource.py
@@ -55,48 +55,36 @@ class UploadResource(Resource):
requester = yield self.auth.get_user_by_req(request)
# TODO: The checks here are a bit late. The content will have
# already been uploaded to a tmp file at this point
- content_length = request.getHeader(b"Content-Length").decode('ascii')
+ content_length = request.getHeader(b"Content-Length").decode("ascii")
if content_length is None:
- raise SynapseError(
- msg="Request must specify a Content-Length", code=400
- )
+ raise SynapseError(msg="Request must specify a Content-Length", code=400)
if int(content_length) > self.max_upload_size:
- raise SynapseError(
- msg="Upload request body is too large",
- code=413,
- )
+ raise SynapseError(msg="Upload request body is too large", code=413)
upload_name = parse_string(request, b"filename", encoding=None)
if upload_name:
try:
- upload_name = upload_name.decode('utf8')
+ upload_name = upload_name.decode("utf8")
except UnicodeDecodeError:
raise SynapseError(
- msg="Invalid UTF-8 filename parameter: %r" % (upload_name),
- code=400,
+ msg="Invalid UTF-8 filename parameter: %r" % (upload_name), code=400
)
headers = request.requestHeaders
if headers.hasHeader(b"Content-Type"):
- media_type = headers.getRawHeaders(b"Content-Type")[0].decode('ascii')
+ media_type = headers.getRawHeaders(b"Content-Type")[0].decode("ascii")
else:
- raise SynapseError(
- msg="Upload request missing 'Content-Type'",
- code=400,
- )
+ raise SynapseError(msg="Upload request missing 'Content-Type'", code=400)
# if headers.hasHeader(b"Content-Disposition"):
# disposition = headers.getRawHeaders(b"Content-Disposition")[0]
# TODO(markjh): parse content-dispostion
content_uri = yield self.media_repo.create_content(
- media_type, upload_name, request.content,
- content_length, requester.user
+ media_type, upload_name, request.content, content_length, requester.user
)
logger.info("Uploaded content with URI %r", content_uri)
- respond_with_json(
- request, 200, {"content_uri": content_uri}, send_cors=True
- )
+ respond_with_json(request, 200, {"content_uri": content_uri}, send_cors=True)
|
