diff options
Diffstat (limited to 'synapse/rest/media/v1/media_repository.py')
| -rw-r--r-- | synapse/rest/media/v1/media_repository.py | 215 |
1 files changed, 113 insertions, 102 deletions
diff --git a/synapse/rest/media/v1/media_repository.py b/synapse/rest/media/v1/media_repository.py index a4929dd5db..df3d985a38 100644 --- a/synapse/rest/media/v1/media_repository.py +++ b/synapse/rest/media/v1/media_repository.py @@ -100,17 +100,16 @@ class MediaRepository(object): storage_providers.append(provider) self.media_storage = MediaStorage( - self.hs, self.primary_base_path, self.filepaths, storage_providers, + self.hs, self.primary_base_path, self.filepaths, storage_providers ) self.clock.looping_call( - self._start_update_recently_accessed, - UPDATE_RECENTLY_ACCESSED_TS, + self._start_update_recently_accessed, UPDATE_RECENTLY_ACCESSED_TS ) def _start_update_recently_accessed(self): return run_as_background_process( - "update_recently_accessed_media", self._update_recently_accessed, + "update_recently_accessed_media", self._update_recently_accessed ) @defer.inlineCallbacks @@ -138,8 +137,9 @@ class MediaRepository(object): self.recently_accessed_locals.add(media_id) @defer.inlineCallbacks - def create_content(self, media_type, upload_name, content, content_length, - auth_user): + def create_content( + self, media_type, upload_name, content, content_length, auth_user + ): """Store uploaded content for a local user and return the mxc URL Args: @@ -154,10 +154,7 @@ class MediaRepository(object): """ media_id = random_string(24) - file_info = FileInfo( - server_name=None, - file_id=media_id, - ) + file_info = FileInfo(server_name=None, file_id=media_id) fname = yield self.media_storage.store_file(content, file_info) @@ -172,9 +169,7 @@ class MediaRepository(object): user_id=auth_user, ) - yield self._generate_thumbnails( - None, media_id, media_id, media_type, - ) + yield self._generate_thumbnails(None, media_id, media_id, media_type) defer.returnValue("mxc://%s/%s" % (self.server_name, media_id)) @@ -205,14 +200,11 @@ class MediaRepository(object): upload_name = name if name else media_info["upload_name"] url_cache = media_info["url_cache"] - file_info = FileInfo( - None, media_id, - url_cache=url_cache, - ) + file_info = FileInfo(None, media_id, url_cache=url_cache) responder = yield self.media_storage.fetch_media(file_info) yield respond_with_responder( - request, responder, media_type, media_length, upload_name, + request, responder, media_type, media_length, upload_name ) @defer.inlineCallbacks @@ -232,8 +224,8 @@ class MediaRepository(object): to request """ if ( - self.federation_domain_whitelist is not None and - server_name not in self.federation_domain_whitelist + self.federation_domain_whitelist is not None + and server_name not in self.federation_domain_whitelist ): raise FederationDeniedError(server_name) @@ -244,7 +236,7 @@ class MediaRepository(object): key = (server_name, media_id) with (yield self.remote_media_linearizer.queue(key)): responder, media_info = yield self._get_remote_media_impl( - server_name, media_id, + server_name, media_id ) # We deliberately stream the file outside the lock @@ -253,7 +245,7 @@ class MediaRepository(object): media_length = media_info["media_length"] upload_name = name if name else media_info["upload_name"] yield respond_with_responder( - request, responder, media_type, media_length, upload_name, + request, responder, media_type, media_length, upload_name ) else: respond_404(request) @@ -272,8 +264,8 @@ class MediaRepository(object): Deferred[dict]: The media_info of the file """ if ( - self.federation_domain_whitelist is not None and - server_name not in self.federation_domain_whitelist + self.federation_domain_whitelist is not None + and server_name not in self.federation_domain_whitelist ): raise FederationDeniedError(server_name) @@ -282,7 +274,7 @@ class MediaRepository(object): key = (server_name, media_id) with (yield self.remote_media_linearizer.queue(key)): responder, media_info = yield self._get_remote_media_impl( - server_name, media_id, + server_name, media_id ) # Ensure we actually use the responder so that it releases resources @@ -305,9 +297,7 @@ class MediaRepository(object): Returns: Deferred[(Responder, media_info)] """ - media_info = yield self.store.get_cached_remote_media( - server_name, media_id - ) + media_info = yield self.store.get_cached_remote_media(server_name, media_id) # file_id is the ID we use to track the file locally. If we've already # seen the file then reuse the existing ID, otherwise genereate a new @@ -331,9 +321,7 @@ class MediaRepository(object): # Failed to find the file anywhere, lets download it. - media_info = yield self._download_remote_file( - server_name, media_id, file_id - ) + media_info = yield self._download_remote_file(server_name, media_id, file_id) responder = yield self.media_storage.fetch_media(file_info) defer.returnValue((responder, media_info)) @@ -354,54 +342,60 @@ class MediaRepository(object): Deferred[MediaInfo] """ - file_info = FileInfo( - server_name=server_name, - file_id=file_id, - ) + file_info = FileInfo(server_name=server_name, file_id=file_id) with self.media_storage.store_into_file(file_info) as (f, fname, finish): - request_path = "/".join(( - "/_matrix/media/v1/download", server_name, media_id, - )) + request_path = "/".join( + ("/_matrix/media/v1/download", server_name, media_id) + ) try: length, headers = yield self.client.get_file( - server_name, request_path, output_stream=f, - max_size=self.max_upload_size, args={ + server_name, + request_path, + output_stream=f, + max_size=self.max_upload_size, + args={ # tell the remote server to 404 if it doesn't # recognise the server_name, to make sure we don't # end up with a routing loop. - "allow_remote": "false", - } + "allow_remote": "false" + }, ) except RequestSendFailed as e: - logger.warn("Request failed fetching remote media %s/%s: %r", - server_name, media_id, e) + logger.warn( + "Request failed fetching remote media %s/%s: %r", + server_name, + media_id, + e, + ) raise SynapseError(502, "Failed to fetch remote media") except HttpResponseException as e: - logger.warn("HTTP error fetching remote media %s/%s: %s", - server_name, media_id, e.response) + logger.warn( + "HTTP error fetching remote media %s/%s: %s", + server_name, + media_id, + e.response, + ) if e.code == twisted.web.http.NOT_FOUND: raise e.to_synapse_error() raise SynapseError(502, "Failed to fetch remote media") except SynapseError: - logger.warn( - "Failed to fetch remote media %s/%s", - server_name, media_id, - ) + logger.warn("Failed to fetch remote media %s/%s", server_name, media_id) raise except NotRetryingDestination: logger.warn("Not retrying destination %r", server_name) raise SynapseError(502, "Failed to fetch remote media") except Exception: - logger.exception("Failed to fetch remote media %s/%s", - server_name, media_id) + logger.exception( + "Failed to fetch remote media %s/%s", server_name, media_id + ) raise SynapseError(502, "Failed to fetch remote media") yield finish() - media_type = headers[b"Content-Type"][0].decode('ascii') + media_type = headers[b"Content-Type"][0].decode("ascii") upload_name = get_filename_from_headers(headers) time_now_ms = self.clock.time_msec() @@ -425,24 +419,23 @@ class MediaRepository(object): "filesystem_id": file_id, } - yield self._generate_thumbnails( - server_name, media_id, file_id, media_type, - ) + yield self._generate_thumbnails(server_name, media_id, file_id, media_type) defer.returnValue(media_info) def _get_thumbnail_requirements(self, media_type): return self.thumbnail_requirements.get(media_type, ()) - def _generate_thumbnail(self, thumbnailer, t_width, t_height, - t_method, t_type): + def _generate_thumbnail(self, thumbnailer, t_width, t_height, t_method, t_type): m_width = thumbnailer.width m_height = thumbnailer.height if m_width * m_height >= self.max_image_pixels: logger.info( "Image too large to thumbnail %r x %r > %r", - m_width, m_height, self.max_image_pixels + m_width, + m_height, + self.max_image_pixels, ) return @@ -462,17 +455,22 @@ class MediaRepository(object): return t_byte_source @defer.inlineCallbacks - def generate_local_exact_thumbnail(self, media_id, t_width, t_height, - t_method, t_type, url_cache): - input_path = yield self.media_storage.ensure_media_is_in_local_cache(FileInfo( - None, media_id, url_cache=url_cache, - )) + def generate_local_exact_thumbnail( + self, media_id, t_width, t_height, t_method, t_type, url_cache + ): + input_path = yield self.media_storage.ensure_media_is_in_local_cache( + FileInfo(None, media_id, url_cache=url_cache) + ) thumbnailer = Thumbnailer(input_path) t_byte_source = yield logcontext.defer_to_thread( self.hs.get_reactor(), self._generate_thumbnail, - thumbnailer, t_width, t_height, t_method, t_type + thumbnailer, + t_width, + t_height, + t_method, + t_type, ) if t_byte_source: @@ -489,7 +487,7 @@ class MediaRepository(object): ) output_path = yield self.media_storage.store_file( - t_byte_source, file_info, + t_byte_source, file_info ) finally: t_byte_source.close() @@ -505,17 +503,22 @@ class MediaRepository(object): defer.returnValue(output_path) @defer.inlineCallbacks - def generate_remote_exact_thumbnail(self, server_name, file_id, media_id, - t_width, t_height, t_method, t_type): - input_path = yield self.media_storage.ensure_media_is_in_local_cache(FileInfo( - server_name, file_id, url_cache=False, - )) + def generate_remote_exact_thumbnail( + self, server_name, file_id, media_id, t_width, t_height, t_method, t_type + ): + input_path = yield self.media_storage.ensure_media_is_in_local_cache( + FileInfo(server_name, file_id, url_cache=False) + ) thumbnailer = Thumbnailer(input_path) t_byte_source = yield logcontext.defer_to_thread( self.hs.get_reactor(), self._generate_thumbnail, - thumbnailer, t_width, t_height, t_method, t_type + thumbnailer, + t_width, + t_height, + t_method, + t_type, ) if t_byte_source: @@ -531,7 +534,7 @@ class MediaRepository(object): ) output_path = yield self.media_storage.store_file( - t_byte_source, file_info, + t_byte_source, file_info ) finally: t_byte_source.close() @@ -541,15 +544,22 @@ class MediaRepository(object): t_len = os.path.getsize(output_path) yield self.store.store_remote_media_thumbnail( - server_name, media_id, file_id, - t_width, t_height, t_type, t_method, t_len + server_name, + media_id, + file_id, + t_width, + t_height, + t_type, + t_method, + t_len, ) defer.returnValue(output_path) @defer.inlineCallbacks - def _generate_thumbnails(self, server_name, media_id, file_id, media_type, - url_cache=False): + def _generate_thumbnails( + self, server_name, media_id, file_id, media_type, url_cache=False + ): """Generate and store thumbnails for an image. Args: @@ -568,9 +578,9 @@ class MediaRepository(object): if not requirements: return - input_path = yield self.media_storage.ensure_media_is_in_local_cache(FileInfo( - server_name, file_id, url_cache=url_cache, - )) + input_path = yield self.media_storage.ensure_media_is_in_local_cache( + FileInfo(server_name, file_id, url_cache=url_cache) + ) thumbnailer = Thumbnailer(input_path) m_width = thumbnailer.width @@ -579,14 +589,15 @@ class MediaRepository(object): if m_width * m_height >= self.max_image_pixels: logger.info( "Image too large to thumbnail %r x %r > %r", - m_width, m_height, self.max_image_pixels + m_width, + m_height, + self.max_image_pixels, ) return if thumbnailer.transpose_method is not None: m_width, m_height = yield logcontext.defer_to_thread( - self.hs.get_reactor(), - thumbnailer.transpose + self.hs.get_reactor(), thumbnailer.transpose ) # We deduplicate the thumbnail sizes by ignoring the cropped versions if @@ -606,15 +617,11 @@ class MediaRepository(object): # Generate the thumbnail if t_method == "crop": t_byte_source = yield logcontext.defer_to_thread( - self.hs.get_reactor(), - thumbnailer.crop, - t_width, t_height, t_type, + self.hs.get_reactor(), thumbnailer.crop, t_width, t_height, t_type ) elif t_method == "scale": t_byte_source = yield logcontext.defer_to_thread( - self.hs.get_reactor(), - thumbnailer.scale, - t_width, t_height, t_type, + self.hs.get_reactor(), thumbnailer.scale, t_width, t_height, t_type ) else: logger.error("Unrecognized method: %r", t_method) @@ -636,7 +643,7 @@ class MediaRepository(object): ) output_path = yield self.media_storage.store_file( - t_byte_source, file_info, + t_byte_source, file_info ) finally: t_byte_source.close() @@ -646,18 +653,21 @@ class MediaRepository(object): # Write to database if server_name: yield self.store.store_remote_media_thumbnail( - server_name, media_id, file_id, - t_width, t_height, t_type, t_method, t_len + server_name, + media_id, + file_id, + t_width, + t_height, + t_type, + t_method, + t_len, ) else: yield self.store.store_local_thumbnail( media_id, t_width, t_height, t_type, t_method, t_len ) - defer.returnValue({ - "width": m_width, - "height": m_height, - }) + defer.returnValue({"width": m_width, "height": m_height}) @defer.inlineCallbacks def delete_old_remote_media(self, before_ts): @@ -749,11 +759,12 @@ class MediaRepositoryResource(Resource): self.putChild(b"upload", UploadResource(hs, media_repo)) self.putChild(b"download", DownloadResource(hs, media_repo)) - self.putChild(b"thumbnail", ThumbnailResource( - hs, media_repo, media_repo.media_storage, - )) + self.putChild( + b"thumbnail", ThumbnailResource(hs, media_repo, media_repo.media_storage) + ) if hs.config.url_preview_enabled: - self.putChild(b"preview_url", PreviewUrlResource( - hs, media_repo, media_repo.media_storage, - )) + self.putChild( + b"preview_url", + PreviewUrlResource(hs, media_repo, media_repo.media_storage), + ) self.putChild(b"config", MediaConfigResource(hs)) |