From 279c48c8b442ec726fb5088e56ce9c1d2ed4bfb5 Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Tue, 2 Dec 2014 15:09:51 +0000 Subject: Write the upload portion of version 1 of the media repository --- synapse/http/content_repository.py | 212 ------------------------------------- synapse/http/server.py | 21 ++-- 2 files changed, 14 insertions(+), 219 deletions(-) delete mode 100644 synapse/http/content_repository.py (limited to 'synapse/http') diff --git a/synapse/http/content_repository.py b/synapse/http/content_repository.py deleted file mode 100644 index 64ecb5346e..0000000000 --- a/synapse/http/content_repository.py +++ /dev/null @@ -1,212 +0,0 @@ -# -*- coding: utf-8 -*- -# Copyright 2014 OpenMarket Ltd -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from .server import respond_with_json_bytes - -from synapse.util.stringutils import random_string -from synapse.api.errors import ( - cs_exception, SynapseError, CodeMessageException, Codes, cs_error -) - -from twisted.protocols.basic import FileSender -from twisted.web import server, resource -from twisted.internet import defer - -import base64 -import json -import logging -import os -import re - -logger = logging.getLogger(__name__) - - -class ContentRepoResource(resource.Resource): - """Provides file uploading and downloading. - - Uploads are POSTed to wherever this Resource is linked to. This resource - returns a "content token" which can be used to GET this content again. The - token is typically a path, but it may not be. Tokens can expire, be - one-time uses, etc. - - In this case, the token is a path to the file and contains 3 interesting - sections: - - User ID base64d (for namespacing content to each user) - - random 24 char string - - Content type base64d (so we can return it when clients GET it) - - """ - isLeaf = True - - def __init__(self, hs, directory, auth, external_addr): - resource.Resource.__init__(self) - self.hs = hs - self.directory = directory - self.auth = auth - self.external_addr = external_addr.rstrip('/') - self.max_upload_size = hs.config.max_upload_size - - if not os.path.isdir(self.directory): - os.mkdir(self.directory) - logger.info("ContentRepoResource : Created %s directory.", - self.directory) - - @defer.inlineCallbacks - def map_request_to_name(self, request): - # auth the user - auth_user = yield self.auth.get_user_by_req(request) - - # namespace all file uploads on the user - prefix = base64.urlsafe_b64encode( - auth_user.to_string() - ).replace('=', '') - - # use a random string for the main portion - main_part = random_string(24) - - # suffix with a file extension if we can make one. This is nice to - # provide a hint to clients on the file information. We will also reuse - # this info to spit back the content type to the client. - suffix = "" - if request.requestHeaders.hasHeader("Content-Type"): - content_type = request.requestHeaders.getRawHeaders( - "Content-Type")[0] - suffix = "." + base64.urlsafe_b64encode(content_type) - if (content_type.split("/")[0].lower() in - ["image", "video", "audio"]): - file_ext = content_type.split("/")[-1] - # be a little paranoid and only allow a-z - file_ext = re.sub("[^a-z]", "", file_ext) - suffix += "." + file_ext - - file_name = prefix + main_part + suffix - file_path = os.path.join(self.directory, file_name) - logger.info("User %s is uploading a file to path %s", - auth_user.to_string(), - file_path) - - # keep trying to make a non-clashing file, with a sensible max attempts - attempts = 0 - while os.path.exists(file_path): - main_part = random_string(24) - file_name = prefix + main_part + suffix - file_path = os.path.join(self.directory, file_name) - attempts += 1 - if attempts > 25: # really? Really? - raise SynapseError(500, "Unable to create file.") - - defer.returnValue(file_path) - - def render_GET(self, request): - # no auth here on purpose, to allow anyone to view, even across home - # servers. - - # TODO: A little crude here, we could do this better. - filename = request.path.split('/')[-1] - # be paranoid - filename = re.sub("[^0-9A-z.-_]", "", filename) - - file_path = self.directory + "/" + filename - - logger.debug("Searching for %s", file_path) - - if os.path.isfile(file_path): - # filename has the content type - base64_contentype = filename.split(".")[1] - content_type = base64.urlsafe_b64decode(base64_contentype) - logger.info("Sending file %s", file_path) - f = open(file_path, 'rb') - request.setHeader('Content-Type', content_type) - - # cache for at least a day. - # XXX: we might want to turn this off for data we don't want to - # recommend caching as it's sensitive or private - or at least - # select private. don't bother setting Expires as all our matrix - # clients are smart enough to be happy with Cache-Control (right?) - request.setHeader( - "Cache-Control", "public,max-age=86400,s-maxage=86400" - ) - - d = FileSender().beginFileTransfer(f, request) - - # after the file has been sent, clean up and finish the request - def cbFinished(ignored): - f.close() - request.finish() - d.addCallback(cbFinished) - else: - respond_with_json_bytes( - request, - 404, - json.dumps(cs_error("Not found", code=Codes.NOT_FOUND)), - send_cors=True) - - return server.NOT_DONE_YET - - def render_POST(self, request): - self._async_render(request) - return server.NOT_DONE_YET - - def render_OPTIONS(self, request): - respond_with_json_bytes(request, 200, {}, send_cors=True) - return server.NOT_DONE_YET - - @defer.inlineCallbacks - def _async_render(self, request): - try: - # TODO: The checks here are a bit late. The content will have - # already been uploaded to a tmp file at this point - content_length = request.getHeader("Content-Length") - if content_length is None: - raise SynapseError( - msg="Request must specify a Content-Length", code=400 - ) - if int(content_length) > self.max_upload_size: - raise SynapseError( - msg="Upload request body is too large", - code=413, - ) - - fname = yield self.map_request_to_name(request) - - # TODO I have a suspicious feeling this is just going to block - with open(fname, "wb") as f: - f.write(request.content.read()) - - # FIXME (erikj): These should use constants. - file_name = os.path.basename(fname) - # FIXME: we can't assume what the repo's public mounted path is - # ...plus self-signed SSL won't work to remote clients anyway - # ...and we can't assume that it's SSL anyway, as we might want to - # serve it via the non-SSL listener... - url = "%s/_matrix/content/%s" % ( - self.external_addr, file_name - ) - - respond_with_json_bytes(request, 200, - json.dumps({"content_token": url}), - send_cors=True) - - except CodeMessageException as e: - logger.exception(e) - respond_with_json_bytes(request, e.code, - json.dumps(cs_exception(e))) - except Exception as e: - logger.error("Failed to store file: %s" % e) - respond_with_json_bytes( - request, - 500, - json.dumps({"error": "Internal server error"}), - send_cors=True) diff --git a/synapse/http/server.py b/synapse/http/server.py index 8024ff5bde..046e230361 100644 --- a/synapse/http/server.py +++ b/synapse/http/server.py @@ -166,14 +166,10 @@ class JsonResource(HttpServer, resource.Resource): request) return - if not self._request_user_agent_is_curl(request): - json_bytes = encode_canonical_json(response_json_object) - else: - json_bytes = encode_pretty_printed_json(response_json_object) - # TODO: Only enable CORS for the requests that need it. - respond_with_json_bytes(request, code, json_bytes, send_cors=True, - response_code_message=response_code_message) + respond_with_json(request, code, response_json_object, send_cors=True, + response_code_message=response_code_message, + pretty_print=self._request_user_agent_is_curl) @staticmethod def _request_user_agent_is_curl(request): @@ -202,6 +198,17 @@ class RootRedirect(resource.Resource): return resource.Resource.getChild(self, name, request) +def respond_with_json(request, code, json_object, send_cors=False, + response_code_message=None, pretty_print=False): + if not pretty_print: + json_bytes = encode_pretty_printed_json(response_json_object) + else: + json_bytes = encode_canonical_json(response_json_object) + + return respond_with_json_bytes(request, code, json_bytes, send_cors, + response_code_message=response_code_message) + + def respond_with_json_bytes(request, code, json_bytes, send_cors=False, response_code_message=None): """Sends encoded JSON in response to the given request. -- cgit 1.4.1 From 5da65085d106e98cf7b762836cb300d01226bf92 Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Tue, 2 Dec 2014 19:51:47 +0000 Subject: Get uploads working with new media repo --- synapse/api/urls.py | 1 + synapse/app/homeserver.py | 9 +++++++-- synapse/config/_base.py | 14 ++++++++++++++ synapse/config/repository.py | 4 ++++ synapse/http/server.py | 4 ++-- synapse/media/__init__.py | 0 synapse/media/v0/__init__.py | 0 synapse/media/v0/content_repository.py | 2 +- synapse/media/v1/__init__.py | 0 synapse/media/v1/media_repository.py | 23 +++++++---------------- synapse/media/v1/upload_resource.py | 14 ++++++++------ synapse/server.py | 1 + synapse/storage/__init__.py | 6 +++++- synapse/storage/media_repository.py | 7 ++++++- 14 files changed, 56 insertions(+), 29 deletions(-) create mode 100644 synapse/media/__init__.py create mode 100644 synapse/media/v0/__init__.py create mode 100644 synapse/media/v1/__init__.py (limited to 'synapse/http') diff --git a/synapse/api/urls.py b/synapse/api/urls.py index 6dc19305b7..d7625127f8 100644 --- a/synapse/api/urls.py +++ b/synapse/api/urls.py @@ -20,3 +20,4 @@ FEDERATION_PREFIX = "/_matrix/federation/v1" WEB_CLIENT_PREFIX = "/_matrix/client" CONTENT_REPO_PREFIX = "/_matrix/content" SERVER_KEY_PREFIX = "/_matrix/key/v1" +MEDIA_PREFIX = "/_matrix/media/v1" diff --git a/synapse/app/homeserver.py b/synapse/app/homeserver.py index 855fe8e170..a6e29c0860 100755 --- a/synapse/app/homeserver.py +++ b/synapse/app/homeserver.py @@ -24,12 +24,13 @@ from twisted.web.resource import Resource from twisted.web.static import File from twisted.web.server import Site from synapse.http.server import JsonResource, RootRedirect -from synapse.http.content_repository import ContentRepoResource +from synapse.media.v0.content_repository import ContentRepoResource +from synapse.media.v1.media_repository import MediaRepositoryResource from synapse.http.server_key_resource import LocalKey from synapse.http.matrixfederationclient import MatrixFederationHttpClient from synapse.api.urls import ( CLIENT_PREFIX, FEDERATION_PREFIX, WEB_CLIENT_PREFIX, CONTENT_REPO_PREFIX, - SERVER_KEY_PREFIX, + SERVER_KEY_PREFIX, MEDIA_PREFIX ) from synapse.config.homeserver import HomeServerConfig from synapse.crypto import context_factory @@ -69,6 +70,9 @@ class SynapseHomeServer(HomeServer): self, self.upload_dir, self.auth, self.content_addr ) + def build_resource_for_media_repository(self): + return MediaRepositoryResource(self) + def build_resource_for_server_key(self): return LocalKey(self) @@ -99,6 +103,7 @@ class SynapseHomeServer(HomeServer): (FEDERATION_PREFIX, self.get_resource_for_federation()), (CONTENT_REPO_PREFIX, self.get_resource_for_content_repo()), (SERVER_KEY_PREFIX, self.get_resource_for_server_key()), + (MEDIA_PREFIX, self.get_resource_for_media_repository()), ] if web_client: logger.info("Adding the web client.") diff --git a/synapse/config/_base.py b/synapse/config/_base.py index 6870af10e8..1426436dcb 100644 --- a/synapse/config/_base.py +++ b/synapse/config/_base.py @@ -50,12 +50,26 @@ class Config(object): ) return cls.abspath(file_path) + @staticmethod + def ensure_directory(dir_path): + if not os.path.exists(dir_path): + os.makedirs(dir_path) + if not os.path.isdir(dir_path): + raise ConfigError( + "%s is not a directory" % (dir_path,) + ) + return dir_path + @classmethod def read_file(cls, file_path, config_name): cls.check_file(file_path, config_name) with open(file_path) as file_stream: return file_stream.read() + @staticmethod + def default_path(name): + return os.path.abspath(os.path.join(os.path.curdir, name)) + @staticmethod def read_config_file(file_path): with open(file_path) as file_stream: diff --git a/synapse/config/repository.py b/synapse/config/repository.py index 743bc26474..6eec930a03 100644 --- a/synapse/config/repository.py +++ b/synapse/config/repository.py @@ -20,6 +20,7 @@ class ContentRepositoryConfig(Config): def __init__(self, args): super(ContentRepositoryConfig, self).__init__(args) self.max_upload_size = self.parse_size(args.max_upload_size) + self.media_store_path = self.ensure_directory(args.media_store_path) def parse_size(self, string): sizes = {"K": 1024, "M": 1024 * 1024} @@ -37,3 +38,6 @@ class ContentRepositoryConfig(Config): db_group.add_argument( "--max-upload-size", default="1M" ) + db_group.add_argument( + "--media-store-path", default=cls.default_path("media_store") + ) diff --git a/synapse/http/server.py b/synapse/http/server.py index 046e230361..02277c4998 100644 --- a/synapse/http/server.py +++ b/synapse/http/server.py @@ -201,9 +201,9 @@ class RootRedirect(resource.Resource): def respond_with_json(request, code, json_object, send_cors=False, response_code_message=None, pretty_print=False): if not pretty_print: - json_bytes = encode_pretty_printed_json(response_json_object) + json_bytes = encode_pretty_printed_json(json_object) else: - json_bytes = encode_canonical_json(response_json_object) + json_bytes = encode_canonical_json(json_object) return respond_with_json_bytes(request, code, json_bytes, send_cors, response_code_message=response_code_message) diff --git a/synapse/media/__init__.py b/synapse/media/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/synapse/media/v0/__init__.py b/synapse/media/v0/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/synapse/media/v0/content_repository.py b/synapse/media/v0/content_repository.py index 64ecb5346e..ce5d3d153e 100644 --- a/synapse/media/v0/content_repository.py +++ b/synapse/media/v0/content_repository.py @@ -13,7 +13,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -from .server import respond_with_json_bytes +from synapse.http.server import respond_with_json_bytes from synapse.util.stringutils import random_string from synapse.api.errors import ( diff --git a/synapse/media/v1/__init__.py b/synapse/media/v1/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/synapse/media/v1/media_repository.py b/synapse/media/v1/media_repository.py index 9c36a8e933..0f4eeef278 100644 --- a/synapse/media/v1/media_repository.py +++ b/synapse/media/v1/media_repository.py @@ -13,27 +13,17 @@ # See the License for the specific language governing permissions and # limitations under the License. -from synapse.http.server import respond_with_json_bytes +from .upload_resource import UploadResource +from .filepath import MediaFilePaths -from synapse.util.stringutils import random_string -from synapse.api.errors import ( - cs_exception, SynapseError, CodeMessageException, Codes, cs_error -) +from twisted.web.resource import Resource -from twisted.protocols.basic import FileSender -from twisted.web import server, resource -from twisted.internet import defer - -import base64 -import json import logging -import os -import re logger = logging.getLogger(__name__) -class MediaRepository(): +class MediaRepositoryResource(Resource): """Profiles file uploading and downloading. Uploads are POSTed to a resource which returns a token which is used to GET @@ -68,5 +58,6 @@ class MediaRepository(): """ def __init__(self, hs): - filepaths = MediaFilePaths - + Resource.__init__(self) + filepaths = MediaFilePaths(hs.config.media_store_path) + self.putChild("upload", UploadResource(hs, filepaths)) diff --git a/synapse/media/v1/upload_resource.py b/synapse/media/v1/upload_resource.py index 3721a0173d..d9d7825b2b 100644 --- a/synapse/media/v1/upload_resource.py +++ b/synapse/media/v1/upload_resource.py @@ -23,6 +23,8 @@ from synapse.api.errors import ( from twisted.web import server, resource from twisted.internet import defer +import os + import logging logger = logging.getLogger(__name__) @@ -31,8 +33,9 @@ class UploadResource(resource.Resource): def __init__(self, hs, filepaths): self.auth = hs.get_auth() + self.clock = hs.get_clock() self.store = hs.get_datastore() - self.max_upload_size = hs.config.max_upload_size() + self.max_upload_size = hs.config.max_upload_size self.filepaths = filepaths def render_POST(self, request): @@ -45,10 +48,8 @@ class UploadResource(resource.Resource): @defer.inlineCallbacks def _async_render_POST(self, request): - - auth_user = yield self.auth.get_user_by_req(request) - try: + auth_user = yield self.auth.get_user_by_req(request) # TODO: The checks here are a bit late. The content will have # already been uploaded to a tmp file at this point content_length = request.getHeader("Content-Length") @@ -62,7 +63,7 @@ class UploadResource(resource.Resource): code=413, ) - headers = request.requestHeaders() + headers = request.requestHeaders if headers.hasHeader("Content-Type"): media_type = headers.getRawHeaders("Content-Type")[0] @@ -78,7 +79,8 @@ class UploadResource(resource.Resource): media_id = random_string(24) - fname = self.filepaths.local_media_file_path(media_id) + fname = self.filepaths.local_media_filepath(media_id) + os.makedirs(os.path.dirname(fname)) # This shouldn't block for very long because the content will have # already been uploaded at this point. diff --git a/synapse/server.py b/synapse/server.py index da0a44433a..7eb15270fc 100644 --- a/synapse/server.py +++ b/synapse/server.py @@ -78,6 +78,7 @@ class BaseHomeServer(object): 'resource_for_web_client', 'resource_for_content_repo', 'resource_for_server_key', + 'resource_for_media_repository', 'event_sources', 'ratelimiter', 'keyring', diff --git a/synapse/storage/__init__.py b/synapse/storage/__init__.py index 1231794de0..f6811a8117 100644 --- a/synapse/storage/__init__.py +++ b/synapse/storage/__init__.py @@ -33,6 +33,7 @@ from .stream import StreamStore from .transactions import TransactionStore from .keys import KeyStore from .event_federation import EventFederationStore +from .media_repository import MediaRepositoryStore from .state import StateStore from .signatures import SignatureStore @@ -62,6 +63,7 @@ SCHEMAS = [ "state", "event_edges", "event_signatures", + "media_repository", ] @@ -81,7 +83,9 @@ class DataStore(RoomMemberStore, RoomStore, RegistrationStore, StreamStore, ProfileStore, FeedbackStore, PresenceStore, TransactionStore, DirectoryStore, KeyStore, StateStore, SignatureStore, - EventFederationStore, ): + EventFederationStore, + MediaRepositoryStore, + ): def __init__(self, hs): super(DataStore, self).__init__(hs) diff --git a/synapse/storage/media_repository.py b/synapse/storage/media_repository.py index 73ceba3f2c..db03619a80 100644 --- a/synapse/storage/media_repository.py +++ b/synapse/storage/media_repository.py @@ -20,10 +20,15 @@ class MediaRepositoryStore(SQLBaseStore): """Persistence for attachments and avatars""" def get_local_media(self, media_id): + """Get the metadata for a local piece of media + Returns: + None if the media_id doesn't exist. + """ return self._simple_select_one( "local_media_repository", {"media_id": media_id}, ("media_type", "media_length", "upload_name", "created_ts"), + True, ) def store_local_media(self, media_id, media_type, time_now_ms, upload_name, @@ -36,7 +41,7 @@ class MediaRepositoryStore(SQLBaseStore): "created_ts": time_now_ms, "upload_name": upload_name, "media_length": media_length, - "user_id": user_id, + "user_id": user_id.to_string(), } ) -- cgit 1.4.1 From c01fd5573c92c7c6da258bac7ff377a91cbebfd1 Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Thu, 4 Dec 2014 14:22:31 +0000 Subject: Implement download support for media_repository --- synapse/http/matrixfederationclient.py | 73 ++++++++++++- synapse/media/v1/download_resource.py | 194 +++++++++++++++++++++++++++++++++ synapse/media/v1/media_repository.py | 2 + synapse/media/v1/upload_resource.py | 11 +- synapse/storage/media_repository.py | 10 +- 5 files changed, 278 insertions(+), 12 deletions(-) create mode 100644 synapse/media/v1/download_resource.py (limited to 'synapse/http') diff --git a/synapse/http/matrixfederationclient.py b/synapse/http/matrixfederationclient.py index 510f07dd7b..c7082b83a7 100644 --- a/synapse/http/matrixfederationclient.py +++ b/synapse/http/matrixfederationclient.py @@ -14,10 +14,11 @@ # limitations under the License. -from twisted.internet import defer, reactor +from twisted.internet import defer, reactor, protocol from twisted.internet.error import DNSLookupError from twisted.web.client import readBody, _AgentBase, _URI from twisted.web.http_headers import Headers +from twisted.web._newclient import ResponseDone from synapse.http.endpoint import matrix_federation_endpoint from synapse.util.async import sleep @@ -227,7 +228,7 @@ class MatrixFederationHttpClient(object): @defer.inlineCallbacks def get_json(self, destination, path, args={}, retry_on_dns_fail=True): - """ Get's some json from the given host homeserver and path + """ GETs some json from the given host homeserver and path Args: destination (str): The remote server to send the HTTP request @@ -235,9 +236,6 @@ class MatrixFederationHttpClient(object): path (str): The HTTP path. args (dict): A dictionary used to create query strings, defaults to None. - **Note**: The value of each key is assumed to be an iterable - and *not* a string. - Returns: Deferred: Succeeds when we get *any* HTTP response. @@ -272,6 +270,48 @@ class MatrixFederationHttpClient(object): defer.returnValue(json.loads(body)) + @defer.inlineCallbacks + def get_file(self, destination, path, output_stream, args={}, + retry_on_dns_fail=True): + """GETs a file from a given homeserver + Args: + destination (str): The remote server to send the HTTP request to. + path (str): The HTTP path to GET. + output_stream (file): File to write the response body to. + args (dict): Optional dictionary used to create the query string. + Returns: + A (int,dict) tuple of the file length and a dict of the response + headers. + """ + + encoded_args = {} + for k, vs in args.items(): + if isinstance(vs, basestring): + vs = [vs] + encoded_args[k] = [v.encode("UTF-8") for v in vs] + + query_bytes = urllib.urlencode(encoded_args, True) + logger.debug("Query bytes: %s Retry DNS: %s", args, retry_on_dns_fail) + + def body_callback(method, url_bytes, headers_dict): + self.sign_request(destination, method, url_bytes, headers_dict) + return None + + response = yield self._create_request( + destination.encode("ascii"), + "GET", + path.encode("ascii"), + query_bytes=query_bytes, + body_callback=body_callback, + retry_on_dns_fail=retry_on_dns_fail + ) + + headers = dict(response.headers.getAllRawHeaders()) + + length = yield _readBodyToFile(response, output_stream) + + defer.returnValue((length, headers)) + def _getEndpoint(self, reactor, destination): return matrix_federation_endpoint( reactor, destination, timeout=10, @@ -279,6 +319,29 @@ class MatrixFederationHttpClient(object): ) +class _ReadBodyToFileProtocol(protocol.Protocol): + def __init__(self, stream, deferred): + self.stream = stream + self.deferred = deferred + self.length = 0 + + def dataReceived(self, data): + self.stream.write(data) + self.length += len(data) + + def connectionLost(self, reason): + if reason.check(ResponseDone): + self.deferred.callback(self.length) + else: + self.deferred.errback(reason) + + +def _readBodyToFile(response, stream): + d = defer.Deferred() + response.deliverBody(_ReadBodyToFileProtocol(stream, d)) + return d + + def _print_ex(e): if hasattr(e, "reasons") and e.reasons: for ex in e.reasons: diff --git a/synapse/media/v1/download_resource.py b/synapse/media/v1/download_resource.py new file mode 100644 index 0000000000..c243f16a74 --- /dev/null +++ b/synapse/media/v1/download_resource.py @@ -0,0 +1,194 @@ +# -*- coding: utf-8 -*- +# Copyright 2014 OpenMarket Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from synapse.http.server import respond_with_json +from synapse.util.stringutils import random_string +from synapse.api.errors import ( + cs_exception, CodeMessageException, cs_error, Codes +) + +from twisted.protocols.basic import FileSender +from twisted.web.resource import Resource +from twisted.web.server import NOT_DONE_YET +from twisted.internet import defer + +import os + +import logging + +logger = logging.getLogger(__name__) + + +class DownloadResource(Resource): + isLeaf = True + + def __init__(self, hs, filepaths): + Resource.__init__(self) + self.client = hs.get_http_client() + self.clock = hs.get_clock() + self.server_name = hs.hostname + self.store = hs.get_datastore() + self.filepaths = filepaths + + def render_GET(self, request): + self._async_render_GET(request) + return NOT_DONE_YET + + def _respond_404(self, request): + respond_with_json( + request, 404, + cs_error( + "Not found %r" % (request.postpath,), + code=Codes.NOT_FOUND, + ), + send_cors=True + ) + + @defer.inlineCallbacks + def _async_render_GET(self, request): + + try: + server_name, media_id = request.postpath + except: + self._respond_404(request) + return + + try: + if server_name == self.server_name: + yield self._respond_local_file(request, media_id) + else: + yield self._respond_remote_file(request, server_name, media_id) + except CodeMessageException as e: + logger.exception(e) + respond_with_json(request, e.code, cs_exception(e), send_cors=True) + except: + logger.exception("Failed to serve file") + respond_with_json( + request, + 500, + {"error": "Internal server error"}, + send_cors=True + ) + + @defer.inlineCallbacks + def _download_remote_file(self, server_name, media_id): + filesystem_id = random_string(24) + + fname = self.filepaths.remote_media_filepath( + server_name, filesystem_id + ) + os.makedirs(os.path.dirname(fname)) + + try: + with open(fname, "wb") as f: + length, headers = yield self.client.get_file( + server_name, + "/".join(( + "/_matrix/media/v1/download", server_name, media_id, + )), + output_stream=f, + ) + except: + os.remove(fname) + raise + + media_type = headers["Content-Type"][0] + time_now_ms = self.clock.time_msec() + + yield self.store.store_cached_remote_media( + origin=server_name, + media_id=media_id, + media_type=media_type, + time_now_ms=self.clock.time_msec(), + upload_name=None, + media_length=length, + filesystem_id=filesystem_id, + ) + + defer.returnValue({ + "media_type": media_type, + "media_length": length, + "upload_name": None, + "created_ts": time_now_ms, + "filesystem_id": filesystem_id, + }) + + @defer.inlineCallbacks + def _respond_remote_file(self, request, server_name, media_id): + media_info = yield self.store.get_cached_remote_media( + server_name, media_id + ) + + if not media_info: + media_info = yield self._download_remote_file( + server_name, media_id + ) + + filesystem_id = media_info["filesystem_id"] + + file_path = self.filepaths.remote_media_filepath( + server_name, filesystem_id + ) + + if os.path.isfile(file_path): + media_type = media_info["media_type"] + request.setHeader(b"Content-Type", media_type.encode("UTF-8")) + + # cache for at least a day. + # XXX: we might want to turn this off for data we don't want to + # recommend caching as it's sensitive or private - or at least + # select private. don't bother setting Expires as all our + # clients are smart enough to be happy with Cache-Control + request.setHeader( + b"Cache-Control", b"public,max-age=86400,s-maxage=86400" + ) + + with open(file_path, "rb") as f: + yield FileSender().beginFileTransfer(f, request) + + request.finish() + else: + self._respond_404() + + @defer.inlineCallbacks + def _respond_local_file(self, request, media_id): + media_info = yield self.store.get_local_media(media_id) + if not media_info: + self._respond_404() + return + + file_path = self.filepaths.local_media_filepath(media_id) + + logger.debug("Searching for %s", file_path) + + if os.path.isfile(file_path): + media_type = media_info["media_type"] + request.setHeader(b"Content-Type", media_type.encode("UTF-8")) + + # cache for at least a day. + # XXX: we might want to turn this off for data we don't want to + # recommend caching as it's sensitive or private - or at least + # select private. don't bother setting Expires as all our + # clients are smart enough to be happy with Cache-Control + request.setHeader( + b"Cache-Control", b"public,max-age=86400,s-maxage=86400" + ) + + with open(file_path, "rb") as f: + yield FileSender().beginFileTransfer(f, request) + + request.finish() + else: + self._respond_404() diff --git a/synapse/media/v1/media_repository.py b/synapse/media/v1/media_repository.py index afd92874cf..e0a4cd01ee 100644 --- a/synapse/media/v1/media_repository.py +++ b/synapse/media/v1/media_repository.py @@ -14,6 +14,7 @@ # limitations under the License. from .upload_resource import UploadResource +from .download_resource import DownloadResource from .filepath import MediaFilePaths from twisted.web.resource import Resource @@ -62,3 +63,4 @@ class MediaRepositoryResource(Resource): Resource.__init__(self) filepaths = MediaFilePaths(hs.config.media_store_path) self.putChild("upload", UploadResource(hs, filepaths)) + self.putChild("download", DownloadResource(hs, filepaths)) diff --git a/synapse/media/v1/upload_resource.py b/synapse/media/v1/upload_resource.py index 2919fee12f..91bcc5caff 100644 --- a/synapse/media/v1/upload_resource.py +++ b/synapse/media/v1/upload_resource.py @@ -20,7 +20,8 @@ from synapse.api.errors import ( cs_exception, SynapseError, CodeMessageException ) -from twisted.web import server, resource +from twisted.web.resource import Resource +from twisted.web.server import NOT_DONE_YET from twisted.internet import defer import os @@ -30,9 +31,11 @@ import logging logger = logging.getLogger(__name__) -class UploadResource(resource.Resource): +class UploadResource(Resource): + isLeaf = True def __init__(self, hs, filepaths): + Resource.__init__(self) self.auth = hs.get_auth() self.clock = hs.get_clock() self.store = hs.get_datastore() @@ -41,11 +44,11 @@ class UploadResource(resource.Resource): def render_POST(self, request): self._async_render_POST(request) - return server.NOT_DONE_YET + return NOT_DONE_YET def render_OPTIONS(self, request): respond_with_json(request, 200, {}, send_cors=True) - return server.NOT_DONE_YET + return NOT_DONE_YET @defer.inlineCallbacks def _async_render_POST(self, request): diff --git a/synapse/storage/media_repository.py b/synapse/storage/media_repository.py index eda191ad5b..2d3a2d1ccb 100644 --- a/synapse/storage/media_repository.py +++ b/synapse/storage/media_repository.py @@ -22,13 +22,13 @@ class MediaRepositoryStore(SQLBaseStore): def get_local_media(self, media_id): """Get the metadata for a local piece of media Returns: - None if the media_id doesn't exist. + None if the meia_id doesn't exist. """ return self._simple_select_one( "local_media_repository", {"media_id": media_id}, ("media_type", "media_length", "upload_name", "created_ts"), - True, + allow_none=True, ) def store_local_media(self, media_id, media_type, time_now_ms, upload_name, @@ -73,7 +73,11 @@ class MediaRepositoryStore(SQLBaseStore): return self._simple_select_one( "remote_media_cache", {"media_origin": origin, "media_id": media_id}, - ("media_type", "media_length", "upload_name", "created_ts"), + ( + "media_type", "media_length", "upload_name", "created_ts", + "filesystem_id", + ), + allow_none=True, ) def store_cached_remote_media(self, origin, media_id, media_type, -- cgit 1.4.1 From d80d505b1f70eae128990ce1a9517e5c5edead73 Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Thu, 11 Dec 2014 14:19:32 +0000 Subject: Limit the size of images that are thumbnailed serverside. Limit the size of file that a server will download from a remote server --- synapse/api/errors.py | 1 + synapse/config/repository.py | 5 +++++ synapse/http/matrixfederationclient.py | 25 +++++++++++++++++++------ synapse/media/v1/base_resource.py | 18 ++++++++++++++++++ 4 files changed, 43 insertions(+), 6 deletions(-) (limited to 'synapse/http') diff --git a/synapse/api/errors.py b/synapse/api/errors.py index 581439ceb3..e250b9b211 100644 --- a/synapse/api/errors.py +++ b/synapse/api/errors.py @@ -34,6 +34,7 @@ class Codes(object): LIMIT_EXCEEDED = "M_LIMIT_EXCEEDED" CAPTCHA_NEEDED = "M_CAPTCHA_NEEDED" CAPTCHA_INVALID = "M_CAPTCHA_INVALID" + TOO_LARGE = "M_TOO_LARGE" class CodeMessageException(Exception): diff --git a/synapse/config/repository.py b/synapse/config/repository.py index 6eec930a03..f1b7b1b74e 100644 --- a/synapse/config/repository.py +++ b/synapse/config/repository.py @@ -20,6 +20,7 @@ class ContentRepositoryConfig(Config): def __init__(self, args): super(ContentRepositoryConfig, self).__init__(args) self.max_upload_size = self.parse_size(args.max_upload_size) + self.max_image_pixels = self.parse_size(args.max_image_pixels) self.media_store_path = self.ensure_directory(args.media_store_path) def parse_size(self, string): @@ -41,3 +42,7 @@ class ContentRepositoryConfig(Config): db_group.add_argument( "--media-store-path", default=cls.default_path("media_store") ) + db_group.add_argument( + "--max-image-pixels", default="32M", + help="Maximum number of pixels that will be thumbnailed" + ) diff --git a/synapse/http/matrixfederationclient.py b/synapse/http/matrixfederationclient.py index f05269cdfb..8f4db59c75 100644 --- a/synapse/http/matrixfederationclient.py +++ b/synapse/http/matrixfederationclient.py @@ -26,7 +26,7 @@ from synapse.util.logcontext import PreserveLoggingContext from syutil.jsonutil import encode_canonical_json -from synapse.api.errors import CodeMessageException, SynapseError +from synapse.api.errors import CodeMessageException, SynapseError, Codes from syutil.crypto.jsonsign import sign_json @@ -289,7 +289,7 @@ class MatrixFederationHttpClient(object): @defer.inlineCallbacks def get_file(self, destination, path, output_stream, args={}, - retry_on_dns_fail=True): + retry_on_dns_fail=True, max_size=None): """GETs a file from a given homeserver Args: destination (str): The remote server to send the HTTP request to. @@ -325,7 +325,11 @@ class MatrixFederationHttpClient(object): headers = dict(response.headers.getAllRawHeaders()) - length = yield _readBodyToFile(response, output_stream) + try: + length = yield _readBodyToFile(response, output_stream, max_size) + except: + logger.exception("Failed to download body") + raise defer.returnValue((length, headers)) @@ -337,14 +341,23 @@ class MatrixFederationHttpClient(object): class _ReadBodyToFileProtocol(protocol.Protocol): - def __init__(self, stream, deferred): + def __init__(self, stream, deferred, max_size): self.stream = stream self.deferred = deferred self.length = 0 + self.max_size = max_size def dataReceived(self, data): self.stream.write(data) self.length += len(data) + if self.max_size is not None and self.length >= self.max_size: + self.deferred.errback(SynapseError( + 502, + "Requested file is too large > %r bytes" % (self.max_size,), + Codes.TOO_LARGE, + )) + self.deferred = defer.Deferred() + self.transport.loseConnection() def connectionLost(self, reason): if reason.check(ResponseDone): @@ -353,9 +366,9 @@ class _ReadBodyToFileProtocol(protocol.Protocol): self.deferred.errback(reason) -def _readBodyToFile(response, stream): +def _readBodyToFile(response, stream, max_size): d = defer.Deferred() - response.deliverBody(_ReadBodyToFileProtocol(stream, d)) + response.deliverBody(_ReadBodyToFileProtocol(stream, d, max_size)) return d diff --git a/synapse/media/v1/base_resource.py b/synapse/media/v1/base_resource.py index 8c62ecd597..77b05c6548 100644 --- a/synapse/media/v1/base_resource.py +++ b/synapse/media/v1/base_resource.py @@ -43,6 +43,7 @@ class BaseMediaResource(Resource): self.server_name = hs.hostname self.store = hs.get_datastore() self.max_upload_size = hs.config.max_upload_size + self.max_image_pixels = hs.config.max_image_pixels self.filepaths = filepaths @staticmethod @@ -143,6 +144,7 @@ class BaseMediaResource(Resource): )) length, headers = yield self.client.get_file( server_name, request_path, output_stream=f, + max_size=self.max_upload_size, ) media_type = headers["Content-Type"][0] time_now_ms = self.clock.time_msec() @@ -226,6 +228,14 @@ class BaseMediaResource(Resource): thumbnailer = Thumbnailer(input_path) m_width = thumbnailer.width m_height = thumbnailer.height + + if m_width * m_height >= self.max_image_pixels: + logger.info( + "Image too large to thumbnail %r x %r > %r" + m_width, m_height, self.max_image_pixels + ) + return + scales = set() crops = set() for r_width, r_height, r_method, r_type in requirements: @@ -281,6 +291,14 @@ class BaseMediaResource(Resource): thumbnailer = Thumbnailer(input_path) m_width = thumbnailer.width m_height = thumbnailer.height + + if m_width * m_height >= self.max_image_pixels: + logger.info( + "Image too large to thumbnail %r x %r > %r" + m_width, m_height, self.max_image_pixels + ) + return + scales = set() crops = set() for r_width, r_height, r_method, r_type in requirements: -- cgit 1.4.1