diff --git a/docs/media_repository.rst b/docs/media_repository.rst
new file mode 100644
index 0000000000..e554d0f495
--- /dev/null
+++ b/docs/media_repository.rst
@@ -0,0 +1,20 @@
+Media Repository
+================
+
+The media repository is where attachments and avatar photos are stored.
+It stores attachment content and thumbnails for media uploaded by local users.
+It caches attachment content and thumbnails for media uploaded by remote users.
+
+Storage
+-------
+
+Each item of media is assigned a ``media_id`` when it is uploaded.
+The ``media_id`` is a randomly chosen, URL safe 24 character string.
+Metadata such as the MIME type, upload time and length are stored in the
+sqlite3 database indexed by ``media_id``.
+Content is stored on the filesystem under a "content" directory. Thumbnails are
+stored under a "thumbnails" directory.
+The item with ``media_id`` ``"aabbccccccccdddddddddddd"`` is stored under
+``"local/content/aa/bb/ccccccccdddddddddddd"``. Its thumbnail with width
+``128`` and height ``96`` and type ``"image/jpeg"`` is stored under
+``"local/thumbnails/aa/bb/ccccccccdddddddddddd/128-96-image-jpeg"``
diff --git a/synapse/http/server.py b/synapse/http/server.py
index 8024ff5bde..046e230361 100644
--- a/synapse/http/server.py
+++ b/synapse/http/server.py
@@ -166,14 +166,10 @@ class JsonResource(HttpServer, resource.Resource):
request)
return
- if not self._request_user_agent_is_curl(request):
- json_bytes = encode_canonical_json(response_json_object)
- else:
- json_bytes = encode_pretty_printed_json(response_json_object)
-
# TODO: Only enable CORS for the requests that need it.
- respond_with_json_bytes(request, code, json_bytes, send_cors=True,
- response_code_message=response_code_message)
+ respond_with_json(request, code, response_json_object, send_cors=True,
+ response_code_message=response_code_message,
+ pretty_print=self._request_user_agent_is_curl)
@staticmethod
def _request_user_agent_is_curl(request):
@@ -202,6 +198,17 @@ class RootRedirect(resource.Resource):
return resource.Resource.getChild(self, name, request)
+def respond_with_json(request, code, json_object, send_cors=False,
+ response_code_message=None, pretty_print=False):
+ if not pretty_print:
+ json_bytes = encode_pretty_printed_json(response_json_object)
+ else:
+ json_bytes = encode_canonical_json(response_json_object)
+
+ return respond_with_json_bytes(request, code, json_bytes, send_cors,
+ response_code_message=response_code_message)
+
+
def respond_with_json_bytes(request, code, json_bytes, send_cors=False,
response_code_message=None):
"""Sends encoded JSON in response to the given request.
diff --git a/synapse/http/content_repository.py b/synapse/media/v0/content_repository.py
index 64ecb5346e..64ecb5346e 100644
--- a/synapse/http/content_repository.py
+++ b/synapse/media/v0/content_repository.py
diff --git a/synapse/media/v1/filepath.py b/synapse/media/v1/filepath.py
new file mode 100644
index 0000000000..d23564e03e
--- /dev/null
+++ b/synapse/media/v1/filepath.py
@@ -0,0 +1,53 @@
+# -*- coding: utf-8 -*-
+# Copyright 2014 OpenMarket Ltd
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import os
+
+
+class MediaFilePaths(object):
+
+ def __init__(self, base_path):
+ self.base_path = base_path
+
+ def local_media_filepath(self, media_id):
+ return os.path.join(
+ self.base_path, "local", "content",
+ media_id[0:2], media_id[2:4], media_id[4:]
+ )
+
+ def local_media_thumbnail(self, media_id, width, height, content_type):
+ top_level_type, sub_type = content_type.split("/")
+ file_name = "%i-%i-%s-%s" % (width, height, top_level_type, sub_type)
+ return os.path.join(
+ self.base_path, "local", "thumbnails",
+ media_id[0:2], media_id[2:4], media_id[4:],
+ file_name
+ )
+
+ def remote_media_filepath(self, server_name, file_id):
+ return os.path.join(
+ self.base_path, "remote", "content", server_name,
+ file_id[0:2], file_id[2:4], file_id[4:]
+ )
+
+ def remote_media_thumbnail(self, server_name, file_id, width, height,
+ content_type):
+ top_level_type, sub_type = content_type.split("/")
+ file_name = "%i-%i-%s-%s" % (width, height, top_level_type, sub_type)
+ return os.path.join(
+ self.base_path, "remote", "content", server_name,
+ file_id[0:2], file_id[2:4], file_id[4:],
+ file_name
+ )
diff --git a/synapse/media/v1/media_repository.py b/synapse/media/v1/media_repository.py
new file mode 100644
index 0000000000..9c36a8e933
--- /dev/null
+++ b/synapse/media/v1/media_repository.py
@@ -0,0 +1,72 @@
+# -*- coding: utf-8 -*-
+# Copyright 2014 OpenMarket Ltd
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from synapse.http.server import respond_with_json_bytes
+
+from synapse.util.stringutils import random_string
+from synapse.api.errors import (
+ cs_exception, SynapseError, CodeMessageException, Codes, cs_error
+)
+
+from twisted.protocols.basic import FileSender
+from twisted.web import server, resource
+from twisted.internet import defer
+
+import base64
+import json
+import logging
+import os
+import re
+
+logger = logging.getLogger(__name__)
+
+
+class MediaRepository():
+ """Profiles file uploading and downloading.
+
+ Uploads are POSTed to a resource which returns a token which is used to GET
+ the download::
+
+ => POST /_matrix/media/v1/upload HTTP/1.1
+ Content-Type: <media-type>
+
+ <media>
+
+ <= HTTP/1.1 200 OK
+ Content-Type: application/json
+
+ { "token": <media-id> }
+
+ => GET /_matrix/media/v1/download/<media-id> HTTP/1.1
+
+ <= HTTP/1.1 200 OK
+ Content-Type: <media-type>
+ Content-Disposition: attachment;filename=<upload-filename>
+
+ <media>
+
+ Clients can get thumbnails by supplying a desired width and height::
+
+ => GET /_matrix/media/v1/thumbnail/<media-id>?width=<w>&height=<h> HTTP/1.1
+
+ <= HTTP/1.1 200 OK
+ Content-Type: image/jpeg or image/png
+
+ <thumbnail>
+ """
+
+ def __init__(self, hs):
+ filepaths = MediaFilePaths
+
diff --git a/synapse/media/v1/upload_resource.py b/synapse/media/v1/upload_resource.py
new file mode 100644
index 0000000000..3721a0173d
--- /dev/null
+++ b/synapse/media/v1/upload_resource.py
@@ -0,0 +1,110 @@
+# -*- coding: utf-8 -*-
+# Copyright 2014 OpenMarket Ltd
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from synapse.http.server import respond_with_json
+
+from synapse.util.stringutils import random_string
+from synapse.api.errors import (
+ cs_exception, SynapseError, CodeMessageException
+)
+
+from twisted.web import server, resource
+from twisted.internet import defer
+
+import logging
+
+logger = logging.getLogger(__name__)
+
+class UploadResource(resource.Resource):
+
+ def __init__(self, hs, filepaths):
+ self.auth = hs.get_auth()
+ self.store = hs.get_datastore()
+ self.max_upload_size = hs.config.max_upload_size()
+ self.filepaths = filepaths
+
+ def render_POST(self, request):
+ self._async_render_POST(request)
+ return server.NOT_DONE_YET
+
+ def render_OPTIONS(self, request):
+ respond_with_json(request, 200, {}, send_cors=True)
+ return server.NOT_DONE_YET
+
+ @defer.inlineCallbacks
+ def _async_render_POST(self, request):
+
+ auth_user = yield self.auth.get_user_by_req(request)
+
+ try:
+ # TODO: The checks here are a bit late. The content will have
+ # already been uploaded to a tmp file at this point
+ content_length = request.getHeader("Content-Length")
+ if content_length is None:
+ raise SynapseError(
+ msg="Request must specify a Content-Length", code=400
+ )
+ if int(content_length) > self.max_upload_size:
+ raise SynapseError(
+ msg="Upload request body is too large",
+ code=413,
+ )
+
+ headers = request.requestHeaders()
+
+ if headers.hasHeader("Content-Type"):
+ media_type = headers.getRawHeaders("Content-Type")[0]
+ else:
+ raise SynapseError(
+ msg="Upload request missing 'Content-Type'",
+ code=400,
+ )
+
+ #if headers.hasHeader("Content-Disposition"):
+ # disposition = headers.getRawHeaders("Content-Disposition")[0]
+ # TODO(markjh): parse content-dispostion
+
+ media_id = random_string(24)
+
+ fname = self.filepaths.local_media_file_path(media_id)
+
+ # This shouldn't block for very long because the content will have
+ # already been uploaded at this point.
+ with open(fname, "wb") as f:
+ f.write(request.content.read())
+
+ yield self.store.store_local_media(
+ media_id=media_id,
+ media_type=media_type,
+ time_now_ms=self.clock.time_msec(),
+ upload_name=None,
+ media_length=content_length,
+ user_id=auth_user,
+ )
+
+ respond_with_json(
+ request, 200, {"content_token": media_id}, send_cors=True
+ )
+ except CodeMessageException as e:
+ logger.exception(e)
+ respond_with_json(request, e.code, cs_exception(e), send_cors=True)
+ except:
+ logger.exception("Failed to store file")
+ respond_with_json(
+ request,
+ 500,
+ {"error": "Internal server error"},
+ send_cors=True
+ )
diff --git a/synapse/storage/media_repository.py b/synapse/storage/media_repository.py
new file mode 100644
index 0000000000..73ceba3f2c
--- /dev/null
+++ b/synapse/storage/media_repository.py
@@ -0,0 +1,116 @@
+# -*- coding: utf-8 -*-
+# Copyright 2014 OpenMarket Ltd
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from _base import SQLBaseStore
+
+
+class MediaRepositoryStore(SQLBaseStore):
+ """Persistence for attachments and avatars"""
+
+ def get_local_media(self, media_id):
+ return self._simple_select_one(
+ "local_media_repository",
+ {"media_id": media_id},
+ ("media_type", "media_length", "upload_name", "created_ts"),
+ )
+
+ def store_local_media(self, media_id, media_type, time_now_ms, upload_name,
+ media_length, user_id):
+ return self._simple_insert(
+ "local_media_repository",
+ {
+ "media_id": media_id,
+ "media_type": media_type,
+ "created_ts": time_now_ms,
+ "upload_name": upload_name,
+ "media_length": media_length,
+ "user_id": user_id,
+ }
+ )
+
+ def get_local_media_thumbnails(self, media_id):
+ return self._simple_select_list(
+ "local_media_thumbnails",
+ {"media_id": media_id},
+ (
+ "thumbnail_width", "thumbnail_height",
+ "thumbnail_type", "thumbnail_length",
+ )
+ )
+
+ def store_local_thumbnail(self, media_id, thumbnail_width,
+ thumbnail_height, thumbnail_type,
+ thumbnail_length):
+ return self._simple_insert(
+ "local_media_thumbnails",
+ {
+ "media_id": media_id,
+ "thumbnail_width": thumbnail_width,
+ "thumbnail_height": thumbnail_height,
+ "thumbnail_type": thumbnail_type,
+ "thumbnail_length": thumbnail_length,
+ }
+ )
+
+ def get_cached_remote_media(self, origin, media_id):
+ return self._simple_select_one(
+ "remote_media_cache",
+ {"media_origin": origin, "media_id": media_id},
+ ("media_type", "media_length", "upload_name", "created_ts"),
+ )
+
+ def store_cached_remote_media(self, origin, media_id, media_type,
+ media_length, time_now_ms, upload_name,
+ filesytem_id):
+ return self._simple_insert(
+ "remote_media_cache",
+ {
+ "media_origin": origin,
+ "media_id": media_id,
+ "media_type": media_type,
+ "media_length": media_length,
+ "created_ts": time_now_ms,
+ "upload_name": upload_name,
+ "filesystem_id": filesystem_id,
+ }
+ )
+
+ def get_remote_media_thumbnails(self, origin, media_id):
+ return self._simple_select_list(
+ "remote_media_cache_thumbnails",
+ {"origin": origin, "media_id": media_id},
+ (
+ "thumbnail_width", "thumbnail_height",
+ "thumbnail_type", "thumbnail_length",
+ "filesystem_id"
+ )
+ )
+
+
+ def store_remote_media_thumbnail(self, origin, media_id, thumbnail_width,
+ thumbnail_height, thumbnail_type,
+ thumbnail_length, filesystem_id):
+ return self._simple_insert(
+ "remote_media_cache_thumbnails",
+ {
+ "media_origin": origin,
+ "media_id": media_id,
+ "thumbnail_width": thumbnail_width,
+ "thumbnail_height": thumbnail_height,
+ "thumbnail_type": thumbnail_type,
+ "thumbnail_length": thumbnail_length,
+ "filesystem_id": filesystem_id,
+ }
+ )
diff --git a/synapse/storage/schema/media_repository.sql b/synapse/storage/schema/media_repository.sql
new file mode 100644
index 0000000000..768752296f
--- /dev/null
+++ b/synapse/storage/schema/media_repository.sql
@@ -0,0 +1,66 @@
+/* Copyright 2014 OpenMarket Ltd
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+CREATE TABLE IF NOT EXISTS local_media_repository (
+ media_id TEXT, -- The id used to refer to the media.
+ media_type TEXT, -- The MIME-type of the media.
+ media_length INTEGER, -- Length of the media in bytes.
+ created_ts INTEGER, -- When the content was uploaded in ms.
+ upload_name TEXT, -- The name the media was uploaded with.
+ user_id TEXT, -- The user who uploaded the file.
+ CONSTRAINT uniqueness UNIQUE (media_id)
+);
+
+CREATE TABLE IF NOT EXISTS local_media_repository_thumbnails (
+ media_id TEXT, -- The id used to refer to the media.
+ thumbnail_width INTEGER, -- The width of the thumbnail in pixels.
+ thumbnail_height INTEGER, -- The height of the thumbnail in pixels.
+ thumbnail_type TEXT, -- The MIME-type of the thumbnail.
+ thumbnail_length INTEGER, -- The length of the thumbnail in bytes.
+ CONSTRAINT uniqueness UNIQUE (
+ media_id, thumbnail_width, thumbnail_height, thumbnail_type
+ )
+);
+
+CREATE INDEX IF NOT EXISTS local_media_repository_thumbnails_media_id
+ ON local_media_repository_thumbnails (media_id);
+
+CREATE TABLE IF NOT EXISTS remote_media_cache (
+ media_origin TEXT, -- The remote HS the media came from.
+ media_id TEXT, -- The id used to refer to the media on that server.
+ media_type TEXT, -- The MIME-type of the media.
+ created_ts INTEGER, -- When the content was uploaded in ms.
+ upload_name TEXT, -- The name the media was uploaded with.
+ media_length INTEGER, -- Length of the media in bytes.
+ filesystem_id TEXT, -- The name used to store the media on disk.
+ CONSTRAINT uniqueness UNIQUE (media_origin, media_id)
+);
+
+CREATE TABLE IF NOT EXISTS remote_media_cache_thumbnails (
+ media_origin TEXT, -- The remote HS the media came from.
+ media_id TEXT, -- The id used to refer to the media.
+ thumbnail_width INTEGER, -- The width of the thumbnail in pixels.
+ thumbnail_height INTEGER, -- The height of the thumbnail in pixels.
+ thumbnail_type TEXT, -- The MIME-type of the thumbnail.
+ thumbnail_length INTEGER, -- The length of the thumbnail in bytes.
+ filesystem_id TEXT, -- The name used to store the media on disk.
+ CONSTRAINT uniqueness UNIQUE (
+ media_origin, media_id, thumbnail_width, thumbnail_height,
+ thumbnail_type
+ )
+);
+
+CREATE INDEX IF NOT EXISTS remote_media_cache_thumbnails_media_id
+ ON local_media_repository_thumbnails (media_id);
|
