From ff7c2e41de2056ab959a2d560c89d397425c61be Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Thu, 23 Jul 2015 14:12:49 +0100 Subject: Always return a thumbnail of the requested size. Before, we returned a thumbnail that was at least as big (if possible) as the requested size. Now, if we don't have a thumbnail of the given size we generate (and persist) one of that size. --- synapse/rest/media/v1/base_resource.py | 83 +++++++++++++++++++++++++++++ synapse/rest/media/v1/thumbnail_resource.py | 81 +++++++++++++++++++++++++++- 2 files changed, 162 insertions(+), 2 deletions(-) (limited to 'synapse') diff --git a/synapse/rest/media/v1/base_resource.py b/synapse/rest/media/v1/base_resource.py index c43ae0314b..00668b3862 100644 --- a/synapse/rest/media/v1/base_resource.py +++ b/synapse/rest/media/v1/base_resource.py @@ -225,6 +225,89 @@ class BaseMediaResource(Resource): else: return () + @defer.inlineCallbacks + def _generate_local_exact_thumbnail(self, media_id, t_width, t_height, + t_method, t_type): + input_path = self.filepaths.local_media_filepath(media_id) + + def thumbnail(): + thumbnailer = Thumbnailer(input_path) + m_width = thumbnailer.width + m_height = thumbnailer.height + + if m_width * m_height >= self.max_image_pixels: + logger.info( + "Image too large to thumbnail %r x %r > %r", + m_width, m_height, self.max_image_pixels + ) + return + + t_path = self.filepaths.local_media_thumbnail( + media_id, t_width, t_height, t_type, t_method + ) + self._makedirs(t_path) + + if t_method == "crop": + t_len = thumbnailer.crop(t_path, t_width, t_height, t_type) + elif t_method == "scale": + t_len = thumbnailer.scale(t_path, t_width, t_height, t_type) + else: + t_len = None + + return t_len, t_path + + res = yield threads.deferToThread(thumbnail) + + if res: + t_len, t_path = res + yield self.store.store_local_thumbnail( + media_id, t_width, t_height, t_type, t_method, t_len + ) + + defer.returnValue(t_path) + + @defer.inlineCallbacks + def _generate_remote_exact_thumbnail(self, server_name, file_id, media_id, + t_width, t_height, t_method, t_type): + input_path = self.filepaths.remote_media_filepath(server_name, file_id) + + def thumbnail(): + thumbnailer = Thumbnailer(input_path) + m_width = thumbnailer.width + m_height = thumbnailer.height + + if m_width * m_height >= self.max_image_pixels: + logger.info( + "Image too large to thumbnail %r x %r > %r", + m_width, m_height, self.max_image_pixels + ) + return + + t_path = self.filepaths.remote_media_thumbnail( + media_id, t_width, t_height, t_type, t_method + ) + self._makedirs(t_path) + + if t_method == "crop": + t_len = thumbnailer.crop(t_path, t_width, t_height, t_type) + elif t_method == "scale": + t_len = thumbnailer.scale(t_path, t_width, t_height, t_type) + else: + t_len = None + + return t_path, t_len + + res = yield threads.deferToThread(thumbnail) + + if res: + t_path, t_len = res + yield self.store.store_remote_media_thumbnail( + server_name, media_id, file_id, + t_width, t_height, t_type, t_method, t_len + ) + + defer.returnValue(t_path) + @defer.inlineCallbacks def _generate_local_thumbnails(self, media_id, media_info): media_type = media_info["media_type"] diff --git a/synapse/rest/media/v1/thumbnail_resource.py b/synapse/rest/media/v1/thumbnail_resource.py index 61f88e486e..58621f45df 100644 --- a/synapse/rest/media/v1/thumbnail_resource.py +++ b/synapse/rest/media/v1/thumbnail_resource.py @@ -43,11 +43,11 @@ class ThumbnailResource(BaseMediaResource): m_type = parse_string(request, "type", "image/png") if server_name == self.server_name: - yield self._respond_local_thumbnail( + yield self._select_or_generate_local_thumbnail( request, media_id, width, height, method, m_type ) else: - yield self._respond_remote_thumbnail( + yield self._select_or_generate_remote_thumbnail( request, server_name, media_id, width, height, method, m_type ) @@ -82,6 +82,83 @@ class ThumbnailResource(BaseMediaResource): request, media_info, width, height, method, m_type, ) + @defer.inlineCallbacks + def _select_or_generate_local_thumbnail(self, request, media_id, desired_width, + desired_height, desired_method, + desired_type): + media_info = yield self.store.get_local_media(media_id) + + if not media_info: + self._respond_404(request) + return + + thumbnail_infos = yield self.store.get_local_media_thumbnails(media_id) + for info in thumbnail_infos: + t_w = info["thumbnail_width"] == desired_width + t_h = info["thumbnail_height"] == desired_height + t_method = info["thumbnail_method"] == desired_method + t_type = info["thumbnail_type"] == desired_type + + if t_w and t_h and t_method and t_type: + file_path = self.filepaths.local_media_thumbnail( + media_id, desired_width, desired_height, desired_type, desired_method, + ) + yield self._respond_with_file(request, desired_type, file_path) + return + + logger.debug("We don't have a local thumbnail of that size. Generating") + + # Okay, so we generate one. + file_path = yield self._generate_local_exact_thumbnail( + media_id, desired_width, desired_height, desired_method, desired_type + ) + + if file_path: + yield self._respond_with_file(request, desired_type, file_path) + else: + yield self._respond_default_thumbnail( + request, media_info, desired_width, desired_height, + desired_method, desired_type, + ) + + @defer.inlineCallbacks + def _select_or_generate_remote_thumbnail(self, request, server_name, media_id, + desired_width, desired_height, + desired_method, desired_type): + media_info = yield self._get_remote_media(server_name, media_id) + + thumbnail_infos = yield self.store.get_remote_media_thumbnails( + server_name, media_id, + ) + + for info in thumbnail_infos: + t_w = info["thumbnail_width"] == desired_width + t_h = info["thumbnail_height"] == desired_height + t_method = info["thumbnail_method"] == desired_method + t_type = info["thumbnail_type"] == desired_type + + if t_w and t_h and t_method and t_type: + file_path = self.filepaths.remote_media_thumbnail( + media_id, desired_width, desired_height, desired_type, desired_method, + ) + yield self._respond_with_file(request, desired_type, file_path) + + logger.debug("We don't have a local thumbnail of that size. Generating") + + # Okay, so we generate one. + path = yield self._generate_remote_exact_thumbnail( + server_name, media_id, desired_width, desired_height, + desired_method, desired_type + ) + + if path: + yield self._respond_with_file(request, t_type, file_path) + else: + yield self._respond_default_thumbnail( + request, media_info, desired_width, desired_height, + desired_method, desired_type, + ) + @defer.inlineCallbacks def _respond_remote_thumbnail(self, request, server_name, media_id, width, height, method, m_type): -- cgit 1.5.1 From 33d83f36158a98111c3dceb605a40d962a9e5812 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Thu, 23 Jul 2015 14:24:21 +0100 Subject: Fix remote thumbnailing --- synapse/rest/media/v1/base_resource.py | 2 +- synapse/rest/media/v1/thumbnail_resource.py | 16 ++++++++++------ 2 files changed, 11 insertions(+), 7 deletions(-) (limited to 'synapse') diff --git a/synapse/rest/media/v1/base_resource.py b/synapse/rest/media/v1/base_resource.py index 00668b3862..74c0cd093c 100644 --- a/synapse/rest/media/v1/base_resource.py +++ b/synapse/rest/media/v1/base_resource.py @@ -284,7 +284,7 @@ class BaseMediaResource(Resource): return t_path = self.filepaths.remote_media_thumbnail( - media_id, t_width, t_height, t_type, t_method + server_name, file_id, t_width, t_height, t_type, t_method ) self._makedirs(t_path) diff --git a/synapse/rest/media/v1/thumbnail_resource.py b/synapse/rest/media/v1/thumbnail_resource.py index 58621f45df..9387258a7a 100644 --- a/synapse/rest/media/v1/thumbnail_resource.py +++ b/synapse/rest/media/v1/thumbnail_resource.py @@ -131,6 +131,8 @@ class ThumbnailResource(BaseMediaResource): server_name, media_id, ) + file_id = media_info["filesystem_id"] + for info in thumbnail_infos: t_w = info["thumbnail_width"] == desired_width t_h = info["thumbnail_height"] == desired_height @@ -139,20 +141,22 @@ class ThumbnailResource(BaseMediaResource): if t_w and t_h and t_method and t_type: file_path = self.filepaths.remote_media_thumbnail( - media_id, desired_width, desired_height, desired_type, desired_method, + server_name, file_id, desired_width, desired_height, + desired_type, desired_method, ) yield self._respond_with_file(request, desired_type, file_path) + return logger.debug("We don't have a local thumbnail of that size. Generating") # Okay, so we generate one. - path = yield self._generate_remote_exact_thumbnail( - server_name, media_id, desired_width, desired_height, - desired_method, desired_type + file_path = yield self._generate_remote_exact_thumbnail( + server_name, file_id, media_id, desired_width, + desired_height, desired_method, desired_type ) - if path: - yield self._respond_with_file(request, t_type, file_path) + if file_path: + yield self._respond_with_file(request, desired_type, file_path) else: yield self._respond_default_thumbnail( request, media_info, desired_width, desired_height, -- cgit 1.5.1 From 459085184ce80c67584bee4e5d3bc43add99bb0b Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Thu, 23 Jul 2015 15:59:32 +0100 Subject: Factor out thumbnail() --- synapse/rest/media/v1/base_resource.py | 96 ++++++++++++++-------------------- 1 file changed, 40 insertions(+), 56 deletions(-) (limited to 'synapse') diff --git a/synapse/rest/media/v1/base_resource.py b/synapse/rest/media/v1/base_resource.py index 74c0cd093c..093ba847d3 100644 --- a/synapse/rest/media/v1/base_resource.py +++ b/synapse/rest/media/v1/base_resource.py @@ -225,41 +225,44 @@ class BaseMediaResource(Resource): else: return () - @defer.inlineCallbacks - def _generate_local_exact_thumbnail(self, media_id, t_width, t_height, - t_method, t_type): - input_path = self.filepaths.local_media_filepath(media_id) + def _generate_thumbnail(self, input_path, t_path, t_width, t_height, + t_method, t_type): + thumbnailer = Thumbnailer(input_path) + m_width = thumbnailer.width + m_height = thumbnailer.height - def thumbnail(): - thumbnailer = Thumbnailer(input_path) - m_width = thumbnailer.width - m_height = thumbnailer.height + if m_width * m_height >= self.max_image_pixels: + logger.info( + "Image too large to thumbnail %r x %r > %r", + m_width, m_height, self.max_image_pixels + ) + return - if m_width * m_height >= self.max_image_pixels: - logger.info( - "Image too large to thumbnail %r x %r > %r", - m_width, m_height, self.max_image_pixels - ) - return + if t_method == "crop": + t_len = thumbnailer.crop(t_path, t_width, t_height, t_type) + elif t_method == "scale": + t_len = thumbnailer.scale(t_path, t_width, t_height, t_type) + else: + t_len = None - t_path = self.filepaths.local_media_thumbnail( - media_id, t_width, t_height, t_type, t_method - ) - self._makedirs(t_path) + return t_len - if t_method == "crop": - t_len = thumbnailer.crop(t_path, t_width, t_height, t_type) - elif t_method == "scale": - t_len = thumbnailer.scale(t_path, t_width, t_height, t_type) - else: - t_len = None + @defer.inlineCallbacks + def _generate_local_exact_thumbnail(self, media_id, t_width, t_height, + t_method, t_type): + input_path = self.filepaths.local_media_filepath(media_id) - return t_len, t_path + t_path = self.filepaths.local_media_thumbnail( + media_id, t_width, t_height, t_type, t_method + ) + self._makedirs(t_path) - res = yield threads.deferToThread(thumbnail) + t_len = yield threads.deferToThread( + self._generate_thumbnail, + input_path, t_path, t_width, t_height, t_method, t_type + ) - if res: - t_len, t_path = res + if t_len: yield self.store.store_local_thumbnail( media_id, t_width, t_height, t_type, t_method, t_len ) @@ -271,36 +274,17 @@ class BaseMediaResource(Resource): t_width, t_height, t_method, t_type): input_path = self.filepaths.remote_media_filepath(server_name, file_id) - def thumbnail(): - thumbnailer = Thumbnailer(input_path) - m_width = thumbnailer.width - m_height = thumbnailer.height - - if m_width * m_height >= self.max_image_pixels: - logger.info( - "Image too large to thumbnail %r x %r > %r", - m_width, m_height, self.max_image_pixels - ) - return - - t_path = self.filepaths.remote_media_thumbnail( - server_name, file_id, t_width, t_height, t_type, t_method - ) - self._makedirs(t_path) - - if t_method == "crop": - t_len = thumbnailer.crop(t_path, t_width, t_height, t_type) - elif t_method == "scale": - t_len = thumbnailer.scale(t_path, t_width, t_height, t_type) - else: - t_len = None - - return t_path, t_len + t_path = self.filepaths.remote_media_thumbnail( + server_name, file_id, t_width, t_height, t_type, t_method + ) + self._makedirs(t_path) - res = yield threads.deferToThread(thumbnail) + t_len = yield threads.deferToThread( + self._generate_thumbnail, + input_path, t_path, t_width, t_height, t_method, t_type + ) - if res: - t_path, t_len = res + if t_len: yield self.store.store_remote_media_thumbnail( server_name, media_id, file_id, t_width, t_height, t_type, t_method, t_len -- cgit 1.5.1 From c77048e12f032842cebbb0f1a0639bb62db88418 Mon Sep 17 00:00:00 2001 From: David Baker Date: Tue, 4 Aug 2015 14:37:09 +0100 Subject: Add endpoint that proxies ID server request token and errors if the given email is in use on this Home Server. --- synapse/api/errors.py | 1 + synapse/handlers/identity.py | 25 ++++++++++++++++++++ synapse/rest/client/v2_alpha/register.py | 27 +++++++++++++++++++++- .../schema/delta/22/user_threepids_unique.sql | 19 +++++++++++++++ 4 files changed, 71 insertions(+), 1 deletion(-) create mode 100644 synapse/storage/schema/delta/22/user_threepids_unique.sql (limited to 'synapse') diff --git a/synapse/api/errors.py b/synapse/api/errors.py index 0b3320e62c..c3b4d971a8 100644 --- a/synapse/api/errors.py +++ b/synapse/api/errors.py @@ -40,6 +40,7 @@ class Codes(object): TOO_LARGE = "M_TOO_LARGE" EXCLUSIVE = "M_EXCLUSIVE" THREEPID_AUTH_FAILED = "M_THREEPID_AUTH_FAILED" + THREEPID_IN_USE = "THREEPID_IN_USE" class CodeMessageException(RuntimeError): diff --git a/synapse/handlers/identity.py b/synapse/handlers/identity.py index c1095708a0..2a99921d5f 100644 --- a/synapse/handlers/identity.py +++ b/synapse/handlers/identity.py @@ -117,3 +117,28 @@ class IdentityHandler(BaseHandler): except CodeMessageException as e: data = json.loads(e.msg) defer.returnValue(data) + + @defer.inlineCallbacks + def requestEmailToken(self, id_server, email, client_secret, send_attempt, **kwargs): + yield run_on_reactor() + http_client = SimpleHttpClient(self.hs) + + params = { + 'email': email, + 'client_secret': client_secret, + 'send_attempt': send_attempt, + } + params.update(kwargs) + + try: + data = yield http_client.post_urlencoded_get_json( + "https://%s%s" % ( + id_server, + "/_matrix/identity/api/v1/validate/email/requestToken" + ), + params + ) + defer.returnValue(data) + except CodeMessageException as e: + logger.info("Proxied requestToken failed: %r", e) + raise e diff --git a/synapse/rest/client/v2_alpha/register.py b/synapse/rest/client/v2_alpha/register.py index b5926f9ca6..7b97a73df6 100644 --- a/synapse/rest/client/v2_alpha/register.py +++ b/synapse/rest/client/v2_alpha/register.py @@ -41,7 +41,7 @@ logger = logging.getLogger(__name__) class RegisterRestServlet(RestServlet): - PATTERN = client_v2_pattern("/register") + PATTERN = client_v2_pattern("/register*") def __init__(self, hs): super(RegisterRestServlet, self).__init__() @@ -55,6 +55,11 @@ class RegisterRestServlet(RestServlet): @defer.inlineCallbacks def on_POST(self, request): yield run_on_reactor() + + if '/register/email/requestToken' in request.path: + ret = yield self.onEmailTokenRequest(request) + defer.returnValue(ret) + body = parse_json_dict_from_request(request) # we do basic sanity checks here because the auth layer will store these @@ -209,6 +214,26 @@ class RegisterRestServlet(RestServlet): "home_server": self.hs.hostname, } + @defer.inlineCallbacks + def onEmailTokenRequest(self, request): + body = parse_json_dict_from_request(request) + + required = ['id_server', 'client_secret', 'email', 'send_attempt'] + absent = [] + for k in required: + if k not in body: + absent.append(k) + + existingUid = self.hs.get_datastore().get_user_id_by_threepid('email', body['email']) + if existingUid is not None: + raise SynapseError(400, "Email is already in use", Codes.THREEPID_IN_USE) + + if len(absent) > 0: + raise SynapseError(400, "Missing params: %r" % absent, Codes.MISSING_PARAM) + + ret = yield self.identity_handler.requestEmailToken(**body) + defer.returnValue((200, ret)) + def register_servlets(hs, http_server): RegisterRestServlet(hs).register(http_server) diff --git a/synapse/storage/schema/delta/22/user_threepids_unique.sql b/synapse/storage/schema/delta/22/user_threepids_unique.sql new file mode 100644 index 0000000000..87edfa454c --- /dev/null +++ b/synapse/storage/schema/delta/22/user_threepids_unique.sql @@ -0,0 +1,19 @@ +CREATE TABLE IF NOT EXISTS user_threepids2 ( + user_id TEXT NOT NULL, + medium TEXT NOT NULL, + address TEXT NOT NULL, + validated_at BIGINT NOT NULL, + added_at BIGINT NOT NULL, + CONSTRAINT medium_address UNIQUE (medium, address) +); + +INSERT INTO user_threepids2 + SELECT * FROM user_threepids WHERE added_at IN ( + SELECT max(added_at) FROM user_threepids GROUP BY medium, address + ) +; + +DROP TABLE user_threepids; +ALTER TABLE user_threepids2 RENAME TO user_threepids; + +CREATE INDEX user_threepids_user_id ON user_threepids(user_id); -- cgit 1.5.1 From e1241285420556b8afb16aacb3eeb8f29e9859f0 Mon Sep 17 00:00:00 2001 From: David Baker Date: Tue, 4 Aug 2015 14:50:31 +0100 Subject: Bump schema version --- synapse/storage/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'synapse') diff --git a/synapse/storage/__init__.py b/synapse/storage/__init__.py index 71d5d92500..99467dde02 100644 --- a/synapse/storage/__init__.py +++ b/synapse/storage/__init__.py @@ -54,7 +54,7 @@ logger = logging.getLogger(__name__) # Remember to update this number every time a change is made to database # schema files, so the users will be informed on server restarts. -SCHEMA_VERSION = 21 +SCHEMA_VERSION = 22 dir_path = os.path.abspath(os.path.dirname(__file__)) -- cgit 1.5.1 From 883aabe4236f466b404bae7982dcb3b375dc53e1 Mon Sep 17 00:00:00 2001 From: David Baker Date: Tue, 4 Aug 2015 15:20:35 +0100 Subject: splt long line --- synapse/rest/client/v2_alpha/register.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) (limited to 'synapse') diff --git a/synapse/rest/client/v2_alpha/register.py b/synapse/rest/client/v2_alpha/register.py index 7b97a73df6..93f9223708 100644 --- a/synapse/rest/client/v2_alpha/register.py +++ b/synapse/rest/client/v2_alpha/register.py @@ -224,7 +224,9 @@ class RegisterRestServlet(RestServlet): if k not in body: absent.append(k) - existingUid = self.hs.get_datastore().get_user_id_by_threepid('email', body['email']) + existingUid = self.hs.get_datastore().get_user_id_by_threepid( + 'email', body['email'] + ) if existingUid is not None: raise SynapseError(400, "Email is already in use", Codes.THREEPID_IN_USE) -- cgit 1.5.1 From a0dea6eaed2315bff018f86820ed7a866ab0d2ef Mon Sep 17 00:00:00 2001 From: David Baker Date: Tue, 4 Aug 2015 16:18:17 +0100 Subject: Remember to yield: not much point testing is a deferred is not None --- synapse/rest/client/v2_alpha/register.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) (limited to 'synapse') diff --git a/synapse/rest/client/v2_alpha/register.py b/synapse/rest/client/v2_alpha/register.py index 93f9223708..25dab6f9c0 100644 --- a/synapse/rest/client/v2_alpha/register.py +++ b/synapse/rest/client/v2_alpha/register.py @@ -224,9 +224,10 @@ class RegisterRestServlet(RestServlet): if k not in body: absent.append(k) - existingUid = self.hs.get_datastore().get_user_id_by_threepid( + existingUid = yield self.hs.get_datastore().get_user_id_by_threepid( 'email', body['email'] ) + if existingUid is not None: raise SynapseError(400, "Email is already in use", Codes.THREEPID_IN_USE) -- cgit 1.5.1 From 185ac7ee6cde22b9b491ac97013b029071ec9d53 Mon Sep 17 00:00:00 2001 From: David Baker Date: Tue, 4 Aug 2015 16:29:54 +0100 Subject: Allow sign in using email address --- synapse/rest/client/v1/login.py | 17 ++++++++++++----- 1 file changed, 12 insertions(+), 5 deletions(-) (limited to 'synapse') diff --git a/synapse/rest/client/v1/login.py b/synapse/rest/client/v1/login.py index 998d4d44c6..8ce3364820 100644 --- a/synapse/rest/client/v1/login.py +++ b/synapse/rest/client/v1/login.py @@ -74,17 +74,24 @@ class LoginRestServlet(ClientV1RestServlet): @defer.inlineCallbacks def do_password_login(self, login_submission): - if not login_submission["user"].startswith('@'): - login_submission["user"] = UserID.create( - login_submission["user"], self.hs.hostname).to_string() + if 'medium' in login_submission and 'address' in login_submission: + user_id = yield self.hs.get_datastore().get_user_id_by_threepid( + login_submission['medium'], login_submission['address'] + ) + else: + user_id = login_submission['user'] + + if not user_id.startswith('@'): + user_id = UserID.create( + user_id, self.hs.hostname).to_string() handler = self.handlers.login_handler token = yield handler.login( - user=login_submission["user"], + user=user_id, password=login_submission["password"]) result = { - "user_id": login_submission["user"], # may have changed + "user_id": user_id, # may have changed "access_token": token, "home_server": self.hs.hostname, } -- cgit 1.5.1 From 7e3d1c7d92157a3cce8ed975f2a982a6a80693d0 Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Wed, 12 Aug 2015 10:54:38 +0100 Subject: Make a config option for whether to generate new thumbnail sizes dynamically --- synapse/config/repository.py | 8 ++++++++ synapse/rest/media/v1/base_resource.py | 1 + synapse/rest/media/v1/thumbnail_resource.py | 25 ++++++++++++++++++------- 3 files changed, 27 insertions(+), 7 deletions(-) (limited to 'synapse') diff --git a/synapse/config/repository.py b/synapse/config/repository.py index 6891abd71d..748dd14e23 100644 --- a/synapse/config/repository.py +++ b/synapse/config/repository.py @@ -22,6 +22,7 @@ class ContentRepositoryConfig(Config): self.max_image_pixels = self.parse_size(config["max_image_pixels"]) self.media_store_path = self.ensure_directory(config["media_store_path"]) self.uploads_path = self.ensure_directory(config["uploads_path"]) + self.dynamic_thumbnails = config["dynamic_thumbnails"] def default_config(self, config_dir_path, server_name): media_store = self.default_path("media_store") @@ -38,4 +39,11 @@ class ContentRepositoryConfig(Config): # Maximum number of pixels that will be thumbnailed max_image_pixels: "32M" + + # Whether to generate new thumbnails on the fly to precisely match + # the resolution requested by the client. If true then whenever + # a new resolution is requested by the client the server will + # generate a new thumbnail. If false the server will pick a thumbnail + # from a precalcualted list. + dynamic_thumbnails: false """ % locals() diff --git a/synapse/rest/media/v1/base_resource.py b/synapse/rest/media/v1/base_resource.py index 093ba847d3..e39729489e 100644 --- a/synapse/rest/media/v1/base_resource.py +++ b/synapse/rest/media/v1/base_resource.py @@ -69,6 +69,7 @@ class BaseMediaResource(Resource): self.filepaths = filepaths self.version_string = hs.version_string self.downloads = {} + self.dynamic_thumbnails = hs.config.dynamic_thumbnails def _respond_404(self, request): respond_with_json( diff --git a/synapse/rest/media/v1/thumbnail_resource.py b/synapse/rest/media/v1/thumbnail_resource.py index 9387258a7a..e506dad934 100644 --- a/synapse/rest/media/v1/thumbnail_resource.py +++ b/synapse/rest/media/v1/thumbnail_resource.py @@ -43,14 +43,25 @@ class ThumbnailResource(BaseMediaResource): m_type = parse_string(request, "type", "image/png") if server_name == self.server_name: - yield self._select_or_generate_local_thumbnail( - request, media_id, width, height, method, m_type - ) + if self.dynamic_thumbnails: + yield self._select_or_generate_local_thumbnail( + request, media_id, width, height, method, m_type + ) + else: + yield self._respond_local_thumbnail( + request, media_id, width, height, method, m_type + ) else: - yield self._select_or_generate_remote_thumbnail( - request, server_name, media_id, - width, height, method, m_type - ) + if self.dynamic_thumbnails: + yield self._select_or_generate_remote_thumbnail( + request, server_name, media_id, + width, height, method, m_type + ) + else: + yield self._respond_remote_thumbnail( + request, server_name, media_id, + width, height, method, m_type + ) @defer.inlineCallbacks def _respond_local_thumbnail(self, request, media_id, width, height, -- cgit 1.5.1 From fdb724cb7040a7746b2a6c6d8aabbf3654daf8dd Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Wed, 12 Aug 2015 10:55:27 +0100 Subject: Add config option for setting the list of thumbnail sizes to precalculate --- synapse/config/repository.py | 39 ++++++++++++++++++++++++++++++++++ synapse/rest/media/v1/base_resource.py | 18 ++-------------- 2 files changed, 41 insertions(+), 16 deletions(-) (limited to 'synapse') diff --git a/synapse/config/repository.py b/synapse/config/repository.py index 748dd14e23..7cab874422 100644 --- a/synapse/config/repository.py +++ b/synapse/config/repository.py @@ -14,6 +14,27 @@ # limitations under the License. from ._base import Config +from collections import namedtuple + +ThumbnailRequirement = namedtuple( + "ThumbnailRequirement", ["width", "height", "method", "media_type"] +) + +def parse_thumbnail_requirements(thumbnail_sizes): + requirements = {} + for size in thumbnail_sizes: + width = size["width"] + height = size["height"] + method = size["method"] + jpeg_thumbnail = ThumbnailRequirement(width, height, method, "image/jpeg") + png_thumbnail = ThumbnailRequirement(width, height, method, "image/png") + requirements.setdefault("image/jpeg", []).append(jpeg_thumbnail) + requirements.setdefault("image/gif", []).append(png_thumbnail) + requirements.setdefault("image/png", []).append(png_thumbnail) + return { + media_type: tuple(thumbnails) + for media_type, thumbnails in requirements.items() + } class ContentRepositoryConfig(Config): @@ -23,6 +44,9 @@ class ContentRepositoryConfig(Config): self.media_store_path = self.ensure_directory(config["media_store_path"]) self.uploads_path = self.ensure_directory(config["uploads_path"]) self.dynamic_thumbnails = config["dynamic_thumbnails"] + self.thumbnail_requirements = parse_thumbnail_requirements( + config["thumbnail_sizes"] + ) def default_config(self, config_dir_path, server_name): media_store = self.default_path("media_store") @@ -46,4 +70,19 @@ class ContentRepositoryConfig(Config): # generate a new thumbnail. If false the server will pick a thumbnail # from a precalcualted list. dynamic_thumbnails: false + + # List of thumbnail to precalculate when an image is uploaded. + thumbnail_sizes: + - width: 32 + height: 32 + method: crop + - width: 96 + height: 96 + method: crop + - width: 320 + height: 240 + method: scale + - width: 640 + height: 480 + method: scale """ % locals() diff --git a/synapse/rest/media/v1/base_resource.py b/synapse/rest/media/v1/base_resource.py index e39729489e..271cbca2d6 100644 --- a/synapse/rest/media/v1/base_resource.py +++ b/synapse/rest/media/v1/base_resource.py @@ -70,6 +70,7 @@ class BaseMediaResource(Resource): self.version_string = hs.version_string self.downloads = {} self.dynamic_thumbnails = hs.config.dynamic_thumbnails + self.thumbnail_requirements = hs.config.thumbnail_requirements def _respond_404(self, request): respond_with_json( @@ -209,22 +210,7 @@ class BaseMediaResource(Resource): self._respond_404(request) def _get_thumbnail_requirements(self, media_type): - if media_type == "image/jpeg": - return ( - (32, 32, "crop", "image/jpeg"), - (96, 96, "crop", "image/jpeg"), - (320, 240, "scale", "image/jpeg"), - (640, 480, "scale", "image/jpeg"), - ) - elif (media_type == "image/png") or (media_type == "image/gif"): - return ( - (32, 32, "crop", "image/png"), - (96, 96, "crop", "image/png"), - (320, 240, "scale", "image/png"), - (640, 480, "scale", "image/png"), - ) - else: - return () + return self.thumbnail_requirements.get(media_type, ()) def _generate_thumbnail(self, input_path, t_path, t_width, t_height, t_method, t_type): -- cgit 1.5.1 From de3b7b55d68ebad3d535dad2643e747460dc22a3 Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Wed, 12 Aug 2015 14:29:17 +0100 Subject: Doc-string for config ultility function --- synapse/config/repository.py | 11 +++++++++++ 1 file changed, 11 insertions(+) (limited to 'synapse') diff --git a/synapse/config/repository.py b/synapse/config/repository.py index 7cab874422..2cb0812d7d 100644 --- a/synapse/config/repository.py +++ b/synapse/config/repository.py @@ -21,6 +21,17 @@ ThumbnailRequirement = namedtuple( ) def parse_thumbnail_requirements(thumbnail_sizes): + """ Takes a list of dictionaries with "width", "height", and "method" keys + and creates a map from image media types to the thumbnail size, thumnailing + method, and thumbnail media type to precalculate + + Args: + thumbnail_sizes(list): List of dicts with "width", "height", and + "method" keys + Returns: + Dictionary mapping from media type string to list of + ThumbnailRequirement tuples. + """ requirements = {} for size in thumbnail_sizes: width = size["width"] -- cgit 1.5.1 From 73605f80705f4a0c7271cabfee40a47c7764d72d Mon Sep 17 00:00:00 2001 From: David Baker Date: Wed, 12 Aug 2015 15:40:54 +0100 Subject: Just leaving off the $ is fine. r* == registerrrrrrrrr --- synapse/rest/client/v2_alpha/register.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'synapse') diff --git a/synapse/rest/client/v2_alpha/register.py b/synapse/rest/client/v2_alpha/register.py index 25dab6f9c0..e6ad35aa16 100644 --- a/synapse/rest/client/v2_alpha/register.py +++ b/synapse/rest/client/v2_alpha/register.py @@ -41,7 +41,7 @@ logger = logging.getLogger(__name__) class RegisterRestServlet(RestServlet): - PATTERN = client_v2_pattern("/register*") + PATTERN = client_v2_pattern("/register") def __init__(self, hs): super(RegisterRestServlet, self).__init__() -- cgit 1.5.1 From f43041aacd36b7d9052476bcb0d083ea4213a9f9 Mon Sep 17 00:00:00 2001 From: David Baker Date: Wed, 12 Aug 2015 15:44:08 +0100 Subject: Check absent before trying to access keys --- synapse/rest/client/v2_alpha/register.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) (limited to 'synapse') diff --git a/synapse/rest/client/v2_alpha/register.py b/synapse/rest/client/v2_alpha/register.py index e6ad35aa16..254c5f1ddf 100644 --- a/synapse/rest/client/v2_alpha/register.py +++ b/synapse/rest/client/v2_alpha/register.py @@ -224,6 +224,9 @@ class RegisterRestServlet(RestServlet): if k not in body: absent.append(k) + if len(absent) > 0: + raise SynapseError(400, "Missing params: %r" % absent, Codes.MISSING_PARAM) + existingUid = yield self.hs.get_datastore().get_user_id_by_threepid( 'email', body['email'] ) @@ -231,9 +234,6 @@ class RegisterRestServlet(RestServlet): if existingUid is not None: raise SynapseError(400, "Email is already in use", Codes.THREEPID_IN_USE) - if len(absent) > 0: - raise SynapseError(400, "Missing params: %r" % absent, Codes.MISSING_PARAM) - ret = yield self.identity_handler.requestEmailToken(**body) defer.returnValue((200, ret)) -- cgit 1.5.1 From 95b0f5449d2b7539a7817b231090d607e55d6ac7 Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Thu, 13 Aug 2015 17:34:22 +0100 Subject: Fix flake8 warning --- synapse/config/repository.py | 1 + 1 file changed, 1 insertion(+) (limited to 'synapse') diff --git a/synapse/config/repository.py b/synapse/config/repository.py index 2cb0812d7d..64644b9a7a 100644 --- a/synapse/config/repository.py +++ b/synapse/config/repository.py @@ -20,6 +20,7 @@ ThumbnailRequirement = namedtuple( "ThumbnailRequirement", ["width", "height", "method", "media_type"] ) + def parse_thumbnail_requirements(thumbnail_sizes): """ Takes a list of dictionaries with "width", "height", and "method" keys and creates a map from image media types to the thumbnail size, thumnailing -- cgit 1.5.1 From a45ec7c651abdbe54bba4a8fe420e02c9059f390 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Wed, 19 Aug 2015 10:08:12 +0100 Subject: Block on storing the current last_tokens --- synapse/push/__init__.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) (limited to 'synapse') diff --git a/synapse/push/__init__.py b/synapse/push/__init__.py index 36f450c31d..483deb624c 100644 --- a/synapse/push/__init__.py +++ b/synapse/push/__init__.py @@ -345,7 +345,7 @@ class Pusher(object): if processed: self.backoff_delay = Pusher.INITIAL_BACKOFF self.last_token = chunk['end'] - self.store.update_pusher_last_token_and_success( + yield self.store.update_pusher_last_token_and_success( self.app_id, self.pushkey, self.user_name, @@ -354,7 +354,7 @@ class Pusher(object): ) if self.failing_since: self.failing_since = None - self.store.update_pusher_failing_since( + yield self.store.update_pusher_failing_since( self.app_id, self.pushkey, self.user_name, @@ -362,7 +362,7 @@ class Pusher(object): else: if not self.failing_since: self.failing_since = self.clock.time_msec() - self.store.update_pusher_failing_since( + yield self.store.update_pusher_failing_since( self.app_id, self.pushkey, self.user_name, @@ -380,7 +380,7 @@ class Pusher(object): self.user_name, self.pushkey) self.backoff_delay = Pusher.INITIAL_BACKOFF self.last_token = chunk['end'] - self.store.update_pusher_last_token( + yield self.store.update_pusher_last_token( self.app_id, self.pushkey, self.user_name, @@ -388,7 +388,7 @@ class Pusher(object): ) self.failing_since = None - self.store.update_pusher_failing_since( + yield self.store.update_pusher_failing_since( self.app_id, self.pushkey, self.user_name, -- cgit 1.5.1 From 78fa346b0781cf42ef8772638f9f8abb26b9a36f Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Wed, 19 Aug 2015 10:08:31 +0100 Subject: Store the 'last_token' in the db, even if we processed no events --- synapse/push/__init__.py | 6 ++++++ 1 file changed, 6 insertions(+) (limited to 'synapse') diff --git a/synapse/push/__init__.py b/synapse/push/__init__.py index 483deb624c..13002e0db4 100644 --- a/synapse/push/__init__.py +++ b/synapse/push/__init__.py @@ -294,6 +294,12 @@ class Pusher(object): if not single_event: self.last_token = chunk['end'] logger.debug("Event stream timeout for pushkey %s", self.pushkey) + yield self.store.update_pusher_last_token( + self.app_id, + self.pushkey, + self.user_name, + self.last_token + ) return if not self.alive: -- cgit 1.5.1 From d7272f8d9d0ce3ac9a4095969453efef5aecce40 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Wed, 19 Aug 2015 12:03:09 +0100 Subject: Add canonical alias to the default power levels --- synapse/api/constants.py | 1 + synapse/handlers/room.py | 1 + 2 files changed, 2 insertions(+) (limited to 'synapse') diff --git a/synapse/api/constants.py b/synapse/api/constants.py index 7156ee4e7d..60a0d336da 100644 --- a/synapse/api/constants.py +++ b/synapse/api/constants.py @@ -76,6 +76,7 @@ class EventTypes(object): Feedback = "m.room.message.feedback" RoomHistoryVisibility = "m.room.history_visibility" + CanonicalAlias = "m.room.canonical_alias" # These are used for validation Message = "m.room.message" diff --git a/synapse/handlers/room.py b/synapse/handlers/room.py index 7511d294f3..c56112a92a 100644 --- a/synapse/handlers/room.py +++ b/synapse/handlers/room.py @@ -250,6 +250,7 @@ class RoomCreationHandler(BaseHandler): EventTypes.Name: 100, EventTypes.PowerLevels: 100, EventTypes.RoomHistoryVisibility: 100, + EventTypes.CanonicalAlias: 100, }, "events_default": 0, "state_default": 50, -- cgit 1.5.1 From aadb2238c9647186711933666851def5e37a8dbf Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Thu, 20 Aug 2015 09:55:04 +0100 Subject: Check that the canonical room alias actually points to the room --- synapse/handlers/_base.py | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) (limited to 'synapse') diff --git a/synapse/handlers/_base.py b/synapse/handlers/_base.py index d6c064b398..e91f1129db 100644 --- a/synapse/handlers/_base.py +++ b/synapse/handlers/_base.py @@ -18,7 +18,7 @@ from twisted.internet import defer from synapse.api.errors import LimitExceededError, SynapseError from synapse.crypto.event_signing import add_hashes_and_signatures from synapse.api.constants import Membership, EventTypes -from synapse.types import UserID +from synapse.types import UserID, RoomAlias from synapse.util.logcontext import PreserveLoggingContext @@ -130,6 +130,22 @@ class BaseHandler(object): returned_invite.signatures ) + if event.type == EventTypes.CanonicalAlias: + # Check the alias is acually valid (at this time at least) + room_alias_str = event.content.get("alias", None) + if room_alias_str: + room_alias = RoomAlias.from_string(room_alias_str) + directory_handler = self.hs.get_handlers().directory_handler + mapping = yield directory_handler.get_association(room_alias) + + if mapping["room_id"] != event.room_id: + raise SynapseError( + 400, + "Room alias %s does not point to the room" % ( + room_alias_str, + ) + ) + destinations = set(extra_destinations) for k, s in context.current_state.items(): try: -- cgit 1.5.1 From 4cf302de5b1b4f4494a9445e0d85e4c9c24ff73d Mon Sep 17 00:00:00 2001 From: David Baker Date: Thu, 20 Aug 2015 10:31:18 +0100 Subject: Comma comma comma comma comma chameleon --- synapse/rest/client/v1/login.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'synapse') diff --git a/synapse/rest/client/v1/login.py b/synapse/rest/client/v1/login.py index cb97917608..0d5eafd0fa 100644 --- a/synapse/rest/client/v1/login.py +++ b/synapse/rest/client/v1/login.py @@ -86,7 +86,7 @@ class LoginRestServlet(ClientV1RestServlet): user_id, self.hs.hostname).to_string() token = yield self.handlers.auth_handler.login_with_password( - user_id=user_id + user_id=user_id, password=login_submission["password"]) result = { -- cgit 1.5.1 From 9d720223f25982500ba4076caef9b70b608e0292 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Thu, 20 Aug 2015 14:12:01 +0100 Subject: Bump version and changelog --- CHANGES.rst | 73 +++++++++++++++++++++++++++++++++++++++++++++++++++++ synapse/__init__.py | 2 +- 2 files changed, 74 insertions(+), 1 deletion(-) (limited to 'synapse') diff --git a/CHANGES.rst b/CHANGES.rst index 6a5fce899a..e2c6afb6ae 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -1,3 +1,76 @@ +Changes in synapse v0.10.0-rc1 (2015-08-20) +=========================================== + +Also see v0.9.4-rc1 changelog, which has been amalgamated into this release. + +General: + +* Upgrade to Twisted 15 (PR #173) +* Add support for serving and fetching encryption keys over federation. + (PR #208) +* Add support for logging in with email address (PR #234) +* Add support for new ``m.room.canonical_alias`` event. (PR #233) +* Error if a user tries to register with an email already in use. (PR #211) +* Add extra and improve existing caches (PR #212, #219, #226, #228) +* Batch various storage request (PR #226, #228) +* Fix bug where we didn't correctly log the entity that triggered the request + if the request came in via an application service (PR #230) +* Fix bug where we needlessly regenerated the full list of rooms an AS is + interested in. (PR #232) + + +Configuration: + +* Add ``--generate-keys`` that will generate any missing cert and key files in + the configuration files. This is equivalent to running ``--generate-config`` + on an existing configuration file. (PR #220) +* ``--generate-config`` now no longer requires a ``--server-name`` parameter + when used on existing configuration files. (PR #220) +* Add ``--print-pidfile`` flag that controls the printing of the pid to stdout + of the demonised process. (PR #213) + +Media Repository: + +* Fix bug where we picked a lower resolution image than requested. (PR #205) +* Add support for specifying if a the media repository should dynamically + thumbnail images or not. (PR #206) + +Metrics: + +* Add statistics from the reactor to the metrics API. (PR #224, #225) + +Demo Homeservers: + +* Fix starting the demo homeservers without rate-limiting enabled. (PR #182) +* Fix enabling registration on demo homeservers (PR #223) + + +Changes in synapse v0.9.4-rc1 (2015-07-21) +========================================== + +General: + +* Add basic implementation of receipts. (SPEC-99) +* Add support for configuration presets in room creation API. (PR #203) +* Add auth event that limits the visibility of history for new users. + (SPEC-134) +* Add SAML2 login/registration support. (PR #201. Thanks Muthu Subramanian!) +* Add client side key management APIs for end to end encryption. (PR #198) +* Change power level semantics so that you cannot kick, ban or change power + levels of users that have equal or greater power level than you. (SYN-192) +* Improve performance by bulk inserting events where possible. (PR #193) +* Improve performance by bulk verifying signatures where possible. (PR #194) + + +Configuration: + +* Add support for including TLS certificate chains. + +Media Repository: + +* Add Content-Disposition headers to content repository responses. (SYN-150) + + Changes in synapse v0.9.3 (2015-07-01) ====================================== diff --git a/synapse/__init__.py b/synapse/__init__.py index 96e37308d6..5853165a21 100644 --- a/synapse/__init__.py +++ b/synapse/__init__.py @@ -16,4 +16,4 @@ """ This is a reference implementation of a Matrix home server. """ -__version__ = "0.9.3" +__version__ = "0.10.0-rc1" -- cgit 1.5.1 From 9b63def3887779c7c9a1aeadd2d16df506155953 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Thu, 20 Aug 2015 14:35:40 +0100 Subject: Add m.room.avatar to default power levels. Change default required power levels of such events to 50 --- synapse/api/constants.py | 1 + synapse/handlers/room.py | 5 +++-- 2 files changed, 4 insertions(+), 2 deletions(-) (limited to 'synapse') diff --git a/synapse/api/constants.py b/synapse/api/constants.py index 60a0d336da..1423986c1e 100644 --- a/synapse/api/constants.py +++ b/synapse/api/constants.py @@ -77,6 +77,7 @@ class EventTypes(object): RoomHistoryVisibility = "m.room.history_visibility" CanonicalAlias = "m.room.canonical_alias" + RoomAvatar = "m.room.avatar" # These are used for validation Message = "m.room.message" diff --git a/synapse/handlers/room.py b/synapse/handlers/room.py index 8108c2763d..c5d1001b50 100644 --- a/synapse/handlers/room.py +++ b/synapse/handlers/room.py @@ -247,10 +247,11 @@ class RoomCreationHandler(BaseHandler): }, "users_default": 0, "events": { - EventTypes.Name: 100, + EventTypes.Name: 50, EventTypes.PowerLevels: 100, EventTypes.RoomHistoryVisibility: 100, - EventTypes.CanonicalAlias: 100, + EventTypes.CanonicalAlias: 50, + EventTypes.RoomAvatar: 50, }, "events_default": 0, "state_default": 50, -- cgit 1.5.1 From ca0d28ef34022874ebc9168146df53a10bcb925e Mon Sep 17 00:00:00 2001 From: David Baker Date: Thu, 20 Aug 2015 15:35:14 +0100 Subject: Another use of check_password that got missed in the yield fix --- synapse/handlers/auth.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'synapse') diff --git a/synapse/handlers/auth.py b/synapse/handlers/auth.py index be2baeaece..ff2c66f442 100644 --- a/synapse/handlers/auth.py +++ b/synapse/handlers/auth.py @@ -162,7 +162,7 @@ class AuthHandler(BaseHandler): if not user_id.startswith('@'): user_id = UserID.create(user_id, self.hs.hostname).to_string() - self._check_password(user_id, password) + yield self._check_password(user_id, password) defer.returnValue(user_id) @defer.inlineCallbacks -- cgit 1.5.1 From f764f9264734fdcf83869022fed23bdab5e4c8dc Mon Sep 17 00:00:00 2001 From: David Baker Date: Thu, 20 Aug 2015 15:35:54 +0100 Subject: Remove spurious extra arg to set_password --- synapse/rest/client/v2_alpha/account.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'synapse') diff --git a/synapse/rest/client/v2_alpha/account.py b/synapse/rest/client/v2_alpha/account.py index 897c54b539..522a312c9e 100644 --- a/synapse/rest/client/v2_alpha/account.py +++ b/synapse/rest/client/v2_alpha/account.py @@ -79,7 +79,7 @@ class PasswordRestServlet(RestServlet): new_password = params['new_password'] yield self.auth_handler.set_password( - user_id, new_password, None + user_id, new_password ) defer.returnValue((200, {})) -- cgit 1.5.1