diff options
| -rw-r--r-- | CHANGES.rst | 74 | ||||
| -rw-r--r-- | README.rst | 87 | ||||
| -rw-r--r-- | UPGRADE.rst | 33 | ||||
| -rw-r--r-- | synapse/__init__.py | 2 | ||||
| -rw-r--r-- | synapse/api/constants.py | 2 | ||||
| -rw-r--r-- | synapse/api/errors.py | 1 | ||||
| -rw-r--r-- | synapse/config/repository.py | 59 | ||||
| -rw-r--r-- | synapse/handlers/_base.py | 18 | ||||
| -rw-r--r-- | synapse/handlers/auth.py | 2 | ||||
| -rw-r--r-- | synapse/handlers/identity.py | 25 | ||||
| -rw-r--r-- | synapse/handlers/room.py | 4 | ||||
| -rw-r--r-- | synapse/push/__init__.py | 16 | ||||
| -rw-r--r-- | synapse/rest/client/v1/login.py | 15 | ||||
| -rw-r--r-- | synapse/rest/client/v2_alpha/account.py | 2 | ||||
| -rw-r--r-- | synapse/rest/client/v2_alpha/register.py | 28 | ||||
| -rw-r--r-- | synapse/rest/media/v1/base_resource.py | 82 | ||||
| -rw-r--r-- | synapse/rest/media/v1/thumbnail_resource.py | 106 | ||||
| -rw-r--r-- | synapse/storage/schema/delta/22/user_threepids_unique.sql | 19 | ||||
| -rw-r--r-- | tests/rest/client/v2_alpha/test_register.py | 3 |
19 files changed, 498 insertions, 80 deletions
diff --git a/CHANGES.rst b/CHANGES.rst index 6a5fce899a..e6d1a37307 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -1,3 +1,77 @@ +Changes in synapse v0.10.0-rc1 (2015-08-20) +=========================================== + +Also see v0.9.4-rc1 changelog, which has been amalgamated into this release. + +General: + +* Upgrade to Twisted 15 (PR #173) +* Add support for serving and fetching encryption keys over federation. + (PR #208) +* Add support for logging in with email address (PR #234) +* Add support for new ``m.room.canonical_alias`` event. (PR #233) +* Error if a user tries to register with an email already in use. (PR #211) +* Add extra and improve existing caches (PR #212, #219, #226, #228) +* Batch various storage request (PR #226, #228) +* Fix bug where we didn't correctly log the entity that triggered the request + if the request came in via an application service (PR #230) +* Fix bug where we needlessly regenerated the full list of rooms an AS is + interested in. (PR #232) +* Add support for AS's to use v2_alpha registration API (PR #210) + + +Configuration: + +* Add ``--generate-keys`` that will generate any missing cert and key files in + the configuration files. This is equivalent to running ``--generate-config`` + on an existing configuration file. (PR #220) +* ``--generate-config`` now no longer requires a ``--server-name`` parameter + when used on existing configuration files. (PR #220) +* Add ``--print-pidfile`` flag that controls the printing of the pid to stdout + of the demonised process. (PR #213) + +Media Repository: + +* Fix bug where we picked a lower resolution image than requested. (PR #205) +* Add support for specifying if a the media repository should dynamically + thumbnail images or not. (PR #206) + +Metrics: + +* Add statistics from the reactor to the metrics API. (PR #224, #225) + +Demo Homeservers: + +* Fix starting the demo homeservers without rate-limiting enabled. (PR #182) +* Fix enabling registration on demo homeservers (PR #223) + + +Changes in synapse v0.9.4-rc1 (2015-07-21) +========================================== + +General: + +* Add basic implementation of receipts. (SPEC-99) +* Add support for configuration presets in room creation API. (PR #203) +* Add auth event that limits the visibility of history for new users. + (SPEC-134) +* Add SAML2 login/registration support. (PR #201. Thanks Muthu Subramanian!) +* Add client side key management APIs for end to end encryption. (PR #198) +* Change power level semantics so that you cannot kick, ban or change power + levels of users that have equal or greater power level than you. (SYN-192) +* Improve performance by bulk inserting events where possible. (PR #193) +* Improve performance by bulk verifying signatures where possible. (PR #194) + + +Configuration: + +* Add support for including TLS certificate chains. + +Media Repository: + +* Add Content-Disposition headers to content repository responses. (SYN-150) + + Changes in synapse v0.9.3 (2015-07-01) ====================================== diff --git a/README.rst b/README.rst index 5ff53f2df7..d8d179135b 100644 --- a/README.rst +++ b/README.rst @@ -7,7 +7,7 @@ Matrix is an ambitious new ecosystem for open federated Instant Messaging and VoIP. The basics you need to know to get up and running are: - Everything in Matrix happens in a room. Rooms are distributed and do not - exist on any single server. Rooms can be located using convenience aliases + exist on any single server. Rooms can be located using convenience aliases like ``#matrix:matrix.org`` or ``#test:localhost:8448``. - Matrix user IDs look like ``@matthew:matrix.org`` (although in the future @@ -23,7 +23,7 @@ The overall architecture is:: accessed by the web client at http://matrix.org/beta or via an IRC bridge at irc://irc.freenode.net/matrix. -Synapse is currently in rapid development, but as of version 0.5 we believe it +Synapse is currently in rapid development, but as of version 0.5 we believe it is sufficiently stable to be run as an internet-facing service for real usage! About Matrix @@ -104,7 +104,7 @@ Installing prerequisites on Ubuntu or Debian:: sudo apt-get install build-essential python2.7-dev libffi-dev \ python-pip python-setuptools sqlite3 \ libssl-dev python-virtualenv libjpeg-dev - + Installing prerequisites on ArchLinux:: sudo pacman -S base-devel python2 python-pip \ @@ -115,7 +115,7 @@ Installing prerequisites on Mac OS X:: xcode-select --install sudo easy_install pip sudo pip install virtualenv - + To install the synapse homeserver run:: virtualenv -p python2.7 ~/.synapse @@ -174,13 +174,13 @@ traditionally used for convenience and simplicity. The advantages of Postgres include: - * significant performance improvements due to the superior threading and - caching model, smarter query optimiser - * allowing the DB to be run on separate hardware - * allowing basic active/backup high-availability with a "hot spare" synapse - pointing at the same DB master, as well as enabling DB replication in - synapse itself. - +* significant performance improvements due to the superior threading and + caching model, smarter query optimiser +* allowing the DB to be run on separate hardware +* allowing basic active/backup high-availability with a "hot spare" synapse + pointing at the same DB master, as well as enabling DB replication in + synapse itself. + The only disadvantage is that the code is relatively new as of April 2015 and may have a few regressions relative to SQLite. @@ -190,8 +190,8 @@ For information on how to install and use PostgreSQL, please see Running Synapse =============== -To actually run your new homeserver, pick a working directory for Synapse to run -(e.g. ``~/.synapse``), and:: +To actually run your new homeserver, pick a working directory for Synapse to +run (e.g. ``~/.synapse``), and:: cd ~/.synapse source ./bin/activate @@ -214,13 +214,13 @@ defaults to python 3, but synapse currently assumes python 2.7 by default: pip may be outdated (6.0.7-1 and needs to be upgraded to 6.0.8-1 ):: sudo pip2.7 install --upgrade pip - + You also may need to explicitly specify python 2.7 again during the install request:: pip2.7 install --process-dependency-links \ https://github.com/matrix-org/synapse/tarball/master - + If you encounter an error with lib bcrypt causing an Wrong ELF Class: ELFCLASS32 (x64 Systems), you may need to reinstall py-bcrypt to correctly compile it under the right architecture. (This should not be needed if @@ -228,7 +228,7 @@ installing under virtualenv):: sudo pip2.7 uninstall py-bcrypt sudo pip2.7 install py-bcrypt - + During setup of Synapse you need to call python2.7 directly again:: cd ~/.synapse @@ -236,25 +236,27 @@ During setup of Synapse you need to call python2.7 directly again:: --server-name machine.my.domain.name \ --config-path homeserver.yaml \ --generate-config - + ...substituting your host and domain name as appropriate. Windows Install --------------- Synapse can be installed on Cygwin. It requires the following Cygwin packages: - - gcc - - git - - libffi-devel - - openssl (and openssl-devel, python-openssl) - - python - - python-setuptools +- gcc +- git +- libffi-devel +- openssl (and openssl-devel, python-openssl) +- python +- python-setuptools The content repository requires additional packages and will be unable to process uploads without them: - - libjpeg8 - - libjpeg8-devel - - zlib + +- libjpeg8 +- libjpeg8-devel +- zlib + If you choose to install Synapse without these packages, you will need to reinstall ``pillow`` for changes to be applied, e.g. ``pip uninstall pillow`` ``pip install pillow --user`` @@ -276,8 +278,8 @@ Troubleshooting Troubleshooting Installation ---------------------------- -Synapse requires pip 1.7 or later, so if your OS provides too old a version and -you get errors about ``error: no such option: --process-dependency-links`` you +Synapse requires pip 1.7 or later, so if your OS provides too old a version and +you get errors about ``error: no such option: --process-dependency-links`` you may need to manually upgrade it:: sudo pip install --upgrade pip @@ -288,9 +290,9 @@ created. To reset the installation:: rm -rf /tmp/pip_install_matrix -pip seems to leak *lots* of memory during installation. For instance, a Linux -host with 512MB of RAM may run out of memory whilst installing Twisted. If this -happens, you will have to individually install the dependencies which are +pip seems to leak *lots* of memory during installation. For instance, a Linux +host with 512MB of RAM may run out of memory whilst installing Twisted. If this +happens, you will have to individually install the dependencies which are failing, e.g.:: pip install twisted @@ -301,8 +303,8 @@ will need to export CFLAGS=-Qunused-arguments. Troubleshooting Running ----------------------- -If synapse fails with ``missing "sodium.h"`` crypto errors, you may need -to manually upgrade PyNaCL, as synapse uses NaCl (http://nacl.cr.yp.to/) for +If synapse fails with ``missing "sodium.h"`` crypto errors, you may need +to manually upgrade PyNaCL, as synapse uses NaCl (http://nacl.cr.yp.to/) for encryption and digital signatures. Unfortunately PyNACL currently has a few issues (https://github.com/pyca/pynacl/issues/53) and @@ -313,7 +315,7 @@ fix try re-installing from PyPI or directly from # Install from PyPI pip install --user --upgrade --force pynacl - + # Install from github pip install --user https://github.com/pyca/pynacl/tarball/master @@ -324,7 +326,7 @@ If running `$ synctl start` fails with 'returned non-zero exit status 1', you will need to explicitly call Python2.7 - either running as:: python2.7 -m synapse.app.homeserver --daemonize -c homeserver.yaml - + ...or by editing synctl with the correct python executable. Synapse Development @@ -362,14 +364,11 @@ This should end with a 'PASSED' result:: Upgrading an existing Synapse ============================= -IMPORTANT: Before upgrading an existing synapse to a new version, please -refer to UPGRADE.rst for any additional instructions. - -Otherwise, simply re-install the new codebase over the current one - e.g. -by ``pip install --process-dependency-links -https://github.com/matrix-org/synapse/tarball/master`` -if using pip, or by ``git pull`` if running off a git working copy. +The instructions for upgrading synapse are in `UPGRADE.rst`_. +Please check these instructions as upgrading may require extra steps for some +versions of synapse. +.. _UPGRADE.rst: UPGRADE.rst Setting up Federation ===================== @@ -431,7 +430,7 @@ private federation (``localhost:8080``, ``localhost:8081`` and http://localhost:8080. Simply run:: demo/start.sh - + This is mainly useful just for development purposes. Running The Demo Web Client @@ -494,7 +493,7 @@ time. Where's the spec?! ================== -The source of the matrix spec lives at https://github.com/matrix-org/matrix-doc. +The source of the matrix spec lives at https://github.com/matrix-org/matrix-doc. A recent HTML snapshot of this lives at http://matrix.org/docs/spec diff --git a/UPGRADE.rst b/UPGRADE.rst index d98460f643..35a0333a7e 100644 --- a/UPGRADE.rst +++ b/UPGRADE.rst @@ -1,3 +1,36 @@ +Upgrading Synapse +================= + +Before upgrading check if any special steps are required to upgrade from the +what you currently have installed to current version of synapse. The extra +instructions that may be required are listed later in this document. + +If synapse was installed in a virtualenv then active that virtualenv before +upgrading. If synapse is installed in a virtualenv in ``~/.synapse/`` then run: + +.. code:: bash + + source ~/.synapse/bin/activate + +If synapse was installed using pip then upgrade to the latest version by +running: + +.. code:: bash + + pip install --upgrade --process-dependency-links https://github.com/matrix-org/synapse/tarball/master + +If synapse was installed using git then upgrade to the latest version by +running: + +.. code:: bash + + # Pull the latest version of the master branch. + git pull + # Update the versions of synapse's python dependencies. + python synapse/python_dependencies.py | xargs -n1 pip install + + + Upgrading to v0.9.0 =================== diff --git a/synapse/__init__.py b/synapse/__init__.py index 96e37308d6..5853165a21 100644 --- a/synapse/__init__.py +++ b/synapse/__init__.py @@ -16,4 +16,4 @@ """ This is a reference implementation of a Matrix home server. """ -__version__ = "0.9.3" +__version__ = "0.10.0-rc1" diff --git a/synapse/api/constants.py b/synapse/api/constants.py index 7156ee4e7d..1423986c1e 100644 --- a/synapse/api/constants.py +++ b/synapse/api/constants.py @@ -76,6 +76,8 @@ class EventTypes(object): Feedback = "m.room.message.feedback" RoomHistoryVisibility = "m.room.history_visibility" + CanonicalAlias = "m.room.canonical_alias" + RoomAvatar = "m.room.avatar" # These are used for validation Message = "m.room.message" diff --git a/synapse/api/errors.py b/synapse/api/errors.py index 0b3320e62c..c3b4d971a8 100644 --- a/synapse/api/errors.py +++ b/synapse/api/errors.py @@ -40,6 +40,7 @@ class Codes(object): TOO_LARGE = "M_TOO_LARGE" EXCLUSIVE = "M_EXCLUSIVE" THREEPID_AUTH_FAILED = "M_THREEPID_AUTH_FAILED" + THREEPID_IN_USE = "THREEPID_IN_USE" class CodeMessageException(RuntimeError): diff --git a/synapse/config/repository.py b/synapse/config/repository.py index 6891abd71d..64644b9a7a 100644 --- a/synapse/config/repository.py +++ b/synapse/config/repository.py @@ -14,6 +14,39 @@ # limitations under the License. from ._base import Config +from collections import namedtuple + +ThumbnailRequirement = namedtuple( + "ThumbnailRequirement", ["width", "height", "method", "media_type"] +) + + +def parse_thumbnail_requirements(thumbnail_sizes): + """ Takes a list of dictionaries with "width", "height", and "method" keys + and creates a map from image media types to the thumbnail size, thumnailing + method, and thumbnail media type to precalculate + + Args: + thumbnail_sizes(list): List of dicts with "width", "height", and + "method" keys + Returns: + Dictionary mapping from media type string to list of + ThumbnailRequirement tuples. + """ + requirements = {} + for size in thumbnail_sizes: + width = size["width"] + height = size["height"] + method = size["method"] + jpeg_thumbnail = ThumbnailRequirement(width, height, method, "image/jpeg") + png_thumbnail = ThumbnailRequirement(width, height, method, "image/png") + requirements.setdefault("image/jpeg", []).append(jpeg_thumbnail) + requirements.setdefault("image/gif", []).append(png_thumbnail) + requirements.setdefault("image/png", []).append(png_thumbnail) + return { + media_type: tuple(thumbnails) + for media_type, thumbnails in requirements.items() + } class ContentRepositoryConfig(Config): @@ -22,6 +55,10 @@ class ContentRepositoryConfig(Config): self.max_image_pixels = self.parse_size(config["max_image_pixels"]) self.media_store_path = self.ensure_directory(config["media_store_path"]) self.uploads_path = self.ensure_directory(config["uploads_path"]) + self.dynamic_thumbnails = config["dynamic_thumbnails"] + self.thumbnail_requirements = parse_thumbnail_requirements( + config["thumbnail_sizes"] + ) def default_config(self, config_dir_path, server_name): media_store = self.default_path("media_store") @@ -38,4 +75,26 @@ class ContentRepositoryConfig(Config): # Maximum number of pixels that will be thumbnailed max_image_pixels: "32M" + + # Whether to generate new thumbnails on the fly to precisely match + # the resolution requested by the client. If true then whenever + # a new resolution is requested by the client the server will + # generate a new thumbnail. If false the server will pick a thumbnail + # from a precalcualted list. + dynamic_thumbnails: false + + # List of thumbnail to precalculate when an image is uploaded. + thumbnail_sizes: + - width: 32 + height: 32 + method: crop + - width: 96 + height: 96 + method: crop + - width: 320 + height: 240 + method: scale + - width: 640 + height: 480 + method: scale """ % locals() diff --git a/synapse/handlers/_base.py b/synapse/handlers/_base.py index d6c064b398..e91f1129db 100644 --- a/synapse/handlers/_base.py +++ b/synapse/handlers/_base.py @@ -18,7 +18,7 @@ from twisted.internet import defer from synapse.api.errors import LimitExceededError, SynapseError from synapse.crypto.event_signing import add_hashes_and_signatures from synapse.api.constants import Membership, EventTypes -from synapse.types import UserID +from synapse.types import UserID, RoomAlias from synapse.util.logcontext import PreserveLoggingContext @@ -130,6 +130,22 @@ class BaseHandler(object): returned_invite.signatures ) + if event.type == EventTypes.CanonicalAlias: + # Check the alias is acually valid (at this time at least) + room_alias_str = event.content.get("alias", None) + if room_alias_str: + room_alias = RoomAlias.from_string(room_alias_str) + directory_handler = self.hs.get_handlers().directory_handler + mapping = yield directory_handler.get_association(room_alias) + + if mapping["room_id"] != event.room_id: + raise SynapseError( + 400, + "Room alias %s does not point to the room" % ( + room_alias_str, + ) + ) + destinations = set(extra_destinations) for k, s in context.current_state.items(): try: diff --git a/synapse/handlers/auth.py b/synapse/handlers/auth.py index 65bd8189dc..af602bee44 100644 --- a/synapse/handlers/auth.py +++ b/synapse/handlers/auth.py @@ -163,7 +163,7 @@ class AuthHandler(BaseHandler): if not user_id.startswith('@'): user_id = UserID.create(user_id, self.hs.hostname).to_string() - self._check_password(user_id, password) + yield self._check_password(user_id, password) defer.returnValue(user_id) @defer.inlineCallbacks diff --git a/synapse/handlers/identity.py b/synapse/handlers/identity.py index c1095708a0..2a99921d5f 100644 --- a/synapse/handlers/identity.py +++ b/synapse/handlers/identity.py @@ -117,3 +117,28 @@ class IdentityHandler(BaseHandler): except CodeMessageException as e: data = json.loads(e.msg) defer.returnValue(data) + + @defer.inlineCallbacks + def requestEmailToken(self, id_server, email, client_secret, send_attempt, **kwargs): + yield run_on_reactor() + http_client = SimpleHttpClient(self.hs) + + params = { + 'email': email, + 'client_secret': client_secret, + 'send_attempt': send_attempt, + } + params.update(kwargs) + + try: + data = yield http_client.post_urlencoded_get_json( + "https://%s%s" % ( + id_server, + "/_matrix/identity/api/v1/validate/email/requestToken" + ), + params + ) + defer.returnValue(data) + except CodeMessageException as e: + logger.info("Proxied requestToken failed: %r", e) + raise e diff --git a/synapse/handlers/room.py b/synapse/handlers/room.py index 82c16013a3..c5d1001b50 100644 --- a/synapse/handlers/room.py +++ b/synapse/handlers/room.py @@ -247,9 +247,11 @@ class RoomCreationHandler(BaseHandler): }, "users_default": 0, "events": { - EventTypes.Name: 100, + EventTypes.Name: 50, EventTypes.PowerLevels: 100, EventTypes.RoomHistoryVisibility: 100, + EventTypes.CanonicalAlias: 50, + EventTypes.RoomAvatar: 50, }, "events_default": 0, "state_default": 50, diff --git a/synapse/push/__init__.py b/synapse/push/__init__.py index 36f450c31d..13002e0db4 100644 --- a/synapse/push/__init__.py +++ b/synapse/push/__init__.py @@ -294,6 +294,12 @@ class Pusher(object): if not single_event: self.last_token = chunk['end'] logger.debug("Event stream timeout for pushkey %s", self.pushkey) + yield self.store.update_pusher_last_token( + self.app_id, + self.pushkey, + self.user_name, + self.last_token + ) return if not self.alive: @@ -345,7 +351,7 @@ class Pusher(object): if processed: self.backoff_delay = Pusher.INITIAL_BACKOFF self.last_token = chunk['end'] - self.store.update_pusher_last_token_and_success( + yield self.store.update_pusher_last_token_and_success( self.app_id, self.pushkey, self.user_name, @@ -354,7 +360,7 @@ class Pusher(object): ) if self.failing_since: self.failing_since = None - self.store.update_pusher_failing_since( + yield self.store.update_pusher_failing_since( self.app_id, self.pushkey, self.user_name, @@ -362,7 +368,7 @@ class Pusher(object): else: if not self.failing_since: self.failing_since = self.clock.time_msec() - self.store.update_pusher_failing_since( + yield self.store.update_pusher_failing_since( self.app_id, self.pushkey, self.user_name, @@ -380,7 +386,7 @@ class Pusher(object): self.user_name, self.pushkey) self.backoff_delay = Pusher.INITIAL_BACKOFF self.last_token = chunk['end'] - self.store.update_pusher_last_token( + yield self.store.update_pusher_last_token( self.app_id, self.pushkey, self.user_name, @@ -388,7 +394,7 @@ class Pusher(object): ) self.failing_since = None - self.store.update_pusher_failing_since( + yield self.store.update_pusher_failing_since( self.app_id, self.pushkey, self.user_name, diff --git a/synapse/rest/client/v1/login.py b/synapse/rest/client/v1/login.py index b963a38618..67323a16bb 100644 --- a/synapse/rest/client/v1/login.py +++ b/synapse/rest/client/v1/login.py @@ -74,13 +74,20 @@ class LoginRestServlet(ClientV1RestServlet): @defer.inlineCallbacks def do_password_login(self, login_submission): - if not login_submission["user"].startswith('@'): - login_submission["user"] = UserID.create( - login_submission["user"], self.hs.hostname).to_string() + if 'medium' in login_submission and 'address' in login_submission: + user_id = yield self.hs.get_datastore().get_user_id_by_threepid( + login_submission['medium'], login_submission['address'] + ) + else: + user_id = login_submission['user'] + + if not user_id.startswith('@'): + user_id = UserID.create( + user_id, self.hs.hostname).to_string() auth_handler = self.handlers.auth_handler access_token, refresh_token = yield auth_handler.login_with_password( - user_id=login_submission["user"], + user_id=user_id, password=login_submission["password"]) result = { diff --git a/synapse/rest/client/v2_alpha/account.py b/synapse/rest/client/v2_alpha/account.py index 897c54b539..522a312c9e 100644 --- a/synapse/rest/client/v2_alpha/account.py +++ b/synapse/rest/client/v2_alpha/account.py @@ -79,7 +79,7 @@ class PasswordRestServlet(RestServlet): new_password = params['new_password'] yield self.auth_handler.set_password( - user_id, new_password, None + user_id, new_password ) defer.returnValue((200, {})) diff --git a/synapse/rest/client/v2_alpha/register.py b/synapse/rest/client/v2_alpha/register.py index 012c447e88..1ba2f29711 100644 --- a/synapse/rest/client/v2_alpha/register.py +++ b/synapse/rest/client/v2_alpha/register.py @@ -54,6 +54,11 @@ class RegisterRestServlet(RestServlet): @defer.inlineCallbacks def on_POST(self, request): yield run_on_reactor() + + if '/register/email/requestToken' in request.path: + ret = yield self.onEmailTokenRequest(request) + defer.returnValue(ret) + body = parse_json_dict_from_request(request) # we do basic sanity checks here because the auth layer will store these @@ -208,6 +213,29 @@ class RegisterRestServlet(RestServlet): "home_server": self.hs.hostname, } + @defer.inlineCallbacks + def onEmailTokenRequest(self, request): + body = parse_json_dict_from_request(request) + + required = ['id_server', 'client_secret', 'email', 'send_attempt'] + absent = [] + for k in required: + if k not in body: + absent.append(k) + + if len(absent) > 0: + raise SynapseError(400, "Missing params: %r" % absent, Codes.MISSING_PARAM) + + existingUid = yield self.hs.get_datastore().get_user_id_by_threepid( + 'email', body['email'] + ) + + if existingUid is not None: + raise SynapseError(400, "Email is already in use", Codes.THREEPID_IN_USE) + + ret = yield self.identity_handler.requestEmailToken(**body) + defer.returnValue((200, ret)) + def register_servlets(hs, http_server): RegisterRestServlet(hs).register(http_server) diff --git a/synapse/rest/media/v1/base_resource.py b/synapse/rest/media/v1/base_resource.py index 84e1961a21..4e21527c3d 100644 --- a/synapse/rest/media/v1/base_resource.py +++ b/synapse/rest/media/v1/base_resource.py @@ -69,6 +69,8 @@ class BaseMediaResource(Resource): self.filepaths = filepaths self.version_string = hs.version_string self.downloads = {} + self.dynamic_thumbnails = hs.config.dynamic_thumbnails + self.thumbnail_requirements = hs.config.thumbnail_requirements def _respond_404(self, request): respond_with_json( @@ -208,22 +210,74 @@ class BaseMediaResource(Resource): self._respond_404(request) def _get_thumbnail_requirements(self, media_type): - if media_type == "image/jpeg": - return ( - (32, 32, "crop", "image/jpeg"), - (96, 96, "crop", "image/jpeg"), - (320, 240, "scale", "image/jpeg"), - (640, 480, "scale", "image/jpeg"), - ) - elif (media_type == "image/png") or (media_type == "image/gif"): - return ( - (32, 32, "crop", "image/png"), - (96, 96, "crop", "image/png"), - (320, 240, "scale", "image/png"), - (640, 480, "scale", "image/png"), + return self.thumbnail_requirements.get(media_type, ()) + + def _generate_thumbnail(self, input_path, t_path, t_width, t_height, + t_method, t_type): + thumbnailer = Thumbnailer(input_path) + m_width = thumbnailer.width + m_height = thumbnailer.height + + if m_width * m_height >= self.max_image_pixels: + logger.info( + "Image too large to thumbnail %r x %r > %r", + m_width, m_height, self.max_image_pixels ) + return + + if t_method == "crop": + t_len = thumbnailer.crop(t_path, t_width, t_height, t_type) + elif t_method == "scale": + t_len = thumbnailer.scale(t_path, t_width, t_height, t_type) else: - return () + t_len = None + + return t_len + + @defer.inlineCallbacks + def _generate_local_exact_thumbnail(self, media_id, t_width, t_height, + t_method, t_type): + input_path = self.filepaths.local_media_filepath(media_id) + + t_path = self.filepaths.local_media_thumbnail( + media_id, t_width, t_height, t_type, t_method + ) + self._makedirs(t_path) + + t_len = yield threads.deferToThread( + self._generate_thumbnail, + input_path, t_path, t_width, t_height, t_method, t_type + ) + + if t_len: + yield self.store.store_local_thumbnail( + media_id, t_width, t_height, t_type, t_method, t_len + ) + + defer.returnValue(t_path) + + @defer.inlineCallbacks + def _generate_remote_exact_thumbnail(self, server_name, file_id, media_id, + t_width, t_height, t_method, t_type): + input_path = self.filepaths.remote_media_filepath(server_name, file_id) + + t_path = self.filepaths.remote_media_thumbnail( + server_name, file_id, t_width, t_height, t_type, t_method + ) + self._makedirs(t_path) + + t_len = yield threads.deferToThread( + self._generate_thumbnail, + input_path, t_path, t_width, t_height, t_method, t_type + ) + + if t_len: + yield self.store.store_remote_media_thumbnail( + server_name, media_id, file_id, + t_width, t_height, t_type, t_method, t_len + ) + + defer.returnValue(t_path) @defer.inlineCallbacks def _generate_local_thumbnails(self, media_id, media_info): diff --git a/synapse/rest/media/v1/thumbnail_resource.py b/synapse/rest/media/v1/thumbnail_resource.py index 61f88e486e..e506dad934 100644 --- a/synapse/rest/media/v1/thumbnail_resource.py +++ b/synapse/rest/media/v1/thumbnail_resource.py @@ -43,14 +43,25 @@ class ThumbnailResource(BaseMediaResource): m_type = parse_string(request, "type", "image/png") if server_name == self.server_name: - yield self._respond_local_thumbnail( - request, media_id, width, height, method, m_type - ) + if self.dynamic_thumbnails: + yield self._select_or_generate_local_thumbnail( + request, media_id, width, height, method, m_type + ) + else: + yield self._respond_local_thumbnail( + request, media_id, width, height, method, m_type + ) else: - yield self._respond_remote_thumbnail( - request, server_name, media_id, - width, height, method, m_type - ) + if self.dynamic_thumbnails: + yield self._select_or_generate_remote_thumbnail( + request, server_name, media_id, + width, height, method, m_type + ) + else: + yield self._respond_remote_thumbnail( + request, server_name, media_id, + width, height, method, m_type + ) @defer.inlineCallbacks def _respond_local_thumbnail(self, request, media_id, width, height, @@ -83,6 +94,87 @@ class ThumbnailResource(BaseMediaResource): ) @defer.inlineCallbacks + def _select_or_generate_local_thumbnail(self, request, media_id, desired_width, + desired_height, desired_method, + desired_type): + media_info = yield self.store.get_local_media(media_id) + + if not media_info: + self._respond_404(request) + return + + thumbnail_infos = yield self.store.get_local_media_thumbnails(media_id) + for info in thumbnail_infos: + t_w = info["thumbnail_width"] == desired_width + t_h = info["thumbnail_height"] == desired_height + t_method = info["thumbnail_method"] == desired_method + t_type = info["thumbnail_type"] == desired_type + + if t_w and t_h and t_method and t_type: + file_path = self.filepaths.local_media_thumbnail( + media_id, desired_width, desired_height, desired_type, desired_method, + ) + yield self._respond_with_file(request, desired_type, file_path) + return + + logger.debug("We don't have a local thumbnail of that size. Generating") + + # Okay, so we generate one. + file_path = yield self._generate_local_exact_thumbnail( + media_id, desired_width, desired_height, desired_method, desired_type + ) + + if file_path: + yield self._respond_with_file(request, desired_type, file_path) + else: + yield self._respond_default_thumbnail( + request, media_info, desired_width, desired_height, + desired_method, desired_type, + ) + + @defer.inlineCallbacks + def _select_or_generate_remote_thumbnail(self, request, server_name, media_id, + desired_width, desired_height, + desired_method, desired_type): + media_info = yield self._get_remote_media(server_name, media_id) + + thumbnail_infos = yield self.store.get_remote_media_thumbnails( + server_name, media_id, + ) + + file_id = media_info["filesystem_id"] + + for info in thumbnail_infos: + t_w = info["thumbnail_width"] == desired_width + t_h = info["thumbnail_height"] == desired_height + t_method = info["thumbnail_method"] == desired_method + t_type = info["thumbnail_type"] == desired_type + + if t_w and t_h and t_method and t_type: + file_path = self.filepaths.remote_media_thumbnail( + server_name, file_id, desired_width, desired_height, + desired_type, desired_method, + ) + yield self._respond_with_file(request, desired_type, file_path) + return + + logger.debug("We don't have a local thumbnail of that size. Generating") + + # Okay, so we generate one. + file_path = yield self._generate_remote_exact_thumbnail( + server_name, file_id, media_id, desired_width, + desired_height, desired_method, desired_type + ) + + if file_path: + yield self._respond_with_file(request, desired_type, file_path) + else: + yield self._respond_default_thumbnail( + request, media_info, desired_width, desired_height, + desired_method, desired_type, + ) + + @defer.inlineCallbacks def _respond_remote_thumbnail(self, request, server_name, media_id, width, height, method, m_type): # TODO: Don't download the whole remote file diff --git a/synapse/storage/schema/delta/22/user_threepids_unique.sql b/synapse/storage/schema/delta/22/user_threepids_unique.sql new file mode 100644 index 0000000000..87edfa454c --- /dev/null +++ b/synapse/storage/schema/delta/22/user_threepids_unique.sql @@ -0,0 +1,19 @@ +CREATE TABLE IF NOT EXISTS user_threepids2 ( + user_id TEXT NOT NULL, + medium TEXT NOT NULL, + address TEXT NOT NULL, + validated_at BIGINT NOT NULL, + added_at BIGINT NOT NULL, + CONSTRAINT medium_address UNIQUE (medium, address) +); + +INSERT INTO user_threepids2 + SELECT * FROM user_threepids WHERE added_at IN ( + SELECT max(added_at) FROM user_threepids GROUP BY medium, address + ) +; + +DROP TABLE user_threepids; +ALTER TABLE user_threepids2 RENAME TO user_threepids; + +CREATE INDEX user_threepids_user_id ON user_threepids(user_id); diff --git a/tests/rest/client/v2_alpha/test_register.py b/tests/rest/client/v2_alpha/test_register.py index 66fd25964d..f9a2b22485 100644 --- a/tests/rest/client/v2_alpha/test_register.py +++ b/tests/rest/client/v2_alpha/test_register.py @@ -13,6 +13,7 @@ class RegisterRestServletTestCase(unittest.TestCase): self.request_data = "" self.request = Mock( content=Mock(read=Mock(side_effect=lambda: self.request_data)), + path='/_matrix/api/v2_alpha/register' ) self.request.args = {} @@ -131,4 +132,4 @@ class RegisterRestServletTestCase(unittest.TestCase): }) self.registration_handler.register = Mock(return_value=("@user:id", "t")) d = self.servlet.on_POST(self.request) - return self.assertFailure(d, SynapseError) \ No newline at end of file + return self.assertFailure(d, SynapseError) |