From a18f93279eb7b8505db63af062695e8c7fe263c4 Mon Sep 17 00:00:00 2001 From: David Baker Date: Fri, 10 May 2019 11:11:59 +0100 Subject: Add changelog entry --- changelog.d/5174.bugfix | 1 + 1 file changed, 1 insertion(+) create mode 100644 changelog.d/5174.bugfix (limited to 'changelog.d') diff --git a/changelog.d/5174.bugfix b/changelog.d/5174.bugfix new file mode 100644 index 0000000000..f4d3192bd9 --- /dev/null +++ b/changelog.d/5174.bugfix @@ -0,0 +1 @@ +Allow clients to complete email auth first when registering -- cgit 1.5.1 From 7a3eb8657d66bf0ae479b5a0db9529df5d0226c1 Mon Sep 17 00:00:00 2001 From: David Baker Date: Fri, 10 May 2019 11:18:35 +0100 Subject: Thanks, automated grammar pedantry. --- changelog.d/5174.bugfix | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'changelog.d') diff --git a/changelog.d/5174.bugfix b/changelog.d/5174.bugfix index f4d3192bd9..2d7c71739b 100644 --- a/changelog.d/5174.bugfix +++ b/changelog.d/5174.bugfix @@ -1 +1 @@ -Allow clients to complete email auth first when registering +Allow clients to complete email auth first when registering. -- cgit 1.5.1 From c9f811c5d45ac52d4b55ed74d2919b0d41790983 Mon Sep 17 00:00:00 2001 From: David Baker Date: Fri, 10 May 2019 14:01:19 +0100 Subject: Update changelog --- changelog.d/5174.bugfix | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'changelog.d') diff --git a/changelog.d/5174.bugfix b/changelog.d/5174.bugfix index 2d7c71739b..0f26d46b2c 100644 --- a/changelog.d/5174.bugfix +++ b/changelog.d/5174.bugfix @@ -1 +1 @@ -Allow clients to complete email auth first when registering. +Re-order stages in registration flows such that msisdn and email verification are done last. -- cgit 1.5.1 From 715516284404b9cf5e3f8c21335782b5d47e5cb1 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Wed, 15 May 2019 11:33:22 +0100 Subject: Newsfile --- changelog.d/5191.misc | 1 + 1 file changed, 1 insertion(+) create mode 100644 changelog.d/5191.misc (limited to 'changelog.d') diff --git a/changelog.d/5191.misc b/changelog.d/5191.misc new file mode 100644 index 0000000000..e0615fec9c --- /dev/null +++ b/changelog.d/5191.misc @@ -0,0 +1 @@ +Make generating SQL bounds for pagination generic. -- cgit 1.5.1 From 5fb72e6888c2dbd54a7b7d76657ea36c0faccedc Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 14 May 2019 17:05:45 +0100 Subject: Newsfile --- changelog.d/5198.feature | 1 + 1 file changed, 1 insertion(+) create mode 100644 changelog.d/5198.feature (limited to 'changelog.d') diff --git a/changelog.d/5198.feature b/changelog.d/5198.feature new file mode 100644 index 0000000000..c8f624d172 --- /dev/null +++ b/changelog.d/5198.feature @@ -0,0 +1 @@ +Add experimental support for reactions. -- cgit 1.5.1 From f89f688a55e1ddffbfa3613ac333f43e81d4b995 Mon Sep 17 00:00:00 2001 From: PauRE Date: Thu, 16 May 2019 20:04:26 +0200 Subject: Fix image orientation when generating thumbnail (#5039) --- changelog.d/5039.bugfix | 1 + synapse/python_dependencies.py | 2 +- synapse/rest/media/v1/media_repository.py | 9 ++++++++ synapse/rest/media/v1/thumbnailer.py | 35 +++++++++++++++++++++++++++++++ 4 files changed, 46 insertions(+), 1 deletion(-) create mode 100644 changelog.d/5039.bugfix (limited to 'changelog.d') diff --git a/changelog.d/5039.bugfix b/changelog.d/5039.bugfix new file mode 100644 index 0000000000..212cff7ae8 --- /dev/null +++ b/changelog.d/5039.bugfix @@ -0,0 +1 @@ +Fix image orientation when generating thumbnails (needs pillow>=4.3.0). Contributed by Pau Rodriguez-Estivill. diff --git a/synapse/python_dependencies.py b/synapse/python_dependencies.py index 2708f5e820..fdcfb90a7e 100644 --- a/synapse/python_dependencies.py +++ b/synapse/python_dependencies.py @@ -53,7 +53,7 @@ REQUIREMENTS = [ "pyasn1-modules>=0.0.7", "daemonize>=2.3.1", "bcrypt>=3.1.0", - "pillow>=3.1.2", + "pillow>=4.3.0", "sortedcontainers>=1.4.4", "psutil>=2.0.0", "pymacaroons>=0.13.0", diff --git a/synapse/rest/media/v1/media_repository.py b/synapse/rest/media/v1/media_repository.py index bdffa97805..8569677355 100644 --- a/synapse/rest/media/v1/media_repository.py +++ b/synapse/rest/media/v1/media_repository.py @@ -444,6 +444,9 @@ class MediaRepository(object): ) return + if thumbnailer.transpose_method is not None: + m_width, m_height = thumbnailer.transpose() + if t_method == "crop": t_byte_source = thumbnailer.crop(t_width, t_height, t_type) elif t_method == "scale": @@ -578,6 +581,12 @@ class MediaRepository(object): ) return + if thumbnailer.transpose_method is not None: + m_width, m_height = yield logcontext.defer_to_thread( + self.hs.get_reactor(), + thumbnailer.transpose + ) + # We deduplicate the thumbnail sizes by ignoring the cropped versions if # they have the same dimensions of a scaled one. thumbnails = {} diff --git a/synapse/rest/media/v1/thumbnailer.py b/synapse/rest/media/v1/thumbnailer.py index a4b26c2587..3efd0d80fc 100644 --- a/synapse/rest/media/v1/thumbnailer.py +++ b/synapse/rest/media/v1/thumbnailer.py @@ -20,6 +20,17 @@ import PIL.Image as Image logger = logging.getLogger(__name__) +EXIF_ORIENTATION_TAG = 0x0112 +EXIF_TRANSPOSE_MAPPINGS = { + 2: Image.FLIP_LEFT_RIGHT, + 3: Image.ROTATE_180, + 4: Image.FLIP_TOP_BOTTOM, + 5: Image.TRANSPOSE, + 6: Image.ROTATE_270, + 7: Image.TRANSVERSE, + 8: Image.ROTATE_90 +} + class Thumbnailer(object): @@ -31,6 +42,30 @@ class Thumbnailer(object): def __init__(self, input_path): self.image = Image.open(input_path) self.width, self.height = self.image.size + self.transpose_method = None + try: + # We don't use ImageOps.exif_transpose since it crashes with big EXIF + image_exif = self.image._getexif() + if image_exif is not None: + image_orientation = image_exif.get(EXIF_ORIENTATION_TAG) + self.transpose_method = EXIF_TRANSPOSE_MAPPINGS.get(image_orientation) + except Exception as e: + # A lot of parsing errors can happen when parsing EXIF + logger.info("Error parsing image EXIF information: %s", e) + + def transpose(self): + """Transpose the image using its EXIF Orientation tag + + Returns: + Tuple[int, int]: (width, height) containing the new image size in pixels. + """ + if self.transpose_method is not None: + self.image = self.image.transpose(self.transpose_method) + self.width, self.height = self.image.size + self.transpose_method = None + # We don't need EXIF any more + self.image.info["exif"] = None + return self.image.size def aspect(self, max_width, max_height): """Calculate the largest size that preserves aspect ratio which -- cgit 1.5.1 From 7ce1f97a1353e7bd9232c22a20835e40fa5662e0 Mon Sep 17 00:00:00 2001 From: Richard van der Hoff <1389908+richvdh@users.noreply.github.com> Date: Fri, 17 May 2019 12:38:03 +0100 Subject: Stop telling people to install the optional dependencies. (#5197) * Stop telling people to install the optional dependencies. They're optional. Also update the postgres docs a bit for clarity(?) --- INSTALL.md | 4 ++-- changelog.d/5197.misc | 1 + docs/postgres.rst | 45 ++++++++++++++++++++++----------------------- 3 files changed, 25 insertions(+), 25 deletions(-) create mode 100644 changelog.d/5197.misc (limited to 'changelog.d') diff --git a/INSTALL.md b/INSTALL.md index b88d826f6c..1934593148 100644 --- a/INSTALL.md +++ b/INSTALL.md @@ -35,7 +35,7 @@ virtualenv -p python3 ~/synapse/env source ~/synapse/env/bin/activate pip install --upgrade pip pip install --upgrade setuptools -pip install matrix-synapse[all] +pip install matrix-synapse ``` This will download Synapse from [PyPI](https://pypi.org/project/matrix-synapse) @@ -48,7 +48,7 @@ update flag: ``` source ~/synapse/env/bin/activate -pip install -U matrix-synapse[all] +pip install -U matrix-synapse ``` Before you can start Synapse, you will need to generate a configuration diff --git a/changelog.d/5197.misc b/changelog.d/5197.misc new file mode 100644 index 0000000000..fca1d86b2e --- /dev/null +++ b/changelog.d/5197.misc @@ -0,0 +1 @@ +Stop telling people to install the optional dependencies by default. diff --git a/docs/postgres.rst b/docs/postgres.rst index f7ebbed0c3..e81e10403f 100644 --- a/docs/postgres.rst +++ b/docs/postgres.rst @@ -3,6 +3,28 @@ Using Postgres Postgres version 9.4 or later is known to work. +Install postgres client libraries +================================= + +Synapse will require the python postgres client library in order to connect to +a postgres database. + +* If you are using the `matrix.org debian/ubuntu + packages <../INSTALL.md#matrixorg-packages>`_, + the necessary libraries will already be installed. + +* For other pre-built packages, please consult the documentation from the + relevant package. + +* If you installed synapse `in a virtualenv + <../INSTALL.md#installing-from-source>`_, you can install the library with:: + + ~/synapse/env/bin/pip install matrix-synapse[postgres] + + (substituting the path to your virtualenv for ``~/synapse/env``, if you used a + different path). You will require the postgres development files. These are in + the ``libpq-dev`` package on Debian-derived distributions. + Set up database =============== @@ -26,29 +48,6 @@ encoding use, e.g.:: This would create an appropriate database named ``synapse`` owned by the ``synapse_user`` user (which must already exist). -Set up client in Debian/Ubuntu -=========================== - -Postgres support depends on the postgres python connector ``psycopg2``. In the -virtual env:: - - sudo apt-get install libpq-dev - pip install psycopg2 - -Set up client in RHEL/CentOs 7 -============================== - -Make sure you have the appropriate version of postgres-devel installed. For a -postgres 9.4, use the postgres 9.4 packages from -[here](https://wiki.postgresql.org/wiki/YUM_Installation). - -As with Debian/Ubuntu, postgres support depends on the postgres python connector -``psycopg2``. In the virtual env:: - - sudo yum install postgresql-devel libpqxx-devel.x86_64 - export PATH=/usr/pgsql-9.4/bin/:$PATH - pip install psycopg2 - Tuning Postgres =============== -- cgit 1.5.1 From ce5bcefc609db40740c692bd53a1ef84ab675e8c Mon Sep 17 00:00:00 2001 From: Neil Johnson Date: Fri, 17 May 2019 13:27:19 +0100 Subject: expose SlavedProfileStore to ClientReaderSlavedStore (#5200) * expose SlavedProfileStore to ClientReaderSlavedStore --- changelog.d/5200.bugfix | 1 + synapse/app/client_reader.py | 2 ++ 2 files changed, 3 insertions(+) create mode 100644 changelog.d/5200.bugfix (limited to 'changelog.d') diff --git a/changelog.d/5200.bugfix b/changelog.d/5200.bugfix new file mode 100644 index 0000000000..f346c7b0cc --- /dev/null +++ b/changelog.d/5200.bugfix @@ -0,0 +1 @@ +Fix worker registration bug caused by ClientReaderSlavedStore being unable to see get_profileinfo. diff --git a/synapse/app/client_reader.py b/synapse/app/client_reader.py index 864f1eac48..707f7c2f07 100644 --- a/synapse/app/client_reader.py +++ b/synapse/app/client_reader.py @@ -29,6 +29,7 @@ from synapse.http.server import JsonResource from synapse.http.site import SynapseSite from synapse.metrics import RegistryProxy from synapse.metrics.resource import METRICS_PREFIX, MetricsResource +from synapse.replication.slave.storage import SlavedProfileStore from synapse.replication.slave.storage._base import BaseSlavedStore from synapse.replication.slave.storage.account_data import SlavedAccountDataStore from synapse.replication.slave.storage.appservice import SlavedApplicationServiceStore @@ -83,6 +84,7 @@ class ClientReaderSlavedStore( SlavedTransactionStore, SlavedClientIpStore, BaseSlavedStore, + SlavedProfileStore, ): pass -- cgit 1.5.1 From 291e1eea5eab04df79ad607ca23fb421f11a63ff Mon Sep 17 00:00:00 2001 From: bytepoets-blo Date: Fri, 17 May 2019 18:27:14 +0200 Subject: fix mapping of return values for get_or_register_3pid_guest (#5177) * fix mapping of return values for get_or_register_3pid_guest --- changelog.d/5177.bugfix | 1 + synapse/handlers/room_member.py | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) create mode 100644 changelog.d/5177.bugfix (limited to 'changelog.d') diff --git a/changelog.d/5177.bugfix b/changelog.d/5177.bugfix new file mode 100644 index 0000000000..c2f1644ae5 --- /dev/null +++ b/changelog.d/5177.bugfix @@ -0,0 +1 @@ +Fix 3pid guest invites. diff --git a/synapse/handlers/room_member.py b/synapse/handlers/room_member.py index ffc588d454..93ac986c86 100644 --- a/synapse/handlers/room_member.py +++ b/synapse/handlers/room_member.py @@ -944,7 +944,7 @@ class RoomMemberHandler(object): } if self.config.invite_3pid_guest: - guest_access_token, guest_user_id = yield self.get_or_register_3pid_guest( + guest_user_id, guest_access_token = yield self.get_or_register_3pid_guest( requester=requester, medium=medium, address=address, -- cgit 1.5.1 From 3787133c9e3fcf0e9b85700418bf03c48ec86ab3 Mon Sep 17 00:00:00 2001 From: ReidAnderson Date: Mon, 20 May 2019 05:20:08 -0500 Subject: Limit UserIds to a length that fits in a state key (#5198) --- changelog.d/5198.bugfix | 1 + synapse/api/constants.py | 3 +++ synapse/handlers/register.py | 11 ++++++++++- tests/handlers/test_register.py | 7 +++++++ 4 files changed, 21 insertions(+), 1 deletion(-) create mode 100644 changelog.d/5198.bugfix (limited to 'changelog.d') diff --git a/changelog.d/5198.bugfix b/changelog.d/5198.bugfix new file mode 100644 index 0000000000..c6b156f17d --- /dev/null +++ b/changelog.d/5198.bugfix @@ -0,0 +1 @@ +Prevent registration for user ids that are to long to fit into a state key. Contributed by Reid Anderson. \ No newline at end of file diff --git a/synapse/api/constants.py b/synapse/api/constants.py index 8547a63535..c7bf95b426 100644 --- a/synapse/api/constants.py +++ b/synapse/api/constants.py @@ -23,6 +23,9 @@ MAX_DEPTH = 2**63 - 1 # the maximum length for a room alias is 255 characters MAX_ALIAS_LENGTH = 255 +# the maximum length for a user id is 255 characters +MAX_USERID_LENGTH = 255 + class Membership(object): diff --git a/synapse/handlers/register.py b/synapse/handlers/register.py index a51d11a257..e83ee24f10 100644 --- a/synapse/handlers/register.py +++ b/synapse/handlers/register.py @@ -19,7 +19,7 @@ import logging from twisted.internet import defer from synapse import types -from synapse.api.constants import LoginType +from synapse.api.constants import MAX_USERID_LENGTH, LoginType from synapse.api.errors import ( AuthError, Codes, @@ -123,6 +123,15 @@ class RegistrationHandler(BaseHandler): self.check_user_id_not_appservice_exclusive(user_id) + if len(user_id) > MAX_USERID_LENGTH: + raise SynapseError( + 400, + "User ID may not be longer than %s characters" % ( + MAX_USERID_LENGTH, + ), + Codes.INVALID_USERNAME + ) + users = yield self.store.get_users_by_id_case_insensitive(user_id) if users: if not guest_access_token: diff --git a/tests/handlers/test_register.py b/tests/handlers/test_register.py index 1c253d0579..5ffba2ca7a 100644 --- a/tests/handlers/test_register.py +++ b/tests/handlers/test_register.py @@ -228,3 +228,10 @@ class RegistrationTestCase(unittest.HomeserverTestCase): def test_register_not_support_user(self): res = self.get_success(self.handler.register(localpart='user')) self.assertFalse(self.store.is_support_user(res[0])) + + def test_invalid_user_id_length(self): + invalid_user_id = "x" * 256 + self.get_failure( + self.handler.register(localpart=invalid_user_id), + SynapseError + ) -- cgit 1.5.1 From 06671057b626042ca4645a370cd62edcf4d6211b Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Mon, 20 May 2019 12:38:20 +0100 Subject: Newsfile --- changelog.d/5198.feature | 1 - changelog.d/5209.feature | 1 + 2 files changed, 1 insertion(+), 1 deletion(-) delete mode 100644 changelog.d/5198.feature create mode 100644 changelog.d/5209.feature (limited to 'changelog.d') diff --git a/changelog.d/5198.feature b/changelog.d/5198.feature deleted file mode 100644 index c8f624d172..0000000000 --- a/changelog.d/5198.feature +++ /dev/null @@ -1 +0,0 @@ -Add experimental support for reactions. diff --git a/changelog.d/5209.feature b/changelog.d/5209.feature new file mode 100644 index 0000000000..747098c166 --- /dev/null +++ b/changelog.d/5209.feature @@ -0,0 +1 @@ +Add experimental support for relations (aka reactions and edits). -- cgit 1.5.1 From d642178654a025664fd4ff984c1259a1b1271845 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Mon, 20 May 2019 14:32:16 +0100 Subject: Newsfile --- changelog.d/5211.feature | 1 + 1 file changed, 1 insertion(+) create mode 100644 changelog.d/5211.feature (limited to 'changelog.d') diff --git a/changelog.d/5211.feature b/changelog.d/5211.feature new file mode 100644 index 0000000000..747098c166 --- /dev/null +++ b/changelog.d/5211.feature @@ -0,0 +1 @@ +Add experimental support for relations (aka reactions and edits). -- cgit 1.5.1 From 5206648a4a2c94543d46e5c22da6fd595b120eeb Mon Sep 17 00:00:00 2001 From: Richard van der Hoff <1389908+richvdh@users.noreply.github.com> Date: Mon, 20 May 2019 15:54:42 +0100 Subject: Add a test room version which updates event ID format (#5210) Implements MSC1884 --- changelog.d/5210.feature | 1 + synapse/api/room_versions.py | 13 +++++++++++-- synapse/events/__init__.py | 23 ++++++++++++++++++++++- 3 files changed, 34 insertions(+), 3 deletions(-) create mode 100644 changelog.d/5210.feature (limited to 'changelog.d') diff --git a/changelog.d/5210.feature b/changelog.d/5210.feature new file mode 100644 index 0000000000..a7476bf9b9 --- /dev/null +++ b/changelog.d/5210.feature @@ -0,0 +1 @@ +Add a new room version which uses a new event ID format. diff --git a/synapse/api/room_versions.py b/synapse/api/room_versions.py index e77abe1040..485b3d0237 100644 --- a/synapse/api/room_versions.py +++ b/synapse/api/room_versions.py @@ -19,13 +19,15 @@ class EventFormatVersions(object): """This is an internal enum for tracking the version of the event format, independently from the room version. """ - V1 = 1 # $id:server format - V2 = 2 # MSC1659-style $hash format: introduced for room v3 + V1 = 1 # $id:server event id format + V2 = 2 # MSC1659-style $hash event id format: introduced for room v3 + V3 = 3 # MSC1884-style $hash format: introduced for room v4 KNOWN_EVENT_FORMAT_VERSIONS = { EventFormatVersions.V1, EventFormatVersions.V2, + EventFormatVersions.V3, } @@ -75,6 +77,12 @@ class RoomVersions(object): EventFormatVersions.V2, StateResolutionVersions.V2, ) + EVENTID_NOSLASH_TEST = RoomVersion( + "eventid-noslash-test", + RoomDisposition.UNSTABLE, + EventFormatVersions.V3, + StateResolutionVersions.V2, + ) # the version we will give rooms which are created on this server @@ -87,5 +95,6 @@ KNOWN_ROOM_VERSIONS = { RoomVersions.V2, RoomVersions.V3, RoomVersions.STATE_V2_TEST, + RoomVersions.EVENTID_NOSLASH_TEST, ) } # type: dict[str, RoomVersion] diff --git a/synapse/events/__init__.py b/synapse/events/__init__.py index 12056d5be2..badeb903fc 100644 --- a/synapse/events/__init__.py +++ b/synapse/events/__init__.py @@ -335,13 +335,32 @@ class FrozenEventV2(EventBase): return self.__repr__() def __repr__(self): - return "" % ( + return "<%s event_id='%s', type='%s', state_key='%s'>" % ( + self.__class__.__name__, self.event_id, self.get("type", None), self.get("state_key", None), ) +class FrozenEventV3(FrozenEventV2): + """FrozenEventV3, which differs from FrozenEventV2 only in the event_id format""" + format_version = EventFormatVersions.V3 # All events of this type are V3 + + @property + def event_id(self): + # We have to import this here as otherwise we get an import loop which + # is hard to break. + from synapse.crypto.event_signing import compute_event_reference_hash + + if self._event_id: + return self._event_id + self._event_id = "$" + encode_base64( + compute_event_reference_hash(self)[1], urlsafe=True + ) + return self._event_id + + def room_version_to_event_format(room_version): """Converts a room version string to the event format @@ -376,6 +395,8 @@ def event_type_from_format_version(format_version): return FrozenEvent elif format_version == EventFormatVersions.V2: return FrozenEventV2 + elif format_version == EventFormatVersions.V3: + return FrozenEventV3 else: raise Exception( "No event format %r" % (format_version,) -- cgit 1.5.1 From 24b93b9c76e0623c231abf37b84e0b841879b890 Mon Sep 17 00:00:00 2001 From: Richard van der Hoff Date: Mon, 20 May 2019 16:21:34 +0100 Subject: Revert "expose SlavedProfileStore to ClientReaderSlavedStore (#5200)" This reverts commit ce5bcefc609db40740c692bd53a1ef84ab675e8c. This caused: ``` Traceback (most recent call last): File "/usr/local/lib/python3.7/runpy.py", line 193, in _run_module_as_main "__main__", mod_spec) File "/usr/local/lib/python3.7/runpy.py", line 85, in _run_code exec(code, run_globals) File "/home/synapse/src/synapse/app/client_reader.py", line 32, in from synapse.replication.slave.storage import SlavedProfileStore ImportError: cannot import name 'SlavedProfileStore' from 'synapse.replication.slave.storage' (/home/synapse/src/synapse/replication/slave/storage/__init__.py) error starting synapse.app.client_reader('/home/synapse/config/workers/client_reader.yaml') (exit code: 1); see above for logs ``` --- changelog.d/5200.bugfix | 1 - synapse/app/client_reader.py | 2 -- 2 files changed, 3 deletions(-) delete mode 100644 changelog.d/5200.bugfix (limited to 'changelog.d') diff --git a/changelog.d/5200.bugfix b/changelog.d/5200.bugfix deleted file mode 100644 index f346c7b0cc..0000000000 --- a/changelog.d/5200.bugfix +++ /dev/null @@ -1 +0,0 @@ -Fix worker registration bug caused by ClientReaderSlavedStore being unable to see get_profileinfo. diff --git a/synapse/app/client_reader.py b/synapse/app/client_reader.py index 707f7c2f07..864f1eac48 100644 --- a/synapse/app/client_reader.py +++ b/synapse/app/client_reader.py @@ -29,7 +29,6 @@ from synapse.http.server import JsonResource from synapse.http.site import SynapseSite from synapse.metrics import RegistryProxy from synapse.metrics.resource import METRICS_PREFIX, MetricsResource -from synapse.replication.slave.storage import SlavedProfileStore from synapse.replication.slave.storage._base import BaseSlavedStore from synapse.replication.slave.storage.account_data import SlavedAccountDataStore from synapse.replication.slave.storage.appservice import SlavedApplicationServiceStore @@ -84,7 +83,6 @@ class ClientReaderSlavedStore( SlavedTransactionStore, SlavedClientIpStore, BaseSlavedStore, - SlavedProfileStore, ): pass -- cgit 1.5.1 From 0620dd49db505a7830486c8798eabc455764c79c Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Mon, 20 May 2019 17:40:24 +0100 Subject: Newsfile --- changelog.d/5212.feature | 1 + 1 file changed, 1 insertion(+) create mode 100644 changelog.d/5212.feature (limited to 'changelog.d') diff --git a/changelog.d/5212.feature b/changelog.d/5212.feature new file mode 100644 index 0000000000..747098c166 --- /dev/null +++ b/changelog.d/5212.feature @@ -0,0 +1 @@ +Add experimental support for relations (aka reactions and edits). -- cgit 1.5.1 From 04d53794d6848cca2567d67f494ba8405d0bf1cf Mon Sep 17 00:00:00 2001 From: Richard van der Hoff <1389908+richvdh@users.noreply.github.com> Date: Tue, 21 May 2019 13:47:25 +0100 Subject: Fix error handling for rooms whose versions are unknown. (#5219) If we remove support for a particular room version, we should behave more gracefully. This should make client requests fail with a 400 rather than a 500, and will ignore individiual PDUs in a federation transaction, rather than the whole transaction. --- changelog.d/5219.bugfix | 1 + synapse/api/errors.py | 16 +++++++++++++++- synapse/events/__init__.py | 8 ++++++-- synapse/events/builder.py | 6 +++--- synapse/federation/federation_server.py | 14 +++++++++++++- 5 files changed, 38 insertions(+), 7 deletions(-) create mode 100644 changelog.d/5219.bugfix (limited to 'changelog.d') diff --git a/changelog.d/5219.bugfix b/changelog.d/5219.bugfix new file mode 100644 index 0000000000..c1e17adc5d --- /dev/null +++ b/changelog.d/5219.bugfix @@ -0,0 +1 @@ +Fix error handling for rooms whose versions are unknown. diff --git a/synapse/api/errors.py b/synapse/api/errors.py index ff89259dec..e91697049c 100644 --- a/synapse/api/errors.py +++ b/synapse/api/errors.py @@ -328,9 +328,23 @@ class RoomKeysVersionError(SynapseError): self.current_version = current_version +class UnsupportedRoomVersionError(SynapseError): + """The client's request to create a room used a room version that the server does + not support.""" + def __init__(self): + super(UnsupportedRoomVersionError, self).__init__( + code=400, + msg="Homeserver does not support this room version", + errcode=Codes.UNSUPPORTED_ROOM_VERSION, + ) + + class IncompatibleRoomVersionError(SynapseError): - """A server is trying to join a room whose version it does not support.""" + """A server is trying to join a room whose version it does not support. + Unlike UnsupportedRoomVersionError, it is specific to the case of the make_join + failing. + """ def __init__(self, room_version): super(IncompatibleRoomVersionError, self).__init__( code=400, diff --git a/synapse/events/__init__.py b/synapse/events/__init__.py index badeb903fc..1edd19cc13 100644 --- a/synapse/events/__init__.py +++ b/synapse/events/__init__.py @@ -21,6 +21,7 @@ import six from unpaddedbase64 import encode_base64 +from synapse.api.errors import UnsupportedRoomVersionError from synapse.api.room_versions import KNOWN_ROOM_VERSIONS, EventFormatVersions from synapse.util.caches import intern_dict from synapse.util.frozenutils import freeze @@ -369,12 +370,15 @@ def room_version_to_event_format(room_version): Returns: int + + Raises: + UnsupportedRoomVersionError if the room version is unknown """ v = KNOWN_ROOM_VERSIONS.get(room_version) if not v: - # We should have already checked version, so this should not happen - raise RuntimeError("Unrecognized room version %s" % (room_version,)) + # this can happen if support is withdrawn for a room version + raise UnsupportedRoomVersionError() return v.event_format diff --git a/synapse/events/builder.py b/synapse/events/builder.py index fba27177c7..1fe995f212 100644 --- a/synapse/events/builder.py +++ b/synapse/events/builder.py @@ -18,6 +18,7 @@ import attr from twisted.internet import defer from synapse.api.constants import MAX_DEPTH +from synapse.api.errors import UnsupportedRoomVersionError from synapse.api.room_versions import ( KNOWN_EVENT_FORMAT_VERSIONS, KNOWN_ROOM_VERSIONS, @@ -178,9 +179,8 @@ class EventBuilderFactory(object): """ v = KNOWN_ROOM_VERSIONS.get(room_version) if not v: - raise Exception( - "No event format defined for version %r" % (room_version,) - ) + # this can happen if support is withdrawn for a room version + raise UnsupportedRoomVersionError() return self.for_room_version(v, key_values) def for_room_version(self, room_version, key_values): diff --git a/synapse/federation/federation_server.py b/synapse/federation/federation_server.py index df60828dba..4c28c1dc3c 100644 --- a/synapse/federation/federation_server.py +++ b/synapse/federation/federation_server.py @@ -33,6 +33,7 @@ from synapse.api.errors import ( IncompatibleRoomVersionError, NotFoundError, SynapseError, + UnsupportedRoomVersionError, ) from synapse.api.room_versions import KNOWN_ROOM_VERSIONS from synapse.crypto.event_signing import compute_event_signature @@ -198,11 +199,22 @@ class FederationServer(FederationBase): try: room_version = yield self.store.get_room_version(room_id) - format_ver = room_version_to_event_format(room_version) except NotFoundError: logger.info("Ignoring PDU for unknown room_id: %s", room_id) continue + try: + format_ver = room_version_to_event_format(room_version) + except UnsupportedRoomVersionError: + # this can happen if support for a given room version is withdrawn, + # so that we still get events for said room. + logger.info( + "Ignoring PDU for room %s with unknown version %s", + room_id, + room_version, + ) + continue + event = event_from_pdu_json(p, format_ver) pdus_by_room.setdefault(room_id, []).append(event) -- cgit 1.5.1 From bab3eddac46a0d7bdfccfc980eececbf8f60f721 Mon Sep 17 00:00:00 2001 From: Richard van der Hoff <1389908+richvdh@users.noreply.github.com> Date: Tue, 21 May 2019 15:58:01 +0100 Subject: Pin eliot to <1.8 on python 3.5.2 (#5218) * Pin eliot to <1.8 on python 3.5.2 Fixes https://github.com/matrix-org/synapse/issues/5199 * Add support for 'markers' to python_dependencies * tell xargs not to strip quotes --- changelog.d/5218.bugfix | 1 + synapse/python_dependencies.py | 40 ++++++++++++++++++++++++++++++++++------ tox.ini | 2 +- 3 files changed, 36 insertions(+), 7 deletions(-) create mode 100644 changelog.d/5218.bugfix (limited to 'changelog.d') diff --git a/changelog.d/5218.bugfix b/changelog.d/5218.bugfix new file mode 100644 index 0000000000..cd624ecfd0 --- /dev/null +++ b/changelog.d/5218.bugfix @@ -0,0 +1 @@ +Fix incompatibility between ACME support and Python 3.5.2. diff --git a/synapse/python_dependencies.py b/synapse/python_dependencies.py index fdcfb90a7e..e3f828c4bb 100644 --- a/synapse/python_dependencies.py +++ b/synapse/python_dependencies.py @@ -16,7 +16,12 @@ import logging -from pkg_resources import DistributionNotFound, VersionConflict, get_distribution +from pkg_resources import ( + DistributionNotFound, + Requirement, + VersionConflict, + get_provider, +) logger = logging.getLogger(__name__) @@ -91,7 +96,13 @@ CONDITIONAL_REQUIREMENTS = { # ACME support is required to provision TLS certificates from authorities # that use the protocol, such as Let's Encrypt. - "acme": ["txacme>=0.9.2"], + "acme": [ + "txacme>=0.9.2", + + # txacme depends on eliot. Eliot 1.8.0 is incompatible with + # python 3.5.2, as per https://github.com/itamarst/eliot/issues/418 + 'eliot<1.8.0;python_version<"3.5.3"', + ], "saml2": ["pysaml2>=4.5.0"], "systemd": ["systemd-python>=231"], @@ -125,10 +136,10 @@ class DependencyException(Exception): @property def dependencies(self): for i in self.args[0]: - yield '"' + i + '"' + yield "'" + i + "'" -def check_requirements(for_feature=None, _get_distribution=get_distribution): +def check_requirements(for_feature=None): deps_needed = [] errors = [] @@ -139,7 +150,7 @@ def check_requirements(for_feature=None, _get_distribution=get_distribution): for dependency in reqs: try: - _get_distribution(dependency) + _check_requirement(dependency) except VersionConflict as e: deps_needed.append(dependency) errors.append( @@ -157,7 +168,7 @@ def check_requirements(for_feature=None, _get_distribution=get_distribution): for dependency in OPTS: try: - _get_distribution(dependency) + _check_requirement(dependency) except VersionConflict as e: deps_needed.append(dependency) errors.append( @@ -175,6 +186,23 @@ def check_requirements(for_feature=None, _get_distribution=get_distribution): raise DependencyException(deps_needed) +def _check_requirement(dependency_string): + """Parses a dependency string, and checks if the specified requirement is installed + + Raises: + VersionConflict if the requirement is installed, but with the the wrong version + DistributionNotFound if nothing is found to provide the requirement + """ + req = Requirement.parse(dependency_string) + + # first check if the markers specify that this requirement needs installing + if req.marker is not None and not req.marker.evaluate(): + # not required for this environment + return + + get_provider(req) + + if __name__ == "__main__": import sys diff --git a/tox.ini b/tox.ini index d0e519ce46..543b232ae7 100644 --- a/tox.ini +++ b/tox.ini @@ -94,7 +94,7 @@ commands = # Make all greater-thans equals so we test the oldest version of our direct # dependencies, but make the pyopenssl 17.0, which can work against an # OpenSSL 1.1 compiled cryptography (as older ones don't compile on Travis). - /bin/sh -c 'python -m synapse.python_dependencies | sed -e "s/>=/==/g" -e "s/psycopg2==2.6//" -e "s/pyopenssl==16.0.0/pyopenssl==17.0.0/" | xargs pip install' + /bin/sh -c 'python -m synapse.python_dependencies | sed -e "s/>=/==/g" -e "s/psycopg2==2.6//" -e "s/pyopenssl==16.0.0/pyopenssl==17.0.0/" | xargs -d"\n" pip install' # Add this so that coverage will run on subprocesses /bin/sh -c 'echo "import coverage; coverage.process_startup()" > {envsitepackagesdir}/../sitecustomize.py' -- cgit 1.5.1 From c4aef549ad1f55661675a789f89fe9e041fac874 Mon Sep 17 00:00:00 2001 From: Richard van der Hoff <1389908+richvdh@users.noreply.github.com> Date: Tue, 21 May 2019 16:10:54 +0100 Subject: Exclude soft-failed events from fwd-extremity candidates. (#5146) When considering the candidates to be forward-extremities, we must exclude soft failures. Hopefully fixes #5090. --- changelog.d/5146.bugfix | 1 + synapse/handlers/federation.py | 7 ++++++- synapse/storage/events.py | 9 +++++++-- 3 files changed, 14 insertions(+), 3 deletions(-) create mode 100644 changelog.d/5146.bugfix (limited to 'changelog.d') diff --git a/changelog.d/5146.bugfix b/changelog.d/5146.bugfix new file mode 100644 index 0000000000..a54abed92b --- /dev/null +++ b/changelog.d/5146.bugfix @@ -0,0 +1 @@ +Exclude soft-failed events from forward-extremity candidates: fixes "No forward extremities left!" error. diff --git a/synapse/handlers/federation.py b/synapse/handlers/federation.py index 0684778882..2202ed699a 100644 --- a/synapse/handlers/federation.py +++ b/synapse/handlers/federation.py @@ -1916,6 +1916,11 @@ class FederationHandler(BaseHandler): event.room_id, latest_event_ids=extrem_ids, ) + logger.debug( + "Doing soft-fail check for %s: state %s", + event.event_id, current_state_ids, + ) + # Now check if event pass auth against said current state auth_types = auth_types_for_event(event) current_state_ids = [ @@ -1932,7 +1937,7 @@ class FederationHandler(BaseHandler): self.auth.check(room_version, event, auth_events=current_auth_events) except AuthError as e: logger.warn( - "Failed current state auth resolution for %r because %s", + "Soft-failing %r because %s", event, e, ) event.internal_metadata.soft_failed = True diff --git a/synapse/storage/events.py b/synapse/storage/events.py index 881d6d0126..2ffc27ff41 100644 --- a/synapse/storage/events.py +++ b/synapse/storage/events.py @@ -575,10 +575,11 @@ class EventsStore( def _get_events(txn, batch): sql = """ - SELECT prev_event_id + SELECT prev_event_id, internal_metadata FROM event_edges INNER JOIN events USING (event_id) LEFT JOIN rejections USING (event_id) + LEFT JOIN event_json USING (event_id) WHERE prev_event_id IN (%s) AND NOT events.outlier @@ -588,7 +589,11 @@ class EventsStore( ) txn.execute(sql, batch) - results.extend(r[0] for r in txn) + results.extend( + r[0] + for r in txn + if not json.loads(r[1]).get("soft_failed") + ) for chunk in batch_iter(event_ids, 100): yield self.runInteraction("_get_events_which_are_prevs", _get_events, chunk) -- cgit 1.5.1 From 17f68048374cda8bc639d7c048ae21624a124635 Mon Sep 17 00:00:00 2001 From: Richard van der Hoff <1389908+richvdh@users.noreply.github.com> Date: Tue, 21 May 2019 16:22:54 +0100 Subject: Introduce room v4 which updates event ID format. (#5217) Implements https://github.com/matrix-org/matrix-doc/pull/2002. --- changelog.d/5210.feature | 2 +- changelog.d/5217.feature | 1 + synapse/api/room_versions.py | 8 ++++---- 3 files changed, 6 insertions(+), 5 deletions(-) create mode 100644 changelog.d/5217.feature (limited to 'changelog.d') diff --git a/changelog.d/5210.feature b/changelog.d/5210.feature index a7476bf9b9..c78325a6ac 100644 --- a/changelog.d/5210.feature +++ b/changelog.d/5210.feature @@ -1 +1 @@ -Add a new room version which uses a new event ID format. +Add a room version 4 which uses a new event ID format, as per [MSC2002](https://github.com/matrix-org/matrix-doc/pull/2002). diff --git a/changelog.d/5217.feature b/changelog.d/5217.feature new file mode 100644 index 0000000000..c78325a6ac --- /dev/null +++ b/changelog.d/5217.feature @@ -0,0 +1 @@ +Add a room version 4 which uses a new event ID format, as per [MSC2002](https://github.com/matrix-org/matrix-doc/pull/2002). diff --git a/synapse/api/room_versions.py b/synapse/api/room_versions.py index 485b3d0237..b2895355a8 100644 --- a/synapse/api/room_versions.py +++ b/synapse/api/room_versions.py @@ -77,9 +77,9 @@ class RoomVersions(object): EventFormatVersions.V2, StateResolutionVersions.V2, ) - EVENTID_NOSLASH_TEST = RoomVersion( - "eventid-noslash-test", - RoomDisposition.UNSTABLE, + V4 = RoomVersion( + "4", + RoomDisposition.STABLE, EventFormatVersions.V3, StateResolutionVersions.V2, ) @@ -95,6 +95,6 @@ KNOWN_ROOM_VERSIONS = { RoomVersions.V2, RoomVersions.V3, RoomVersions.STATE_V2_TEST, - RoomVersions.EVENTID_NOSLASH_TEST, + RoomVersions.V4, ) } # type: dict[str, RoomVersion] -- cgit 1.5.1 From 959550b645a0752f6a8592d90fff44683b189523 Mon Sep 17 00:00:00 2001 From: Richard van der Hoff Date: Tue, 21 May 2019 16:51:49 +0100 Subject: 0.99.5rc1 --- CHANGES.md | 39 +++++++++++++++++++++++++++++++++++++++ changelog.d/3484.misc | 1 - changelog.d/5039.bugfix | 1 - changelog.d/5043.feature | 1 - changelog.d/5146.bugfix | 1 - changelog.d/5171.misc | 1 - changelog.d/5174.bugfix | 1 - changelog.d/5177.bugfix | 1 - changelog.d/5181.feature | 1 - changelog.d/5183.misc | 1 - changelog.d/5184.misc | 1 - changelog.d/5185.misc | 1 - changelog.d/5187.bugfix | 1 - changelog.d/5190.feature | 1 - changelog.d/5191.misc | 1 - changelog.d/5196.feature | 1 - changelog.d/5197.misc | 1 - changelog.d/5198.bugfix | 1 - changelog.d/5204.feature | 1 - changelog.d/5209.feature | 1 - changelog.d/5210.feature | 1 - changelog.d/5211.feature | 1 - changelog.d/5217.feature | 1 - changelog.d/5218.bugfix | 1 - changelog.d/5219.bugfix | 1 - synapse/__init__.py | 2 +- 26 files changed, 40 insertions(+), 25 deletions(-) delete mode 100644 changelog.d/3484.misc delete mode 100644 changelog.d/5039.bugfix delete mode 100644 changelog.d/5043.feature delete mode 100644 changelog.d/5146.bugfix delete mode 100644 changelog.d/5171.misc delete mode 100644 changelog.d/5174.bugfix delete mode 100644 changelog.d/5177.bugfix delete mode 100644 changelog.d/5181.feature delete mode 100644 changelog.d/5183.misc delete mode 100644 changelog.d/5184.misc delete mode 100644 changelog.d/5185.misc delete mode 100644 changelog.d/5187.bugfix delete mode 100644 changelog.d/5190.feature delete mode 100644 changelog.d/5191.misc delete mode 100644 changelog.d/5196.feature delete mode 100644 changelog.d/5197.misc delete mode 100644 changelog.d/5198.bugfix delete mode 100644 changelog.d/5204.feature delete mode 100644 changelog.d/5209.feature delete mode 100644 changelog.d/5210.feature delete mode 100644 changelog.d/5211.feature delete mode 100644 changelog.d/5217.feature delete mode 100644 changelog.d/5218.bugfix delete mode 100644 changelog.d/5219.bugfix (limited to 'changelog.d') diff --git a/CHANGES.md b/CHANGES.md index 1e9c3cf953..d23166caa8 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -1,3 +1,42 @@ +Synapse 0.99.5rc1 (2019-05-21) +============================== + +Features +-------- + +- Add ability to blacklist IP ranges for the federation client. ([\#5043](https://github.com/matrix-org/synapse/issues/5043)) +- Ratelimiting configuration for clients sending messages and the federation server has been altered to match login ratelimiting. The old configuration names will continue working. Check the sample config for details of the new names. ([\#5181](https://github.com/matrix-org/synapse/issues/5181)) +- Drop support for the undocumented /_matrix/client/v2_alpha API prefix. ([\#5190](https://github.com/matrix-org/synapse/issues/5190)) +- Add an option to disable per-room profiles. ([\#5196](https://github.com/matrix-org/synapse/issues/5196)) +- Stick an expiration date to any registered user missing one at startup if account validity is enabled. ([\#5204](https://github.com/matrix-org/synapse/issues/5204)) +- Add experimental support for relations (aka reactions and edits). ([\#5209](https://github.com/matrix-org/synapse/issues/5209), [\#5211](https://github.com/matrix-org/synapse/issues/5211)) +- Add a room version 4 which uses a new event ID format, as per [MSC2002](https://github.com/matrix-org/matrix-doc/pull/2002). ([\#5210](https://github.com/matrix-org/synapse/issues/5210), [\#5217](https://github.com/matrix-org/synapse/issues/5217)) + + +Bugfixes +-------- + +- Fix image orientation when generating thumbnails (needs pillow>=4.3.0). Contributed by Pau Rodriguez-Estivill. ([\#5039](https://github.com/matrix-org/synapse/issues/5039)) +- Exclude soft-failed events from forward-extremity candidates: fixes "No forward extremities left!" error. ([\#5146](https://github.com/matrix-org/synapse/issues/5146)) +- Re-order stages in registration flows such that msisdn and email verification are done last. ([\#5174](https://github.com/matrix-org/synapse/issues/5174)) +- Fix 3pid guest invites. ([\#5177](https://github.com/matrix-org/synapse/issues/5177)) +- Fix a bug where the register endpoint would fail with M_THREEPID_IN_USE instead of returning an account previously registered in the same session. ([\#5187](https://github.com/matrix-org/synapse/issues/5187)) +- Prevent registration for user ids that are to long to fit into a state key. Contributed by Reid Anderson. ([\#5198](https://github.com/matrix-org/synapse/issues/5198)) +- Fix incompatibility between ACME support and Python 3.5.2. ([\#5218](https://github.com/matrix-org/synapse/issues/5218)) +- Fix error handling for rooms whose versions are unknown. ([\#5219](https://github.com/matrix-org/synapse/issues/5219)) + + +Internal Changes +---------------- + +- Make /sync attempt to return device updates for both joined and invited users. Note that this doesn't currently work correctly due to other bugs. ([\#3484](https://github.com/matrix-org/synapse/issues/3484)) +- Update tests to consistently be configured via the same code that is used when loading from configuration files. ([\#5171](https://github.com/matrix-org/synapse/issues/5171), [\#5185](https://github.com/matrix-org/synapse/issues/5185)) +- Allow client event serialization to be async. ([\#5183](https://github.com/matrix-org/synapse/issues/5183)) +- Expose DataStore._get_events as get_events_as_list. ([\#5184](https://github.com/matrix-org/synapse/issues/5184)) +- Make generating SQL bounds for pagination generic. ([\#5191](https://github.com/matrix-org/synapse/issues/5191)) +- Stop telling people to install the optional dependencies by default. ([\#5197](https://github.com/matrix-org/synapse/issues/5197)) + + Synapse 0.99.4 (2019-05-15) =========================== diff --git a/changelog.d/3484.misc b/changelog.d/3484.misc deleted file mode 100644 index 3645849844..0000000000 --- a/changelog.d/3484.misc +++ /dev/null @@ -1 +0,0 @@ -Make /sync attempt to return device updates for both joined and invited users. Note that this doesn't currently work correctly due to other bugs. diff --git a/changelog.d/5039.bugfix b/changelog.d/5039.bugfix deleted file mode 100644 index 212cff7ae8..0000000000 --- a/changelog.d/5039.bugfix +++ /dev/null @@ -1 +0,0 @@ -Fix image orientation when generating thumbnails (needs pillow>=4.3.0). Contributed by Pau Rodriguez-Estivill. diff --git a/changelog.d/5043.feature b/changelog.d/5043.feature deleted file mode 100644 index 0f1e0ee30e..0000000000 --- a/changelog.d/5043.feature +++ /dev/null @@ -1 +0,0 @@ -Add ability to blacklist IP ranges for the federation client. diff --git a/changelog.d/5146.bugfix b/changelog.d/5146.bugfix deleted file mode 100644 index a54abed92b..0000000000 --- a/changelog.d/5146.bugfix +++ /dev/null @@ -1 +0,0 @@ -Exclude soft-failed events from forward-extremity candidates: fixes "No forward extremities left!" error. diff --git a/changelog.d/5171.misc b/changelog.d/5171.misc deleted file mode 100644 index d148b03b51..0000000000 --- a/changelog.d/5171.misc +++ /dev/null @@ -1 +0,0 @@ -Update tests to consistently be configured via the same code that is used when loading from configuration files. diff --git a/changelog.d/5174.bugfix b/changelog.d/5174.bugfix deleted file mode 100644 index 0f26d46b2c..0000000000 --- a/changelog.d/5174.bugfix +++ /dev/null @@ -1 +0,0 @@ -Re-order stages in registration flows such that msisdn and email verification are done last. diff --git a/changelog.d/5177.bugfix b/changelog.d/5177.bugfix deleted file mode 100644 index c2f1644ae5..0000000000 --- a/changelog.d/5177.bugfix +++ /dev/null @@ -1 +0,0 @@ -Fix 3pid guest invites. diff --git a/changelog.d/5181.feature b/changelog.d/5181.feature deleted file mode 100644 index 5ce13aa2ea..0000000000 --- a/changelog.d/5181.feature +++ /dev/null @@ -1 +0,0 @@ -Ratelimiting configuration for clients sending messages and the federation server has been altered to match login ratelimiting. The old configuration names will continue working. Check the sample config for details of the new names. diff --git a/changelog.d/5183.misc b/changelog.d/5183.misc deleted file mode 100644 index a8970f29eb..0000000000 --- a/changelog.d/5183.misc +++ /dev/null @@ -1 +0,0 @@ -Allow client event serialization to be async. diff --git a/changelog.d/5184.misc b/changelog.d/5184.misc deleted file mode 100644 index 1588bdef6c..0000000000 --- a/changelog.d/5184.misc +++ /dev/null @@ -1 +0,0 @@ -Expose DataStore._get_events as get_events_as_list. diff --git a/changelog.d/5185.misc b/changelog.d/5185.misc deleted file mode 100644 index d148b03b51..0000000000 --- a/changelog.d/5185.misc +++ /dev/null @@ -1 +0,0 @@ -Update tests to consistently be configured via the same code that is used when loading from configuration files. diff --git a/changelog.d/5187.bugfix b/changelog.d/5187.bugfix deleted file mode 100644 index df176cf5b2..0000000000 --- a/changelog.d/5187.bugfix +++ /dev/null @@ -1 +0,0 @@ -Fix a bug where the register endpoint would fail with M_THREEPID_IN_USE instead of returning an account previously registered in the same session. diff --git a/changelog.d/5190.feature b/changelog.d/5190.feature deleted file mode 100644 index 34904aa7a8..0000000000 --- a/changelog.d/5190.feature +++ /dev/null @@ -1 +0,0 @@ -Drop support for the undocumented /_matrix/client/v2_alpha API prefix. diff --git a/changelog.d/5191.misc b/changelog.d/5191.misc deleted file mode 100644 index e0615fec9c..0000000000 --- a/changelog.d/5191.misc +++ /dev/null @@ -1 +0,0 @@ -Make generating SQL bounds for pagination generic. diff --git a/changelog.d/5196.feature b/changelog.d/5196.feature deleted file mode 100644 index 1ffb928f62..0000000000 --- a/changelog.d/5196.feature +++ /dev/null @@ -1 +0,0 @@ -Add an option to disable per-room profiles. diff --git a/changelog.d/5197.misc b/changelog.d/5197.misc deleted file mode 100644 index fca1d86b2e..0000000000 --- a/changelog.d/5197.misc +++ /dev/null @@ -1 +0,0 @@ -Stop telling people to install the optional dependencies by default. diff --git a/changelog.d/5198.bugfix b/changelog.d/5198.bugfix deleted file mode 100644 index c6b156f17d..0000000000 --- a/changelog.d/5198.bugfix +++ /dev/null @@ -1 +0,0 @@ -Prevent registration for user ids that are to long to fit into a state key. Contributed by Reid Anderson. \ No newline at end of file diff --git a/changelog.d/5204.feature b/changelog.d/5204.feature deleted file mode 100644 index 2a7212ca18..0000000000 --- a/changelog.d/5204.feature +++ /dev/null @@ -1 +0,0 @@ -Stick an expiration date to any registered user missing one at startup if account validity is enabled. diff --git a/changelog.d/5209.feature b/changelog.d/5209.feature deleted file mode 100644 index 747098c166..0000000000 --- a/changelog.d/5209.feature +++ /dev/null @@ -1 +0,0 @@ -Add experimental support for relations (aka reactions and edits). diff --git a/changelog.d/5210.feature b/changelog.d/5210.feature deleted file mode 100644 index c78325a6ac..0000000000 --- a/changelog.d/5210.feature +++ /dev/null @@ -1 +0,0 @@ -Add a room version 4 which uses a new event ID format, as per [MSC2002](https://github.com/matrix-org/matrix-doc/pull/2002). diff --git a/changelog.d/5211.feature b/changelog.d/5211.feature deleted file mode 100644 index 747098c166..0000000000 --- a/changelog.d/5211.feature +++ /dev/null @@ -1 +0,0 @@ -Add experimental support for relations (aka reactions and edits). diff --git a/changelog.d/5217.feature b/changelog.d/5217.feature deleted file mode 100644 index c78325a6ac..0000000000 --- a/changelog.d/5217.feature +++ /dev/null @@ -1 +0,0 @@ -Add a room version 4 which uses a new event ID format, as per [MSC2002](https://github.com/matrix-org/matrix-doc/pull/2002). diff --git a/changelog.d/5218.bugfix b/changelog.d/5218.bugfix deleted file mode 100644 index cd624ecfd0..0000000000 --- a/changelog.d/5218.bugfix +++ /dev/null @@ -1 +0,0 @@ -Fix incompatibility between ACME support and Python 3.5.2. diff --git a/changelog.d/5219.bugfix b/changelog.d/5219.bugfix deleted file mode 100644 index c1e17adc5d..0000000000 --- a/changelog.d/5219.bugfix +++ /dev/null @@ -1 +0,0 @@ -Fix error handling for rooms whose versions are unknown. diff --git a/synapse/__init__.py b/synapse/__init__.py index bf9e810da6..42af03b53d 100644 --- a/synapse/__init__.py +++ b/synapse/__init__.py @@ -27,4 +27,4 @@ try: except ImportError: pass -__version__ = "0.99.4" +__version__ = "0.99.5rc1" -- cgit 1.5.1 From 9259cd4bee2b40f6de9f99361d33473e78f43bef Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 21 May 2019 17:06:21 +0100 Subject: Newsfile --- changelog.d/5203.feature | 1 + 1 file changed, 1 insertion(+) create mode 100644 changelog.d/5203.feature (limited to 'changelog.d') diff --git a/changelog.d/5203.feature b/changelog.d/5203.feature new file mode 100644 index 0000000000..747098c166 --- /dev/null +++ b/changelog.d/5203.feature @@ -0,0 +1 @@ +Add experimental support for relations (aka reactions and edits). -- cgit 1.5.1 From e26e6b3230f0b55376f0f3bf823dd789ac7064d0 Mon Sep 17 00:00:00 2001 From: Richard van der Hoff Date: Tue, 21 May 2019 17:37:19 +0100 Subject: update changelog --- CHANGES.md | 2 +- changelog.d/5203.feature | 1 - changelog.d/5212.feature | 1 - 3 files changed, 1 insertion(+), 3 deletions(-) delete mode 100644 changelog.d/5203.feature delete mode 100644 changelog.d/5212.feature (limited to 'changelog.d') diff --git a/CHANGES.md b/CHANGES.md index 7eab35be30..25ceec8b48 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -9,7 +9,7 @@ Features - Drop support for the undocumented /_matrix/client/v2_alpha API prefix. ([\#5190](https://github.com/matrix-org/synapse/issues/5190)) - Add an option to disable per-room profiles. ([\#5196](https://github.com/matrix-org/synapse/issues/5196)) - Stick an expiration date to any registered user missing one at startup if account validity is enabled. ([\#5204](https://github.com/matrix-org/synapse/issues/5204)) -- Add experimental support for relations (aka reactions and edits). ([\#5209](https://github.com/matrix-org/synapse/issues/5209), [\#5211](https://github.com/matrix-org/synapse/issues/5211)) +- Add experimental support for relations (aka reactions and edits). ([\#5209](https://github.com/matrix-org/synapse/issues/5209), [\#5211](https://github.com/matrix-org/synapse/issues/5211), [\#5203](https://github.com/matrix-org/synapse/issues/5203), [\#5212](https://github.com/matrix-org/synapse/issues/5212)) - Add a room version 4 which uses a new event ID format, as per [MSC2002](https://github.com/matrix-org/matrix-doc/pull/2002). ([\#5210](https://github.com/matrix-org/synapse/issues/5210), [\#5217](https://github.com/matrix-org/synapse/issues/5217)) diff --git a/changelog.d/5203.feature b/changelog.d/5203.feature deleted file mode 100644 index 747098c166..0000000000 --- a/changelog.d/5203.feature +++ /dev/null @@ -1 +0,0 @@ -Add experimental support for relations (aka reactions and edits). diff --git a/changelog.d/5212.feature b/changelog.d/5212.feature deleted file mode 100644 index 747098c166..0000000000 --- a/changelog.d/5212.feature +++ /dev/null @@ -1 +0,0 @@ -Add experimental support for relations (aka reactions and edits). -- cgit 1.5.1 From df9d9005448d837c5e1a2b75edb5730e2062b0f2 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Wed, 29 May 2019 11:56:24 +0100 Subject: Correctly filter out extremities with soft failed prevs (#5274) When we receive a soft failed event we, correctly, *do not* update the forward extremity table with the event. However, if we later receive an event that references the soft failed event we then need to remove the soft failed events prev events from the forward extremities table, otherwise we just build up forward extremities. Fixes #5269 --- changelog.d/5274.bugfix | 1 + synapse/storage/events.py | 82 +++++++++++++++++++++++++++++++++++++++++++++-- 2 files changed, 80 insertions(+), 3 deletions(-) create mode 100644 changelog.d/5274.bugfix (limited to 'changelog.d') diff --git a/changelog.d/5274.bugfix b/changelog.d/5274.bugfix new file mode 100644 index 0000000000..9e14d20289 --- /dev/null +++ b/changelog.d/5274.bugfix @@ -0,0 +1 @@ +Fix bug where we leaked extremities when we soft failed events, leading to performance degradation. diff --git a/synapse/storage/events.py b/synapse/storage/events.py index 2ffc27ff41..6e9f3d1dc0 100644 --- a/synapse/storage/events.py +++ b/synapse/storage/events.py @@ -554,10 +554,18 @@ class EventsStore( e_id for event in new_events for e_id in event.prev_event_ids() ) - # Finally, remove any events which are prev_events of any existing events. + # Remove any events which are prev_events of any existing events. existing_prevs = yield self._get_events_which_are_prevs(result) result.difference_update(existing_prevs) + # Finally handle the case where the new events have soft-failed prev + # events. If they do we need to remove them and their prev events, + # otherwise we end up with dangling extremities. + existing_prevs = yield self._get_prevs_before_rejected( + e_id for event in new_events for e_id in event.prev_event_ids() + ) + result.difference_update(existing_prevs) + defer.returnValue(result) @defer.inlineCallbacks @@ -573,7 +581,7 @@ class EventsStore( """ results = [] - def _get_events(txn, batch): + def _get_events_which_are_prevs_txn(txn, batch): sql = """ SELECT prev_event_id, internal_metadata FROM event_edges @@ -596,10 +604,78 @@ class EventsStore( ) for chunk in batch_iter(event_ids, 100): - yield self.runInteraction("_get_events_which_are_prevs", _get_events, chunk) + yield self.runInteraction( + "_get_events_which_are_prevs", + _get_events_which_are_prevs_txn, + chunk, + ) defer.returnValue(results) + @defer.inlineCallbacks + def _get_prevs_before_rejected(self, event_ids): + """Get soft-failed ancestors to remove from the extremities. + + Given a set of events, find all those that have been soft-failed or + rejected. Returns those soft failed/rejected events and their prev + events (whether soft-failed/rejected or not), and recurses up the + prev-event graph until it finds no more soft-failed/rejected events. + + This is used to find extremities that are ancestors of new events, but + are separated by soft failed events. + + Args: + event_ids (Iterable[str]): Events to find prev events for. Note + that these must have already been persisted. + + Returns: + Deferred[set[str]] + """ + + # The set of event_ids to return. This includes all soft-failed events + # and their prev events. + existing_prevs = set() + + def _get_prevs_before_rejected_txn(txn, batch): + to_recursively_check = batch + + while to_recursively_check: + sql = """ + SELECT + event_id, prev_event_id, internal_metadata, + rejections.event_id IS NOT NULL + FROM event_edges + INNER JOIN events USING (event_id) + LEFT JOIN rejections USING (event_id) + LEFT JOIN event_json USING (event_id) + WHERE + event_id IN (%s) + AND NOT events.outlier + """ % ( + ",".join("?" for _ in to_recursively_check), + ) + + txn.execute(sql, to_recursively_check) + to_recursively_check = [] + + for event_id, prev_event_id, metadata, rejected in txn: + if prev_event_id in existing_prevs: + continue + + soft_failed = json.loads(metadata).get("soft_failed") + if soft_failed or rejected: + to_recursively_check.append(prev_event_id) + existing_prevs.add(prev_event_id) + + for chunk in batch_iter(event_ids, 100): + yield self.runInteraction( + "_get_prevs_before_rejected", + _get_prevs_before_rejected_txn, + chunk, + ) + + defer.returnValue(existing_prevs) + @defer.inlineCallbacks def _get_new_state_after_events( self, room_id, events_context, old_latest_event_ids, new_latest_event_ids -- cgit 1.5.1 From 9f5268388abb266260c30e9855da35a0b6b11bcb Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 28 May 2019 18:56:02 +0100 Subject: Newsfile --- changelog.d/5278.bugfix | 1 + 1 file changed, 1 insertion(+) create mode 100644 changelog.d/5278.bugfix (limited to 'changelog.d') diff --git a/changelog.d/5278.bugfix b/changelog.d/5278.bugfix new file mode 100644 index 0000000000..9e14d20289 --- /dev/null +++ b/changelog.d/5278.bugfix @@ -0,0 +1 @@ +Fix bug where we leaked extremities when we soft failed events, leading to performance degradation. -- cgit 1.5.1 From f5c7f90d7278f199523457ade9587d284b3ce39b Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Thu, 30 May 2019 15:05:26 +0100 Subject: Newsfile --- changelog.d/5291.bugfix | 1 + 1 file changed, 1 insertion(+) create mode 100644 changelog.d/5291.bugfix (limited to 'changelog.d') diff --git a/changelog.d/5291.bugfix b/changelog.d/5291.bugfix new file mode 100644 index 0000000000..9e14d20289 --- /dev/null +++ b/changelog.d/5291.bugfix @@ -0,0 +1 @@ +Fix bug where we leaked extremities when we soft failed events, leading to performance degradation. -- cgit 1.5.1 From c831748f4d243d74e9a3fd2042bc2b35cc30f961 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Thu, 30 May 2019 16:28:52 +0100 Subject: 0.99.5.2 --- CHANGES.md | 9 +++++++++ changelog.d/5274.bugfix | 1 - changelog.d/5278.bugfix | 1 - changelog.d/5291.bugfix | 1 - debian/changelog | 6 ++++++ synapse/__init__.py | 2 +- 6 files changed, 16 insertions(+), 4 deletions(-) delete mode 100644 changelog.d/5274.bugfix delete mode 100644 changelog.d/5278.bugfix delete mode 100644 changelog.d/5291.bugfix (limited to 'changelog.d') diff --git a/CHANGES.md b/CHANGES.md index 350151b62c..0ffdf1aaef 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -1,3 +1,12 @@ +Synapse 0.99.5.2 (2019-05-30) +============================= + +Bugfixes +-------- + +- Fix bug where we leaked extremities when we soft failed events, leading to performance degradation. ([\#5274](https://github.com/matrix-org/synapse/issues/5274), [\#5278](https://github.com/matrix-org/synapse/issues/5278), [\#5291](https://github.com/matrix-org/synapse/issues/5291)) + + Synapse 0.99.5.1 (2019-05-22) ============================= diff --git a/changelog.d/5274.bugfix b/changelog.d/5274.bugfix deleted file mode 100644 index 9e14d20289..0000000000 --- a/changelog.d/5274.bugfix +++ /dev/null @@ -1 +0,0 @@ -Fix bug where we leaked extremities when we soft failed events, leading to performance degradation. diff --git a/changelog.d/5278.bugfix b/changelog.d/5278.bugfix deleted file mode 100644 index 9e14d20289..0000000000 --- a/changelog.d/5278.bugfix +++ /dev/null @@ -1 +0,0 @@ -Fix bug where we leaked extremities when we soft failed events, leading to performance degradation. diff --git a/changelog.d/5291.bugfix b/changelog.d/5291.bugfix deleted file mode 100644 index 9e14d20289..0000000000 --- a/changelog.d/5291.bugfix +++ /dev/null @@ -1 +0,0 @@ -Fix bug where we leaked extremities when we soft failed events, leading to performance degradation. diff --git a/debian/changelog b/debian/changelog index 90c6b86c5b..6a1a72c0e3 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,3 +1,9 @@ +matrix-synapse-py3 (0.99.5.2) stable; urgency=medium + + * New synapse release 0.99.5.2. + + -- Synapse Packaging team Thu, 30 May 2019 16:28:07 +0100 + matrix-synapse-py3 (0.99.5.1) stable; urgency=medium * New synapse release 0.99.5.1. diff --git a/synapse/__init__.py b/synapse/__init__.py index 4f95778eea..d0e8d7c21b 100644 --- a/synapse/__init__.py +++ b/synapse/__init__.py @@ -27,4 +27,4 @@ try: except ImportError: pass -__version__ = "0.99.5.1" +__version__ = "0.99.5.2" -- cgit 1.5.1