diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml
index 6574550447..391e9c96ff 100644
--- a/.github/workflows/docker.yml
+++ b/.github/workflows/docker.yml
@@ -30,7 +30,7 @@ jobs:
run: docker buildx inspect
- name: Install Cosign
- uses: sigstore/cosign-installer@v3.4.0
+ uses: sigstore/cosign-installer@v3.5.0
- name: Checkout repository
uses: actions/checkout@v4
diff --git a/.github/workflows/docs-pr.yaml b/.github/workflows/docs-pr.yaml
index 652ef90095..07dc301b1a 100644
--- a/.github/workflows/docs-pr.yaml
+++ b/.github/workflows/docs-pr.yaml
@@ -19,7 +19,7 @@ jobs:
fetch-depth: 0
- name: Setup mdbook
- uses: peaceiris/actions-mdbook@adeb05db28a0c0004681db83893d56c0388ea9ea # v1.2.0
+ uses: peaceiris/actions-mdbook@ee69d230fe19748b7abf22df32acaa93833fad08 # v2.0.0
with:
mdbook-version: '0.4.17'
@@ -53,7 +53,7 @@ jobs:
- uses: actions/checkout@v4
- name: Setup mdbook
- uses: peaceiris/actions-mdbook@adeb05db28a0c0004681db83893d56c0388ea9ea # v1.2.0
+ uses: peaceiris/actions-mdbook@ee69d230fe19748b7abf22df32acaa93833fad08 # v2.0.0
with:
mdbook-version: '0.4.17'
diff --git a/.github/workflows/docs.yaml b/.github/workflows/docs.yaml
index d611fdc924..fe3212f82a 100644
--- a/.github/workflows/docs.yaml
+++ b/.github/workflows/docs.yaml
@@ -56,7 +56,7 @@ jobs:
fetch-depth: 0
- name: Setup mdbook
- uses: peaceiris/actions-mdbook@adeb05db28a0c0004681db83893d56c0388ea9ea # v1.2.0
+ uses: peaceiris/actions-mdbook@ee69d230fe19748b7abf22df32acaa93833fad08 # v2.0.0
with:
mdbook-version: '0.4.17'
@@ -80,7 +80,7 @@ jobs:
# Deploy to the target directory.
- name: Deploy to gh pages
- uses: peaceiris/actions-gh-pages@373f7f263a76c20808c831209c920827a82a2847 # v3.9.3
+ uses: peaceiris/actions-gh-pages@4f9cc6602d3f66b9c108549d475ec49e8ef4d45e # v4.0.0
with:
github_token: ${{ secrets.GITHUB_TOKEN }}
publish_dir: ./book
@@ -110,7 +110,7 @@ jobs:
# Deploy to the target directory.
- name: Deploy to gh pages
- uses: peaceiris/actions-gh-pages@373f7f263a76c20808c831209c920827a82a2847 # v3.9.3
+ uses: peaceiris/actions-gh-pages@4f9cc6602d3f66b9c108549d475ec49e8ef4d45e # v4.0.0
with:
github_token: ${{ secrets.GITHUB_TOKEN }}
publish_dir: ./dev-docs/_build/html
diff --git a/Cargo.lock b/Cargo.lock
index 65f4807c65..faac6b3c8a 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -13,9 +13,9 @@ dependencies = [
[[package]]
name = "anyhow"
-version = "1.0.81"
+version = "1.0.82"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "0952808a6c2afd1aa8947271f3a60f1a6763c7b912d210184c5149b5cf147247"
+checksum = "f538837af36e6f6a9be0faa67f9a314f8119e4e4b5867c6ab40ed60360142519"
[[package]]
name = "arc-swap"
diff --git a/changelog.d/16923.bugfix b/changelog.d/16923.bugfix
new file mode 100644
index 0000000000..bd6f24925e
--- /dev/null
+++ b/changelog.d/16923.bugfix
@@ -0,0 +1 @@
+Return `400 M_NOT_JSON` upon receiving invalid JSON in query parameters across various client and admin endpoints, rather than an internal server error.
\ No newline at end of file
diff --git a/changelog.d/17069.doc b/changelog.d/17069.doc
new file mode 100644
index 0000000000..f5a7f599d1
--- /dev/null
+++ b/changelog.d/17069.doc
@@ -0,0 +1 @@
+Add a prompt in the contributing guide to manually configure icu4c.
diff --git a/changelog.d/17099.doc b/changelog.d/17099.doc
new file mode 100644
index 0000000000..d8d10fa53a
--- /dev/null
+++ b/changelog.d/17099.doc
@@ -0,0 +1 @@
+Clarify what part of message retention is still experimental.
diff --git a/docs/development/contributing_guide.md b/docs/development/contributing_guide.md
index ac8a7039d1..76c3e790cd 100644
--- a/docs/development/contributing_guide.md
+++ b/docs/development/contributing_guide.md
@@ -86,6 +86,8 @@ poetry install --extras all
This will install the runtime and developer dependencies for the project. Be sure to check
that the `poetry install` step completed cleanly.
+For OSX users, be sure to set `PKG_CONFIG_PATH` to support `icu4c`. Run `brew info icu4c` for more details.
+
## Running Synapse via poetry
To start a local instance of Synapse in the locked poetry environment, create a config file:
diff --git a/docs/message_retention_policies.md b/docs/message_retention_policies.md
index 2746a106b3..c64d1539b0 100644
--- a/docs/message_retention_policies.md
+++ b/docs/message_retention_policies.md
@@ -7,8 +7,10 @@ follow the semantics described in
and allow server and room admins to configure how long messages should
be kept in a homeserver's database before being purged from it.
**Please note that, as this feature isn't part of the Matrix
-specification yet, this implementation is to be considered as
-experimental.**
+specification yet, the use of `m.room.retention` events for per-room
+retention policies is to be considered as experimental. However, the use
+of a default message retention policy is considered a stable feature
+in Synapse.**
A message retention policy is mainly defined by its `max_lifetime`
parameter, which defines how long a message can be kept around after
diff --git a/poetry.lock b/poetry.lock
index 814877b70a..d916c627a0 100644
--- a/poetry.lock
+++ b/poetry.lock
@@ -1848,17 +1848,17 @@ files = [
[[package]]
name = "pyasn1-modules"
-version = "0.3.0"
+version = "0.4.0"
description = "A collection of ASN.1-based protocols modules"
optional = false
-python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,>=2.7"
+python-versions = ">=3.8"
files = [
- {file = "pyasn1_modules-0.3.0-py2.py3-none-any.whl", hash = "sha256:d3ccd6ed470d9ffbc716be08bd90efbd44d0734bc9303818f7336070984a162d"},
- {file = "pyasn1_modules-0.3.0.tar.gz", hash = "sha256:5bd01446b736eb9d31512a30d46c1ac3395d676c6f3cafa4c03eb54b9925631c"},
+ {file = "pyasn1_modules-0.4.0-py3-none-any.whl", hash = "sha256:be04f15b66c206eed667e0bb5ab27e2b1855ea54a842e5037738099e8ca4ae0b"},
+ {file = "pyasn1_modules-0.4.0.tar.gz", hash = "sha256:831dbcea1b177b28c9baddf4c6d1013c24c3accd14a1873fffaa6a2e905f17b6"},
]
[package.dependencies]
-pyasn1 = ">=0.4.6,<0.6.0"
+pyasn1 = ">=0.4.6,<0.7.0"
[[package]]
name = "pycparser"
@@ -1983,13 +1983,13 @@ typing-extensions = ">=4.6.0,<4.7.0 || >4.7.0"
[[package]]
name = "pygithub"
-version = "2.2.0"
+version = "2.3.0"
description = "Use the full Github API v3"
optional = false
python-versions = ">=3.7"
files = [
- {file = "PyGithub-2.2.0-py3-none-any.whl", hash = "sha256:41042ea53e4c372219db708c38d2ca1fd4fadab75475bac27d89d339596cfad1"},
- {file = "PyGithub-2.2.0.tar.gz", hash = "sha256:e39be7c4dc39418bdd6e3ecab5931c636170b8b21b4d26f9ecf7e6102a3b51c3"},
+ {file = "PyGithub-2.3.0-py3-none-any.whl", hash = "sha256:65b499728be3ce7b0cd2cd760da3b32f0f4d7bc55e5e0677617f90f6564e793e"},
+ {file = "PyGithub-2.3.0.tar.gz", hash = "sha256:0148d7347a1cdeed99af905077010aef81a4dad988b0ba51d4108bf66b443f7e"},
]
[package.dependencies]
@@ -2444,28 +2444,28 @@ files = [
[[package]]
name = "ruff"
-version = "0.3.5"
+version = "0.3.7"
description = "An extremely fast Python linter and code formatter, written in Rust."
optional = false
python-versions = ">=3.7"
files = [
- {file = "ruff-0.3.5-py3-none-macosx_10_12_x86_64.macosx_11_0_arm64.macosx_10_12_universal2.whl", hash = "sha256:aef5bd3b89e657007e1be6b16553c8813b221ff6d92c7526b7e0227450981eac"},
- {file = "ruff-0.3.5-py3-none-macosx_10_12_x86_64.whl", hash = "sha256:89b1e92b3bd9fca249153a97d23f29bed3992cff414b222fcd361d763fc53f12"},
- {file = "ruff-0.3.5-py3-none-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5e55771559c89272c3ebab23326dc23e7f813e492052391fe7950c1a5a139d89"},
- {file = "ruff-0.3.5-py3-none-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:dabc62195bf54b8a7876add6e789caae0268f34582333cda340497c886111c39"},
- {file = "ruff-0.3.5-py3-none-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:3a05f3793ba25f194f395578579c546ca5d83e0195f992edc32e5907d142bfa3"},
- {file = "ruff-0.3.5-py3-none-manylinux_2_17_ppc64.manylinux2014_ppc64.whl", hash = "sha256:dfd3504e881082959b4160ab02f7a205f0fadc0a9619cc481982b6837b2fd4c0"},
- {file = "ruff-0.3.5-py3-none-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:87258e0d4b04046cf1d6cc1c56fadbf7a880cc3de1f7294938e923234cf9e498"},
- {file = "ruff-0.3.5-py3-none-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:712e71283fc7d9f95047ed5f793bc019b0b0a29849b14664a60fd66c23b96da1"},
- {file = "ruff-0.3.5-py3-none-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a532a90b4a18d3f722c124c513ffb5e5eaff0cc4f6d3aa4bda38e691b8600c9f"},
- {file = "ruff-0.3.5-py3-none-musllinux_1_2_aarch64.whl", hash = "sha256:122de171a147c76ada00f76df533b54676f6e321e61bd8656ae54be326c10296"},
- {file = "ruff-0.3.5-py3-none-musllinux_1_2_armv7l.whl", hash = "sha256:d80a6b18a6c3b6ed25b71b05eba183f37d9bc8b16ace9e3d700997f00b74660b"},
- {file = "ruff-0.3.5-py3-none-musllinux_1_2_i686.whl", hash = "sha256:a7b6e63194c68bca8e71f81de30cfa6f58ff70393cf45aab4c20f158227d5936"},
- {file = "ruff-0.3.5-py3-none-musllinux_1_2_x86_64.whl", hash = "sha256:a759d33a20c72f2dfa54dae6e85e1225b8e302e8ac655773aff22e542a300985"},
- {file = "ruff-0.3.5-py3-none-win32.whl", hash = "sha256:9d8605aa990045517c911726d21293ef4baa64f87265896e491a05461cae078d"},
- {file = "ruff-0.3.5-py3-none-win_amd64.whl", hash = "sha256:dc56bb16a63c1303bd47563c60482a1512721053d93231cf7e9e1c6954395a0e"},
- {file = "ruff-0.3.5-py3-none-win_arm64.whl", hash = "sha256:faeeae9905446b975dcf6d4499dc93439b131f1443ee264055c5716dd947af55"},
- {file = "ruff-0.3.5.tar.gz", hash = "sha256:a067daaeb1dc2baf9b82a32dae67d154d95212080c80435eb052d95da647763d"},
+ {file = "ruff-0.3.7-py3-none-macosx_10_12_x86_64.macosx_11_0_arm64.macosx_10_12_universal2.whl", hash = "sha256:0e8377cccb2f07abd25e84fc5b2cbe48eeb0fea9f1719cad7caedb061d70e5ce"},
+ {file = "ruff-0.3.7-py3-none-macosx_10_12_x86_64.whl", hash = "sha256:15a4d1cc1e64e556fa0d67bfd388fed416b7f3b26d5d1c3e7d192c897e39ba4b"},
+ {file = "ruff-0.3.7-py3-none-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d28bdf3d7dc71dd46929fafeec98ba89b7c3550c3f0978e36389b5631b793663"},
+ {file = "ruff-0.3.7-py3-none-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:379b67d4f49774ba679593b232dcd90d9e10f04d96e3c8ce4a28037ae473f7bb"},
+ {file = "ruff-0.3.7-py3-none-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:c060aea8ad5ef21cdfbbe05475ab5104ce7827b639a78dd55383a6e9895b7c51"},
+ {file = "ruff-0.3.7-py3-none-manylinux_2_17_ppc64.manylinux2014_ppc64.whl", hash = "sha256:ebf8f615dde968272d70502c083ebf963b6781aacd3079081e03b32adfe4d58a"},
+ {file = "ruff-0.3.7-py3-none-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:d48098bd8f5c38897b03604f5428901b65e3c97d40b3952e38637b5404b739a2"},
+ {file = "ruff-0.3.7-py3-none-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:da8a4fda219bf9024692b1bc68c9cff4b80507879ada8769dc7e985755d662ea"},
+ {file = "ruff-0.3.7-py3-none-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6c44e0149f1d8b48c4d5c33d88c677a4aa22fd09b1683d6a7ff55b816b5d074f"},
+ {file = "ruff-0.3.7-py3-none-musllinux_1_2_aarch64.whl", hash = "sha256:3050ec0af72b709a62ecc2aca941b9cd479a7bf2b36cc4562f0033d688e44fa1"},
+ {file = "ruff-0.3.7-py3-none-musllinux_1_2_armv7l.whl", hash = "sha256:a29cc38e4c1ab00da18a3f6777f8b50099d73326981bb7d182e54a9a21bb4ff7"},
+ {file = "ruff-0.3.7-py3-none-musllinux_1_2_i686.whl", hash = "sha256:5b15cc59c19edca917f51b1956637db47e200b0fc5e6e1878233d3a938384b0b"},
+ {file = "ruff-0.3.7-py3-none-musllinux_1_2_x86_64.whl", hash = "sha256:e491045781b1e38b72c91247cf4634f040f8d0cb3e6d3d64d38dcf43616650b4"},
+ {file = "ruff-0.3.7-py3-none-win32.whl", hash = "sha256:bc931de87593d64fad3a22e201e55ad76271f1d5bfc44e1a1887edd0903c7d9f"},
+ {file = "ruff-0.3.7-py3-none-win_amd64.whl", hash = "sha256:5ef0e501e1e39f35e03c2acb1d1238c595b8bb36cf7a170e7c1df1b73da00e74"},
+ {file = "ruff-0.3.7-py3-none-win_arm64.whl", hash = "sha256:789e144f6dc7019d1f92a812891c645274ed08af6037d11fc65fcbc183b7d59f"},
+ {file = "ruff-0.3.7.tar.gz", hash = "sha256:d5c1aebee5162c2226784800ae031f660c350e7a3402c4d1f8ea4e97e232e3ba"},
]
[[package]]
@@ -2954,13 +2954,13 @@ docs = ["sphinx (<7.0.0)"]
[[package]]
name = "twine"
-version = "4.0.2"
+version = "5.0.0"
description = "Collection of utilities for publishing packages on PyPI"
optional = false
-python-versions = ">=3.7"
+python-versions = ">=3.8"
files = [
- {file = "twine-4.0.2-py3-none-any.whl", hash = "sha256:929bc3c280033347a00f847236564d1c52a3e61b1ac2516c97c48f3ceab756d8"},
- {file = "twine-4.0.2.tar.gz", hash = "sha256:9e102ef5fdd5a20661eb88fad46338806c3bd32cf1db729603fe3697b1bc83c8"},
+ {file = "twine-5.0.0-py3-none-any.whl", hash = "sha256:a262933de0b484c53408f9edae2e7821c1c45a3314ff2df9bdd343aa7ab8edc0"},
+ {file = "twine-5.0.0.tar.gz", hash = "sha256:89b0cc7d370a4b66421cc6102f269aa910fe0f1861c124f573cf2ddedbc10cf4"},
]
[package.dependencies]
@@ -3109,13 +3109,13 @@ files = [
[[package]]
name = "types-pillow"
-version = "10.2.0.20240406"
+version = "10.2.0.20240415"
description = "Typing stubs for Pillow"
optional = false
python-versions = ">=3.8"
files = [
- {file = "types-Pillow-10.2.0.20240406.tar.gz", hash = "sha256:62e0cc1f17caba40e72e7154a483f4c7f3bea0e1c34c0ebba9de3c7745bc306d"},
- {file = "types_Pillow-10.2.0.20240406-py3-none-any.whl", hash = "sha256:5ac182e8afce53de30abca2fdf9cbec7b2500e549d0be84da035a729a84c7c47"},
+ {file = "types-Pillow-10.2.0.20240415.tar.gz", hash = "sha256:dd6058027639bcdc66ba78b228cc25fdae42524c2150c78c804da427e7e76e70"},
+ {file = "types_Pillow-10.2.0.20240415-py3-none-any.whl", hash = "sha256:f933332b7e96010bae9b9cf82a4c9979ff0c270d63f5c5bbffb2d789b85cd00b"},
]
[[package]]
@@ -3451,4 +3451,4 @@ user-search = ["pyicu"]
[metadata]
lock-version = "2.0"
python-versions = "^3.8.0"
-content-hash = "4abda113a01f162bb3978b0372956d569364533aa39f57863c234363f8449a4f"
+content-hash = "1951f2b4623138d47db08a405edd970e67599d05804bb459af21a085e1665f69"
diff --git a/pyproject.toml b/pyproject.toml
index f0f025645f..fb310589f7 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -321,7 +321,7 @@ all = [
# This helps prevents merge conflicts when running a batch of dependabot updates.
isort = ">=5.10.1"
black = ">=22.7.0"
-ruff = "0.3.5"
+ruff = "0.3.7"
# Type checking only works with the pydantic.v1 compat module from pydantic v2
pydantic = "^2"
diff --git a/synapse/handlers/sync.py b/synapse/handlers/sync.py
index 8ff45a3353..1fb7ca4f71 100644
--- a/synapse/handlers/sync.py
+++ b/synapse/handlers/sync.py
@@ -88,6 +88,10 @@ if TYPE_CHECKING:
logger = logging.getLogger(__name__)
+# Logging for https://github.com/matrix-org/matrix-spec/issues/1209 and
+# https://github.com/element-hq/synapse/issues/16940
+client_state_desync_logger = logging.getLogger("synapse.client_state_desync_debug")
+
# Counts the number of times we returned a non-empty sync. `type` is one of
# "initial_sync", "full_state_sync" or "incremental_sync", `lazy_loaded` is
# "true" or "false" depending on if the request asked for lazy loaded members or
@@ -1216,6 +1220,12 @@ class SyncHandler:
previous_timeline_end={},
lazy_load_members=lazy_load_members,
)
+
+ if client_state_desync_logger.isEnabledFor(logging.DEBUG):
+ await self._log_client_state_desync(
+ room_id, None, state_ids, timeline_state, lazy_load_members
+ )
+
return state_ids
async def _compute_state_delta_for_incremental_sync(
@@ -1361,6 +1371,15 @@ class SyncHandler:
lazy_load_members=lazy_load_members,
)
+ if client_state_desync_logger.isEnabledFor(logging.DEBUG):
+ await self._log_client_state_desync(
+ room_id,
+ since_token,
+ state_ids,
+ timeline_state,
+ lazy_load_members,
+ )
+
return state_ids
async def _find_missing_partial_state_memberships(
@@ -1477,6 +1496,125 @@ class SyncHandler:
return additional_state_ids
+ async def _log_client_state_desync(
+ self,
+ room_id: str,
+ since_token: Optional[StreamToken],
+ sync_response_state_state: StateMap[str],
+ sync_response_timeline_state: StateMap[str],
+ lazy_load_members: bool,
+ ) -> None:
+ """
+ Logging to see how often the client's state gets out of sync with the
+ actual current state of the room.
+
+ There are few different potential failure modes here:
+
+ * State resolution can cause changes in the state of the room that don't
+ directly correspond to events with the corresponding (type, state_key).
+ https://github.com/matrix-org/matrix-spec/issues/1209 discusses this in
+ more detail.
+
+ * Even where there is an event that causes a given state change, Synapse
+ may not serve it to the client, since it works on state at specific points
+ in the DAG, rather than "current state".
+ See https://github.com/element-hq/synapse/issues/16940.
+
+ * Lazy-loading adds more complexity, as it means that events that would
+ normally be served via the `state` part of an incremental sync are filtered
+ out.
+
+ To try to get a handle on this, let's put ourselves in the shoes of a client,
+ and compare the state they will calculate against the actual current state.
+ """
+ # We only care about membership events.
+ state_filter = StateFilter.from_types(types=(("m.room.member", None),))
+
+ if since_token is None:
+ if lazy_load_members:
+ # For initial syncs with lazy-loading enabled, there's not too much
+ # concern here. We know the client will do a `/members` query before
+ # doing any encryption, so what sync returns isn't too important.
+ #
+ # (Of course, then `/members` might also return an incomplete list, but
+ # that's a separate problem.)
+ return
+
+ # For regular initial syncs, compare the returned response with the actual
+ # current state.
+ client_calculated_state = {}
+ client_calculated_state.update(sync_response_state_state)
+ client_calculated_state.update(sync_response_timeline_state)
+ else:
+ # For an incremental (gappy or otherwise) sync, let's assume the client has
+ # a complete membership list as of the last sync (or rather, at
+ # `since_token`, which is the closest approximation we have to it
+ # right now), and see what they would calculate as the current state given
+ # this sync update.
+
+ client_calculated_state = dict(
+ await self.get_state_at(
+ room_id,
+ stream_position=since_token,
+ state_filter=state_filter,
+ await_full_state=False,
+ )
+ )
+ client_calculated_state.update(sync_response_state_state)
+ client_calculated_state.update(sync_response_timeline_state)
+
+ current_state = await self._state_storage_controller.get_current_state_ids(
+ room_id, state_filter=state_filter, await_full_state=False
+ )
+ missing_users = await self._calculate_missing_members(
+ current_state, client_calculated_state
+ )
+ if missing_users:
+ client_state_desync_logger.debug(
+ "client state discrepancy in incremental sync in room %s: missing users %s",
+ room_id,
+ missing_users,
+ )
+
+ async def _calculate_missing_members(
+ self,
+ actual_state: StateMap[str],
+ client_calculated_state: StateMap[str],
+ ) -> List[str]:
+ """Helper for `_log_client_state_desync`: calculates the difference in
+ joined members between two state maps.
+
+ Returns:
+ A list of user IDs
+ """
+ missing_users = []
+
+ async def event_id_to_membership(event_id: Optional[str]) -> Optional[str]:
+ if event_id is None:
+ return None
+ event = await self.store.get_event(event_id, allow_none=True)
+ if event is None:
+ return "MISSING_EVENT"
+ return event.membership
+
+ # Check for joined members in the actual state that are missing or have a
+ # different membership in the actual state.
+ for (event_type, state_key), actual_event_id in actual_state.items():
+ if event_type != EventTypes.Member:
+ continue
+
+ calculated_event_id = client_calculated_state.get((event_type, state_key))
+ if calculated_event_id != actual_event_id:
+ actual_membership = event_id_to_membership(actual_event_id)
+ calculated_membership = event_id_to_membership(calculated_event_id)
+ if (
+ actual_membership == Membership.JOIN
+ and calculated_membership != Membership.JOIN
+ ):
+ missing_users.append(state_key)
+
+ return missing_users
+
async def unread_notifs_for_room_id(
self, room_id: str, sync_config: SyncConfig
) -> RoomNotifCounts:
diff --git a/synapse/http/servlet.py b/synapse/http/servlet.py
index 0ca08038f4..ab12951da8 100644
--- a/synapse/http/servlet.py
+++ b/synapse/http/servlet.py
@@ -23,6 +23,7 @@
import enum
import logging
+import urllib.parse as urlparse
from http import HTTPStatus
from typing import (
TYPE_CHECKING,
@@ -450,6 +451,87 @@ def parse_string(
)
+def parse_json(
+ request: Request,
+ name: str,
+ default: Optional[dict] = None,
+ required: bool = False,
+ encoding: str = "ascii",
+) -> Optional[JsonDict]:
+ """
+ Parse a JSON parameter from the request query string.
+
+ Args:
+ request: the twisted HTTP request.
+ name: the name of the query parameter.
+ default: value to use if the parameter is absent,
+ defaults to None.
+ required: whether to raise a 400 SynapseError if the
+ parameter is absent, defaults to False.
+ encoding: The encoding to decode the string content with.
+
+ Returns:
+ A JSON value, or `default` if the named query parameter was not found
+ and `required` was False.
+
+ Raises:
+ SynapseError if the parameter is absent and required, or if the
+ parameter is present and not a JSON object.
+ """
+ args: Mapping[bytes, Sequence[bytes]] = request.args # type: ignore
+ return parse_json_from_args(
+ args,
+ name,
+ default,
+ required=required,
+ encoding=encoding,
+ )
+
+
+def parse_json_from_args(
+ args: Mapping[bytes, Sequence[bytes]],
+ name: str,
+ default: Optional[dict] = None,
+ required: bool = False,
+ encoding: str = "ascii",
+) -> Optional[JsonDict]:
+ """
+ Parse a JSON parameter from the request query string.
+
+ Args:
+ args: a mapping of request args as bytes to a list of bytes (e.g. request.args).
+ name: the name of the query parameter.
+ default: value to use if the parameter is absent,
+ defaults to None.
+ required: whether to raise a 400 SynapseError if the
+ parameter is absent, defaults to False.
+ encoding: the encoding to decode the string content with.
+
+ A JSON value, or `default` if the named query parameter was not found
+ and `required` was False.
+
+ Raises:
+ SynapseError if the parameter is absent and required, or if the
+ parameter is present and not a JSON object.
+ """
+ name_bytes = name.encode("ascii")
+
+ if name_bytes not in args:
+ if not required:
+ return default
+
+ message = f"Missing required integer query parameter {name}"
+ raise SynapseError(HTTPStatus.BAD_REQUEST, message, errcode=Codes.MISSING_PARAM)
+
+ json_str = parse_string_from_args(args, name, required=True, encoding=encoding)
+
+ try:
+ return json_decoder.decode(urlparse.unquote(json_str))
+ except Exception:
+ message = f"Query parameter {name} must be a valid JSON object"
+ raise SynapseError(HTTPStatus.BAD_REQUEST, message, errcode=Codes.NOT_JSON)
+
+
EnumT = TypeVar("EnumT", bound=enum.Enum)
diff --git a/synapse/rest/admin/rooms.py b/synapse/rest/admin/rooms.py
index 4252f98a6c..0d86a4e15f 100644
--- a/synapse/rest/admin/rooms.py
+++ b/synapse/rest/admin/rooms.py
@@ -21,7 +21,6 @@
import logging
from http import HTTPStatus
from typing import TYPE_CHECKING, List, Optional, Tuple, cast
-from urllib import parse as urlparse
import attr
@@ -38,6 +37,7 @@ from synapse.http.servlet import (
assert_params_in_dict,
parse_enum,
parse_integer,
+ parse_json,
parse_json_object_from_request,
parse_string,
)
@@ -51,7 +51,6 @@ from synapse.storage.databases.main.room import RoomSortOrder
from synapse.streams.config import PaginationConfig
from synapse.types import JsonDict, RoomID, ScheduledTask, UserID, create_requester
from synapse.types.state import StateFilter
-from synapse.util import json_decoder
if TYPE_CHECKING:
from synapse.api.auth import Auth
@@ -776,14 +775,8 @@ class RoomEventContextServlet(RestServlet):
limit = parse_integer(request, "limit", default=10)
# picking the API shape for symmetry with /messages
- filter_str = parse_string(request, "filter", encoding="utf-8")
- if filter_str:
- filter_json = urlparse.unquote(filter_str)
- event_filter: Optional[Filter] = Filter(
- self._hs, json_decoder.decode(filter_json)
- )
- else:
- event_filter = None
+ filter_json = parse_json(request, "filter", encoding="utf-8")
+ event_filter = Filter(self._hs, filter_json) if filter_json else None
event_context = await self.room_context_handler.get_event_context(
requester,
@@ -914,21 +907,16 @@ class RoomMessagesRestServlet(RestServlet):
)
# Twisted will have processed the args by now.
assert request.args is not None
+
+ filter_json = parse_json(request, "filter", encoding="utf-8")
+ event_filter = Filter(self._hs, filter_json) if filter_json else None
+
as_client_event = b"raw" not in request.args
- filter_str = parse_string(request, "filter", encoding="utf-8")
- if filter_str:
- filter_json = urlparse.unquote(filter_str)
- event_filter: Optional[Filter] = Filter(
- self._hs, json_decoder.decode(filter_json)
- )
- if (
- event_filter
- and event_filter.filter_json.get("event_format", "client")
- == "federation"
- ):
- as_client_event = False
- else:
- event_filter = None
+ if (
+ event_filter
+ and event_filter.filter_json.get("event_format", "client") == "federation"
+ ):
+ as_client_event = False
msgs = await self._pagination_handler.get_messages(
room_id=room_id,
diff --git a/synapse/rest/client/room.py b/synapse/rest/client/room.py
index 4eeadf8779..e4c7dd1a58 100644
--- a/synapse/rest/client/room.py
+++ b/synapse/rest/client/room.py
@@ -52,6 +52,7 @@ from synapse.http.servlet import (
parse_boolean,
parse_enum,
parse_integer,
+ parse_json,
parse_json_object_from_request,
parse_string,
parse_strings_from_args,
@@ -65,7 +66,6 @@ from synapse.rest.client.transactions import HttpTransactionCache
from synapse.streams.config import PaginationConfig
from synapse.types import JsonDict, Requester, StreamToken, ThirdPartyInstanceID, UserID
from synapse.types.state import StateFilter
-from synapse.util import json_decoder
from synapse.util.cancellation import cancellable
from synapse.util.stringutils import parse_and_validate_server_name, random_string
@@ -703,21 +703,16 @@ class RoomMessageListRestServlet(RestServlet):
)
# Twisted will have processed the args by now.
assert request.args is not None
+
+ filter_json = parse_json(request, "filter", encoding="utf-8")
+ event_filter = Filter(self._hs, filter_json) if filter_json else None
+
as_client_event = b"raw" not in request.args
- filter_str = parse_string(request, "filter", encoding="utf-8")
- if filter_str:
- filter_json = urlparse.unquote(filter_str)
- event_filter: Optional[Filter] = Filter(
- self._hs, json_decoder.decode(filter_json)
- )
- if (
- event_filter
- and event_filter.filter_json.get("event_format", "client")
- == "federation"
- ):
- as_client_event = False
- else:
- event_filter = None
+ if (
+ event_filter
+ and event_filter.filter_json.get("event_format", "client") == "federation"
+ ):
+ as_client_event = False
msgs = await self.pagination_handler.get_messages(
room_id=room_id,
@@ -898,14 +893,8 @@ class RoomEventContextServlet(RestServlet):
limit = parse_integer(request, "limit", default=10)
# picking the API shape for symmetry with /messages
- filter_str = parse_string(request, "filter", encoding="utf-8")
- if filter_str:
- filter_json = urlparse.unquote(filter_str)
- event_filter: Optional[Filter] = Filter(
- self._hs, json_decoder.decode(filter_json)
- )
- else:
- event_filter = None
+ filter_json = parse_json(request, "filter", encoding="utf-8")
+ event_filter = Filter(self._hs, filter_json) if filter_json else None
event_context = await self.room_context_handler.get_event_context(
requester, room_id, event_id, limit, event_filter
diff --git a/tests/rest/admin/test_room.py b/tests/rest/admin/test_room.py
index 0b669b6ee7..7562747260 100644
--- a/tests/rest/admin/test_room.py
+++ b/tests/rest/admin/test_room.py
@@ -21,6 +21,7 @@
import json
import time
import urllib.parse
+from http import HTTPStatus
from typing import List, Optional
from unittest.mock import AsyncMock, Mock
@@ -2190,6 +2191,33 @@ class RoomMessagesTestCase(unittest.HomeserverTestCase):
chunk = channel.json_body["chunk"]
self.assertEqual(len(chunk), 0, [event["content"] for event in chunk])
+ def test_room_message_filter_query_validation(self) -> None:
+ # Test json validation in (filter) query parameter.
+ # Does not test the validity of the filter, only the json validation.
+
+ # Check Get with valid json filter parameter, expect 200.
+ valid_filter_str = '{"types": ["m.room.message"]}'
+ channel = self.make_request(
+ "GET",
+ f"/_synapse/admin/v1/rooms/{self.room_id}/messages?dir=b&filter={valid_filter_str}",
+ access_token=self.admin_user_tok,
+ )
+
+ self.assertEqual(channel.code, HTTPStatus.OK, channel.json_body)
+
+ # Check Get with invalid json filter parameter, expect 400 NOT_JSON.
+ invalid_filter_str = "}}}{}"
+ channel = self.make_request(
+ "GET",
+ f"/_synapse/admin/v1/rooms/{self.room_id}/messages?dir=b&filter={invalid_filter_str}",
+ access_token=self.admin_user_tok,
+ )
+
+ self.assertEqual(channel.code, HTTPStatus.BAD_REQUEST, channel.json_body)
+ self.assertEqual(
+ channel.json_body["errcode"], Codes.NOT_JSON, channel.json_body
+ )
+
class JoinAliasRoomTestCase(unittest.HomeserverTestCase):
servlets = [
@@ -2522,6 +2550,39 @@ class JoinAliasRoomTestCase(unittest.HomeserverTestCase):
else:
self.fail("Event %s from events_after not found" % j)
+ def test_room_event_context_filter_query_validation(self) -> None:
+ # Test json validation in (filter) query parameter.
+ # Does not test the validity of the filter, only the json validation.
+
+ # Create a user with room and event_id.
+ user_id = self.register_user("test", "test")
+ user_tok = self.login("test", "test")
+ room_id = self.helper.create_room_as(user_id, tok=user_tok)
+ event_id = self.helper.send(room_id, "message 1", tok=user_tok)["event_id"]
+
+ # Check Get with valid json filter parameter, expect 200.
+ valid_filter_str = '{"types": ["m.room.message"]}'
+ channel = self.make_request(
+ "GET",
+ f"/_synapse/admin/v1/rooms/{room_id}/context/{event_id}?filter={valid_filter_str}",
+ access_token=self.admin_user_tok,
+ )
+
+ self.assertEqual(channel.code, HTTPStatus.OK, channel.json_body)
+
+ # Check Get with invalid json filter parameter, expect 400 NOT_JSON.
+ invalid_filter_str = "}}}{}"
+ channel = self.make_request(
+ "GET",
+ f"/_synapse/admin/v1/rooms/{room_id}/context/{event_id}?filter={invalid_filter_str}",
+ access_token=self.admin_user_tok,
+ )
+
+ self.assertEqual(channel.code, HTTPStatus.BAD_REQUEST, channel.json_body)
+ self.assertEqual(
+ channel.json_body["errcode"], Codes.NOT_JSON, channel.json_body
+ )
+
class MakeRoomAdminTestCase(unittest.HomeserverTestCase):
servlets = [
diff --git a/tests/rest/client/test_rooms.py b/tests/rest/client/test_rooms.py
index 1364615085..b796163dcb 100644
--- a/tests/rest/client/test_rooms.py
+++ b/tests/rest/client/test_rooms.py
@@ -2175,6 +2175,31 @@ class RoomMessageListTestCase(RoomBase):
chunk = channel.json_body["chunk"]
self.assertEqual(len(chunk), 0, [event["content"] for event in chunk])
+ def test_room_message_filter_query_validation(self) -> None:
+ # Test json validation in (filter) query parameter.
+ # Does not test the validity of the filter, only the json validation.
+
+ # Check Get with valid json filter parameter, expect 200.
+ valid_filter_str = '{"types": ["m.room.message"]}'
+ channel = self.make_request(
+ "GET",
+ f"/rooms/{self.room_id}/messages?access_token=x&dir=b&filter={valid_filter_str}",
+ )
+
+ self.assertEqual(channel.code, HTTPStatus.OK, channel.json_body)
+
+ # Check Get with invalid json filter parameter, expect 400 NOT_JSON.
+ invalid_filter_str = "}}}{}"
+ channel = self.make_request(
+ "GET",
+ f"/rooms/{self.room_id}/messages?access_token=x&dir=b&filter={invalid_filter_str}",
+ )
+
+ self.assertEqual(channel.code, HTTPStatus.BAD_REQUEST, channel.json_body)
+ self.assertEqual(
+ channel.json_body["errcode"], Codes.NOT_JSON, channel.json_body
+ )
+
class RoomMessageFilterTestCase(RoomBase):
"""Tests /rooms/$room_id/messages REST events."""
@@ -3213,6 +3238,33 @@ class ContextTestCase(unittest.HomeserverTestCase):
self.assertDictEqual(events_after[0].get("content"), {}, events_after[0])
self.assertEqual(events_after[1].get("content"), {}, events_after[1])
+ def test_room_event_context_filter_query_validation(self) -> None:
+ # Test json validation in (filter) query parameter.
+ # Does not test the validity of the filter, only the json validation.
+ event_id = self.helper.send(self.room_id, "message 7", tok=self.tok)["event_id"]
+
+ # Check Get with valid json filter parameter, expect 200.
+ valid_filter_str = '{"types": ["m.room.message"]}'
+ channel = self.make_request(
+ "GET",
+ f"/rooms/{self.room_id}/context/{event_id}?filter={valid_filter_str}",
+ access_token=self.tok,
+ )
+ self.assertEqual(channel.code, HTTPStatus.OK, channel.json_body)
+
+ # Check Get with invalid json filter parameter, expect 400 NOT_JSON.
+ invalid_filter_str = "}}}{}"
+ channel = self.make_request(
+ "GET",
+ f"/rooms/{self.room_id}/context/{event_id}?filter={invalid_filter_str}",
+ access_token=self.tok,
+ )
+
+ self.assertEqual(channel.code, HTTPStatus.BAD_REQUEST, channel.json_body)
+ self.assertEqual(
+ channel.json_body["errcode"], Codes.NOT_JSON, channel.json_body
+ )
+
class RoomAliasListTestCase(unittest.HomeserverTestCase):
servlets = [
|
