diff --git a/CHANGES.md b/CHANGES.md
index 3f0c5685ea..238eb8a4ed 100644
--- a/CHANGES.md
+++ b/CHANGES.md
@@ -1,3 +1,73 @@
+Synapse 1.30.0rc1 (2021-03-16)
+==============================
+
+Features
+--------
+
+- Add prometheus metrics for number of users successfully registering and logging in. ([\#9510](https://github.com/matrix-org/synapse/issues/9510), [\#9511](https://github.com/matrix-org/synapse/issues/9511), [\#9573](https://github.com/matrix-org/synapse/issues/9573))
+- Add `synapse_federation_last_sent_pdu_time` and `synapse_federation_last_received_pdu_time` prometheus metrics, which monitor federation delays by reporting the timestamps of messages sent and received to a set of remote servers. ([\#9540](https://github.com/matrix-org/synapse/issues/9540))
+- Add support for generating JSON Web Tokens dynamically for use as OIDC client secrets. ([\#9549](https://github.com/matrix-org/synapse/issues/9549))
+- Optimise handling of incomplete room history for incoming federation. ([\#9601](https://github.com/matrix-org/synapse/issues/9601))
+- Finalise support for allowing clients to pick an SSO Identity Provider ([MSC2858](https://github.com/matrix-org/matrix-doc/pull/2858)). ([\#9617](https://github.com/matrix-org/synapse/issues/9617))
+- Tell spam checker modules about the SSO IdP a user registered through if one was used. ([\#9626](https://github.com/matrix-org/synapse/issues/9626))
+
+
+Bugfixes
+--------
+
+- Fix long-standing bug when generating thumbnails for some images with transparency: `TypeError: cannot unpack non-iterable int object`. ([\#9473](https://github.com/matrix-org/synapse/issues/9473))
+- Purge chain cover indexes for events that were purged prior to Synapse v1.29.0. ([\#9542](https://github.com/matrix-org/synapse/issues/9542), [\#9583](https://github.com/matrix-org/synapse/issues/9583))
+- Fix bug where federation requests were not correctly retried on 5xx responses. ([\#9567](https://github.com/matrix-org/synapse/issues/9567))
+- Re-Activating account with admin API when local passwords are disabled. ([\#9587](https://github.com/matrix-org/synapse/issues/9587))
+- Fix a bug introduced in Synapse 1.20 which caused incoming federation transactions to stack up, causing slow recovery from outages. ([\#9597](https://github.com/matrix-org/synapse/issues/9597))
+- Fix a bug introduced in v1.28.0 where the OpenID Connect callback endpoint could error with a `MacaroonInitException`. ([\#9620](https://github.com/matrix-org/synapse/issues/9620))
+- Fix Internal Server Error on `GET /_synapse/client/saml2/authn_response` request. ([\#9623](https://github.com/matrix-org/synapse/issues/9623))
+
+
+Updates to the Docker image
+---------------------------
+
+- Use jemalloc if available in docker. ([\#8553](https://github.com/matrix-org/synapse/issues/8553))
+
+
+Improved Documentation
+----------------------
+
+- Add relayd entry to reverse proxy example configurations. ([\#9508](https://github.com/matrix-org/synapse/issues/9508))
+- Improve the SAML2 upgrade notes for 1.27.0. ([\#9550](https://github.com/matrix-org/synapse/issues/9550))
+- Link to the "List user's media" admin API from the media admin API docs. ([\#9571](https://github.com/matrix-org/synapse/issues/9571))
+- Clarify the spam checker modules documentation example to mention that `parse_config` is a required method. ([\#9580](https://github.com/matrix-org/synapse/issues/9580))
+- Clarify the sample configuration for `stats` settings. ([\#9604](https://github.com/matrix-org/synapse/issues/9604))
+
+
+Deprecations and Removals
+-------------------------
+
+- The `synapse_federation_last_sent_pdu_age` and `synapse_federation_last_received_pdu_age` prometheus metrics have been removed. They are replaced by `synapse_federation_last_sent_pdu_time` and `synapse_federation_last_received_pdu_time`. ([\#9540](https://github.com/matrix-org/synapse/issues/9540))
+- Registering an Application Service user without using the `m.login.application_service` login type will be unsupported in an upcoming Synapse release. ([\#9559](https://github.com/matrix-org/synapse/issues/9559))
+
+
+Internal Changes
+----------------
+
+- Add tests to ResponseCache. ([\#9458](https://github.com/matrix-org/synapse/issues/9458))
+- Add type hints to purge room and server notice admin API. ([\#9520](https://github.com/matrix-org/synapse/issues/9520))
+- Add extra logging to ObservableDeferred when callbacks throw exceptions. ([\#9523](https://github.com/matrix-org/synapse/issues/9523))
+- Fix incorrect type hints. ([\#9528](https://github.com/matrix-org/synapse/issues/9528), [\#9543](https://github.com/matrix-org/synapse/issues/9543), [\#9591](https://github.com/matrix-org/synapse/issues/9591), [\#9608](https://github.com/matrix-org/synapse/issues/9608), [\#9618](https://github.com/matrix-org/synapse/issues/9618))
+- Add an additional test for purging a room. ([\#9541](https://github.com/matrix-org/synapse/issues/9541))
+- Add a `.git-blame-ignore-revs` file with the hashes of auto-formatting. ([\#9560](https://github.com/matrix-org/synapse/issues/9560))
+- Increase the threshold before which outbound federation to a server goes into "catch up" mode, which is expensive for the remote server to handle. ([\#9561](https://github.com/matrix-org/synapse/issues/9561))
+- Fix spurious errors reported by the `config-lint.sh` script. ([\#9562](https://github.com/matrix-org/synapse/issues/9562))
+- Fix type hints and tests for BlacklistingAgentWrapper and BlacklistingReactorWrapper. ([\#9563](https://github.com/matrix-org/synapse/issues/9563))
+- Do not have mypy ignore type hints from unpaddedbase64. ([\#9568](https://github.com/matrix-org/synapse/issues/9568))
+- Improve efficiency of calculating the auth chain in large rooms. ([\#9576](https://github.com/matrix-org/synapse/issues/9576))
+- Convert `synapse.types.Requester` to an `attrs` class. ([\#9586](https://github.com/matrix-org/synapse/issues/9586))
+- Add logging for redis connection setup. ([\#9590](https://github.com/matrix-org/synapse/issues/9590))
+- Improve logging when processing incoming transactions. ([\#9596](https://github.com/matrix-org/synapse/issues/9596))
+- Remove unused `stats.retention` setting, and emit a warning if stats are disabled. ([\#9604](https://github.com/matrix-org/synapse/issues/9604))
+- Prevent attempting to bundle aggregations for state events in /context APIs. ([\#9619](https://github.com/matrix-org/synapse/issues/9619))
+
+
Removal warning
---------------
diff --git a/changelog.d/9458.misc b/changelog.d/9458.misc
deleted file mode 100644
index 8ceeed1352..0000000000
--- a/changelog.d/9458.misc
+++ /dev/null
@@ -1 +0,0 @@
-Add tests to ResponseCache.
\ No newline at end of file
diff --git a/changelog.d/9473.bugfix b/changelog.d/9473.bugfix
deleted file mode 100644
index 71fb487cf2..0000000000
--- a/changelog.d/9473.bugfix
+++ /dev/null
@@ -1 +0,0 @@
-Fix long-standing bug when generating thumbnails for some images with transparency: `TypeError: cannot unpack non-iterable int object`.
diff --git a/changelog.d/9508.doc b/changelog.d/9508.doc
deleted file mode 100644
index a17a8faecf..0000000000
--- a/changelog.d/9508.doc
+++ /dev/null
@@ -1 +0,0 @@
-Add relayd entry to reverse proxy example configurations.
diff --git a/changelog.d/9510.feature b/changelog.d/9510.feature
deleted file mode 100644
index 5214b50d41..0000000000
--- a/changelog.d/9510.feature
+++ /dev/null
@@ -1 +0,0 @@
-Add prometheus metrics for number of users successfully registering and logging in.
diff --git a/changelog.d/9511.feature b/changelog.d/9511.feature
deleted file mode 100644
index 5214b50d41..0000000000
--- a/changelog.d/9511.feature
+++ /dev/null
@@ -1 +0,0 @@
-Add prometheus metrics for number of users successfully registering and logging in.
diff --git a/changelog.d/9520.misc b/changelog.d/9520.misc
deleted file mode 100644
index 825ba5bbc1..0000000000
--- a/changelog.d/9520.misc
+++ /dev/null
@@ -1 +0,0 @@
-Add type hints to purge room and server notice admin API.
\ No newline at end of file
diff --git a/changelog.d/9523.misc b/changelog.d/9523.misc
deleted file mode 100644
index f03e939efb..0000000000
--- a/changelog.d/9523.misc
+++ /dev/null
@@ -1 +0,0 @@
-Add extra logging to ObservableDeferred when callbacks throw exceptions.
\ No newline at end of file
diff --git a/changelog.d/9528.misc b/changelog.d/9528.misc
deleted file mode 100644
index 14c7b78dd9..0000000000
--- a/changelog.d/9528.misc
+++ /dev/null
@@ -1 +0,0 @@
-Fix incorrect type hints.
diff --git a/changelog.d/9540.feature b/changelog.d/9540.feature
deleted file mode 100644
index 5417e51b93..0000000000
--- a/changelog.d/9540.feature
+++ /dev/null
@@ -1 +0,0 @@
-Add `synapse_federation_last_sent_pdu_time` and `synapse_federation_last_received_pdu_time` prometheus metrics, which monitor federation delays by reporting the timestamps of messages sent and received to a set of remote servers.
diff --git a/changelog.d/9540.removal b/changelog.d/9540.removal
deleted file mode 100644
index d54f553cb9..0000000000
--- a/changelog.d/9540.removal
+++ /dev/null
@@ -1 +0,0 @@
-The `synapse_federation_last_sent_pdu_age` and `synapse_federation_last_received_pdu_age` prometheus metrics have been removed. They are replaced by `synapse_federation_last_sent_pdu_time` and `synapse_federation_last_received_pdu_time`.
diff --git a/changelog.d/9541.misc b/changelog.d/9541.misc
deleted file mode 100644
index a82bef3431..0000000000
--- a/changelog.d/9541.misc
+++ /dev/null
@@ -1 +0,0 @@
-Add an additional test for purging a room.
diff --git a/changelog.d/9542.bugfix b/changelog.d/9542.bugfix
deleted file mode 100644
index 51b1876f3b..0000000000
--- a/changelog.d/9542.bugfix
+++ /dev/null
@@ -1 +0,0 @@
-Purge chain cover indexes for events that were purged prior to Synapse v1.29.0.
diff --git a/changelog.d/9543.misc b/changelog.d/9543.misc
deleted file mode 100644
index 14c7b78dd9..0000000000
--- a/changelog.d/9543.misc
+++ /dev/null
@@ -1 +0,0 @@
-Fix incorrect type hints.
diff --git a/changelog.d/9549.feature b/changelog.d/9549.feature
deleted file mode 100644
index 709e61eced..0000000000
--- a/changelog.d/9549.feature
+++ /dev/null
@@ -1 +0,0 @@
-Add support for generating JSON Web Tokens dynamically for use as OIDC client secrets.
diff --git a/changelog.d/9550.doc b/changelog.d/9550.doc
deleted file mode 100644
index adbbeb0ae4..0000000000
--- a/changelog.d/9550.doc
+++ /dev/null
@@ -1 +0,0 @@
-Improve the SAML2 upgrade notes for 1.27.0.
diff --git a/changelog.d/9559.removal b/changelog.d/9559.removal
deleted file mode 100644
index f97bf56dc0..0000000000
--- a/changelog.d/9559.removal
+++ /dev/null
@@ -1 +0,0 @@
-Registering an Application Service user without using the `m.login.application_service` login type will be unsupported in an upcoming Synapse release.
diff --git a/changelog.d/9560.misc b/changelog.d/9560.misc
deleted file mode 100644
index 57a698f846..0000000000
--- a/changelog.d/9560.misc
+++ /dev/null
@@ -1 +0,0 @@
-Add a `.git-blame-ignore-revs` file with the hashes of auto-formatting.
diff --git a/changelog.d/9561.misc b/changelog.d/9561.misc
deleted file mode 100644
index 6c529a82ee..0000000000
--- a/changelog.d/9561.misc
+++ /dev/null
@@ -1 +0,0 @@
-Increase the threshold before which outbound federation to a server goes into "catch up" mode, which is expensive for the remote server to handle.
diff --git a/changelog.d/9562.misc b/changelog.d/9562.misc
deleted file mode 100644
index 2f0133bff0..0000000000
--- a/changelog.d/9562.misc
+++ /dev/null
@@ -1 +0,0 @@
-Fix spurious errors reported by the `config-lint.sh` script.
\ No newline at end of file
diff --git a/changelog.d/9563.misc b/changelog.d/9563.misc
deleted file mode 100644
index 7a3493e4a1..0000000000
--- a/changelog.d/9563.misc
+++ /dev/null
@@ -1 +0,0 @@
-Fix type hints and tests for BlacklistingAgentWrapper and BlacklistingReactorWrapper.
diff --git a/changelog.d/9567.bugfix b/changelog.d/9567.bugfix
deleted file mode 100644
index e7322c2b5e..0000000000
--- a/changelog.d/9567.bugfix
+++ /dev/null
@@ -1 +0,0 @@
-Fix bug where federation requests were not correctly retried on 5xx responses.
diff --git a/changelog.d/9568.misc b/changelog.d/9568.misc
deleted file mode 100644
index 561963de93..0000000000
--- a/changelog.d/9568.misc
+++ /dev/null
@@ -1 +0,0 @@
-Do not have mypy ignore type hints from unpaddedbase64.
diff --git a/changelog.d/9571.doc b/changelog.d/9571.doc
deleted file mode 100644
index 1bba72e7d0..0000000000
--- a/changelog.d/9571.doc
+++ /dev/null
@@ -1 +0,0 @@
-Link to the "List user's media" admin API from the media admin API docs.
diff --git a/changelog.d/9573.feature b/changelog.d/9573.feature
deleted file mode 100644
index 5214b50d41..0000000000
--- a/changelog.d/9573.feature
+++ /dev/null
@@ -1 +0,0 @@
-Add prometheus metrics for number of users successfully registering and logging in.
diff --git a/changelog.d/9576.misc b/changelog.d/9576.misc
deleted file mode 100644
index bc257d05b7..0000000000
--- a/changelog.d/9576.misc
+++ /dev/null
@@ -1 +0,0 @@
-Improve efficiency of calculating the auth chain in large rooms.
diff --git a/changelog.d/9580.doc b/changelog.d/9580.doc
deleted file mode 100644
index f9c8b328b3..0000000000
--- a/changelog.d/9580.doc
+++ /dev/null
@@ -1 +0,0 @@
-Clarify the spam checker modules documentation example to mention that `parse_config` is a required method.
diff --git a/changelog.d/9583.bugfix b/changelog.d/9583.bugfix
deleted file mode 100644
index 51b1876f3b..0000000000
--- a/changelog.d/9583.bugfix
+++ /dev/null
@@ -1 +0,0 @@
-Purge chain cover indexes for events that were purged prior to Synapse v1.29.0.
diff --git a/changelog.d/9586.misc b/changelog.d/9586.misc
deleted file mode 100644
index 2def9d5f55..0000000000
--- a/changelog.d/9586.misc
+++ /dev/null
@@ -1 +0,0 @@
-Convert `synapse.types.Requester` to an `attrs` class.
diff --git a/changelog.d/9587.bugfix b/changelog.d/9587.bugfix
deleted file mode 100644
index d8f04c4f21..0000000000
--- a/changelog.d/9587.bugfix
+++ /dev/null
@@ -1 +0,0 @@
-Re-Activating account with admin API when local passwords are disabled.
\ No newline at end of file
diff --git a/changelog.d/9590.misc b/changelog.d/9590.misc
deleted file mode 100644
index 186396c45b..0000000000
--- a/changelog.d/9590.misc
+++ /dev/null
@@ -1 +0,0 @@
-Add logging for redis connection setup.
diff --git a/changelog.d/9591.misc b/changelog.d/9591.misc
deleted file mode 100644
index 14c7b78dd9..0000000000
--- a/changelog.d/9591.misc
+++ /dev/null
@@ -1 +0,0 @@
-Fix incorrect type hints.
diff --git a/changelog.d/9596.misc b/changelog.d/9596.misc
deleted file mode 100644
index fc19a95f75..0000000000
--- a/changelog.d/9596.misc
+++ /dev/null
@@ -1 +0,0 @@
-Improve logging when processing incoming transactions.
diff --git a/changelog.d/9597.bugfix b/changelog.d/9597.bugfix
deleted file mode 100644
index 349dc9d664..0000000000
--- a/changelog.d/9597.bugfix
+++ /dev/null
@@ -1 +0,0 @@
-Fix a bug introduced in Synapse 1.20 which caused incoming federation transactions to stack up, causing slow recovery from outages.
diff --git a/changelog.d/9601.feature b/changelog.d/9601.feature
deleted file mode 100644
index 5078d63ffa..0000000000
--- a/changelog.d/9601.feature
+++ /dev/null
@@ -1 +0,0 @@
-Optimise handling of incomplete room history for incoming federation.
diff --git a/changelog.d/9608.misc b/changelog.d/9608.misc
deleted file mode 100644
index 14c7b78dd9..0000000000
--- a/changelog.d/9608.misc
+++ /dev/null
@@ -1 +0,0 @@
-Fix incorrect type hints.
diff --git a/changelog.d/9618.misc b/changelog.d/9618.misc
deleted file mode 100644
index 14c7b78dd9..0000000000
--- a/changelog.d/9618.misc
+++ /dev/null
@@ -1 +0,0 @@
-Fix incorrect type hints.
diff --git a/changelog.d/9623.bugfix b/changelog.d/9623.bugfix
deleted file mode 100644
index ecccb46105..0000000000
--- a/changelog.d/9623.bugfix
+++ /dev/null
@@ -1 +0,0 @@
-Fix Internal Server Error on `GET /_synapse/client/saml2/authn_response` request.
diff --git a/docker/Dockerfile b/docker/Dockerfile
index d619ee08ed..def4501541 100644
--- a/docker/Dockerfile
+++ b/docker/Dockerfile
@@ -69,6 +69,7 @@ RUN apt-get update && apt-get install -y \
libpq5 \
libwebp6 \
xmlsec1 \
+ libjemalloc2 \
&& rm -rf /var/lib/apt/lists/*
COPY --from=builder /install /usr/local
diff --git a/docker/README.md b/docker/README.md
index 7b138df4d3..3a7dc585e7 100644
--- a/docker/README.md
+++ b/docker/README.md
@@ -204,3 +204,8 @@ healthcheck:
timeout: 10s
retries: 3
```
+
+## Using jemalloc
+
+Jemalloc is embedded in the image and will be used instead of the default allocator.
+You can read about jemalloc by reading the Synapse [README](../README.md)
\ No newline at end of file
diff --git a/docker/start.py b/docker/start.py
index 0d2c590b88..16d6a8208a 100755
--- a/docker/start.py
+++ b/docker/start.py
@@ -3,6 +3,7 @@
import codecs
import glob
import os
+import platform
import subprocess
import sys
@@ -213,6 +214,13 @@ def main(args, environ):
if "-m" not in args:
args = ["-m", synapse_worker] + args
+ jemallocpath = "/usr/lib/%s-linux-gnu/libjemalloc.so.2" % (platform.machine(),)
+
+ if os.path.isfile(jemallocpath):
+ environ["LD_PRELOAD"] = jemallocpath
+ else:
+ log("Could not find %s, will not use" % (jemallocpath,))
+
# if there are no config files passed to synapse, try adding the default file
if not any(p.startswith("--config-path") or p.startswith("-c") for p in args):
config_dir = environ.get("SYNAPSE_CONFIG_DIR", "/data")
@@ -248,9 +256,9 @@ running with 'migrate_config'. See the README for more details.
args = ["python"] + args
if ownership is not None:
args = ["gosu", ownership] + args
- os.execv("/usr/sbin/gosu", args)
+ os.execve("/usr/sbin/gosu", args, environ)
else:
- os.execv("/usr/local/bin/python", args)
+ os.execve("/usr/local/bin/python", args, environ)
if __name__ == "__main__":
diff --git a/docs/openid.md b/docs/openid.md
index 01205d1220..cfaafc5015 100644
--- a/docs/openid.md
+++ b/docs/openid.md
@@ -226,7 +226,7 @@ Synapse config:
oidc_providers:
- idp_id: github
idp_name: Github
- idp_brand: "org.matrix.github" # optional: styling hint for clients
+ idp_brand: "github" # optional: styling hint for clients
discover: false
issuer: "https://github.com/"
client_id: "your-client-id" # TO BE FILLED
@@ -252,7 +252,7 @@ oidc_providers:
oidc_providers:
- idp_id: google
idp_name: Google
- idp_brand: "org.matrix.google" # optional: styling hint for clients
+ idp_brand: "google" # optional: styling hint for clients
issuer: "https://accounts.google.com/"
client_id: "your-client-id" # TO BE FILLED
client_secret: "your-client-secret" # TO BE FILLED
@@ -299,7 +299,7 @@ Synapse config:
oidc_providers:
- idp_id: gitlab
idp_name: Gitlab
- idp_brand: "org.matrix.gitlab" # optional: styling hint for clients
+ idp_brand: "gitlab" # optional: styling hint for clients
issuer: "https://gitlab.com/"
client_id: "your-client-id" # TO BE FILLED
client_secret: "your-client-secret" # TO BE FILLED
@@ -334,7 +334,7 @@ Synapse config:
```yaml
- idp_id: facebook
idp_name: Facebook
- idp_brand: "org.matrix.facebook" # optional: styling hint for clients
+ idp_brand: "facebook" # optional: styling hint for clients
discover: false
issuer: "https://facebook.com"
client_id: "your-client-id" # TO BE FILLED
diff --git a/docs/sample_config.yaml b/docs/sample_config.yaml
index e98391684e..1dfba068e6 100644
--- a/docs/sample_config.yaml
+++ b/docs/sample_config.yaml
@@ -2099,7 +2099,7 @@ oidc_providers:
#
#- idp_id: github
# idp_name: Github
- # idp_brand: org.matrix.github
+ # idp_brand: github
# discover: false
# issuer: "https://github.com/"
# client_id: "your-client-id" # TO BE FILLED
@@ -2831,19 +2831,20 @@ user_directory:
-# Local statistics collection. Used in populating the room directory.
+# Settings for local room and user statistics collection. See
+# docs/room_and_user_statistics.md.
#
-# 'bucket_size' controls how large each statistics timeslice is. It can
-# be defined in a human readable short form -- e.g. "1d", "1y".
-#
-# 'retention' controls how long historical statistics will be kept for.
-# It can be defined in a human readable short form -- e.g. "1d", "1y".
-#
-#
-#stats:
-# enabled: true
-# bucket_size: 1d
-# retention: 1y
+stats:
+ # Uncomment the following to disable room and user statistics. Note that doing
+ # so may cause certain features (such as the room directory) not to work
+ # correctly.
+ #
+ #enabled: false
+
+ # The size of each timeslice in the room_stats_historical and
+ # user_stats_historical tables, as a time period. Defaults to "1d".
+ #
+ #bucket_size: 1h
# Server Notices room configuration
diff --git a/docs/spam_checker.md b/docs/spam_checker.md
index 2020eb9006..52947f605e 100644
--- a/docs/spam_checker.md
+++ b/docs/spam_checker.md
@@ -69,7 +69,13 @@ class ExampleSpamChecker:
async def check_username_for_spam(self, user_profile):
return False # allow all usernames
- async def check_registration_for_spam(self, email_threepid, username, request_info):
+ async def check_registration_for_spam(
+ self,
+ email_threepid,
+ username,
+ request_info,
+ auth_provider_id,
+ ):
return RegistrationBehaviour.ALLOW # allow all registrations
async def check_media_file_for_spam(self, file_wrapper, file_info):
diff --git a/synapse/__init__.py b/synapse/__init__.py
index 56ca888862..88be7db196 100644
--- a/synapse/__init__.py
+++ b/synapse/__init__.py
@@ -48,7 +48,7 @@ try:
except ImportError:
pass
-__version__ = "1.29.0"
+__version__ = "1.30.0rc1"
if bool(os.environ.get("SYNAPSE_TEST_PATCH_LOG_CONTEXTS", False)):
# We import here so that we don't have to install a bunch of deps when
diff --git a/synapse/config/oidc_config.py b/synapse/config/oidc_config.py
index 7f5e449eb2..2bfb537c15 100644
--- a/synapse/config/oidc_config.py
+++ b/synapse/config/oidc_config.py
@@ -237,7 +237,7 @@ class OIDCConfig(Config):
#
#- idp_id: github
# idp_name: Github
- # idp_brand: org.matrix.github
+ # idp_brand: github
# discover: false
# issuer: "https://github.com/"
# client_id: "your-client-id" # TO BE FILLED
@@ -272,7 +272,12 @@ OIDC_PROVIDER_CONFIG_SCHEMA = {
"idp_icon": {"type": "string"},
"idp_brand": {
"type": "string",
- # MSC2758-style namespaced identifier
+ "minLength": 1,
+ "maxLength": 255,
+ "pattern": "^[a-z][a-z0-9_.-]*$",
+ },
+ "idp_unstable_brand": {
+ "type": "string",
"minLength": 1,
"maxLength": 255,
"pattern": "^[a-z][a-z0-9_.-]*$",
@@ -466,6 +471,7 @@ def _parse_oidc_config_dict(
idp_name=oidc_config.get("idp_name", "OIDC"),
idp_icon=idp_icon,
idp_brand=oidc_config.get("idp_brand"),
+ unstable_idp_brand=oidc_config.get("unstable_idp_brand"),
discover=oidc_config.get("discover", True),
issuer=oidc_config["issuer"],
client_id=oidc_config["client_id"],
@@ -512,6 +518,9 @@ class OidcProviderConfig:
# Optional brand identifier for this IdP.
idp_brand = attr.ib(type=Optional[str])
+ # Optional brand identifier for the unstable API (see MSC2858).
+ unstable_idp_brand = attr.ib(type=Optional[str])
+
# whether the OIDC discovery mechanism is used to discover endpoints
discover = attr.ib(type=bool)
diff --git a/synapse/config/stats.py b/synapse/config/stats.py
index b559bfa411..2258329a52 100644
--- a/synapse/config/stats.py
+++ b/synapse/config/stats.py
@@ -13,10 +13,22 @@
# See the License for the specific language governing permissions and
# limitations under the License.
-import sys
+import logging
from ._base import Config
+ROOM_STATS_DISABLED_WARN = """\
+WARNING: room/user statistics have been disabled via the stats.enabled
+configuration setting. This means that certain features (such as the room
+directory) will not operate correctly. Future versions of Synapse may ignore
+this setting.
+
+To fix this warning, remove the stats.enabled setting from your configuration
+file.
+--------------------------------------------------------------------------------"""
+
+logger = logging.getLogger(__name__)
+
class StatsConfig(Config):
"""Stats Configuration
@@ -28,30 +40,29 @@ class StatsConfig(Config):
def read_config(self, config, **kwargs):
self.stats_enabled = True
self.stats_bucket_size = 86400 * 1000
- self.stats_retention = sys.maxsize
stats_config = config.get("stats", None)
if stats_config:
self.stats_enabled = stats_config.get("enabled", self.stats_enabled)
self.stats_bucket_size = self.parse_duration(
stats_config.get("bucket_size", "1d")
)
- self.stats_retention = self.parse_duration(
- stats_config.get("retention", "%ds" % (sys.maxsize,))
- )
+ if not self.stats_enabled:
+ logger.warning(ROOM_STATS_DISABLED_WARN)
def generate_config_section(self, config_dir_path, server_name, **kwargs):
return """
- # Local statistics collection. Used in populating the room directory.
+ # Settings for local room and user statistics collection. See
+ # docs/room_and_user_statistics.md.
#
- # 'bucket_size' controls how large each statistics timeslice is. It can
- # be defined in a human readable short form -- e.g. "1d", "1y".
- #
- # 'retention' controls how long historical statistics will be kept for.
- # It can be defined in a human readable short form -- e.g. "1d", "1y".
- #
- #
- #stats:
- # enabled: true
- # bucket_size: 1d
- # retention: 1y
+ stats:
+ # Uncomment the following to disable room and user statistics. Note that doing
+ # so may cause certain features (such as the room directory) not to work
+ # correctly.
+ #
+ #enabled: false
+
+ # The size of each timeslice in the room_stats_historical and
+ # user_stats_historical tables, as a time period. Defaults to "1d".
+ #
+ #bucket_size: 1h
"""
diff --git a/synapse/events/spamcheck.py b/synapse/events/spamcheck.py
index 063af7a81d..b5a9c71ee6 100644
--- a/synapse/events/spamcheck.py
+++ b/synapse/events/spamcheck.py
@@ -15,6 +15,7 @@
# limitations under the License.
import inspect
+import logging
from typing import TYPE_CHECKING, Any, Dict, List, Optional, Tuple, Union
from synapse.rest.media.v1._base import FileInfo
@@ -27,6 +28,8 @@ if TYPE_CHECKING:
import synapse.events
import synapse.server
+logger = logging.getLogger(__name__)
+
class SpamChecker:
def __init__(self, hs: "synapse.server.HomeServer"):
@@ -246,6 +249,7 @@ class SpamChecker:
email_threepid: Optional[dict],
username: Optional[str],
request_info: Collection[Tuple[str, str]],
+ auth_provider_id: Optional[str] = None,
) -> RegistrationBehaviour:
"""Checks if we should allow the given registration request.
@@ -254,6 +258,9 @@ class SpamChecker:
username: The request user name, if any
request_info: List of tuples of user agent and IP that
were used during the registration process.
+ auth_provider_id: The SSO IdP the user used, e.g "oidc", "saml",
+ "cas". If any. Note this does not include users registered
+ via a password provider.
Returns:
Enum for how the request should be handled
@@ -264,9 +271,25 @@ class SpamChecker:
# spam checker
checker = getattr(spam_checker, "check_registration_for_spam", None)
if checker:
- behaviour = await maybe_awaitable(
- checker(email_threepid, username, request_info)
- )
+ # Provide auth_provider_id if the function supports it
+ checker_args = inspect.signature(checker)
+ if len(checker_args.parameters) == 4:
+ d = checker(
+ email_threepid,
+ username,
+ request_info,
+ auth_provider_id,
+ )
+ elif len(checker_args.parameters) == 3:
+ d = checker(email_threepid, username, request_info)
+ else:
+ logger.error(
+ "Invalid signature for %s.check_registration_for_spam. Denying registration",
+ spam_checker.__module__,
+ )
+ return RegistrationBehaviour.DENY
+
+ behaviour = await maybe_awaitable(d)
assert isinstance(behaviour, RegistrationBehaviour)
if behaviour != RegistrationBehaviour.ALLOW:
return behaviour
diff --git a/synapse/handlers/cas_handler.py b/synapse/handlers/cas_handler.py
index 04972f9cf0..cb67589f7d 100644
--- a/synapse/handlers/cas_handler.py
+++ b/synapse/handlers/cas_handler.py
@@ -83,6 +83,7 @@ class CasHandler:
# the SsoIdentityProvider protocol type.
self.idp_icon = None
self.idp_brand = None
+ self.unstable_idp_brand = None
self._sso_handler = hs.get_sso_handler()
diff --git a/synapse/handlers/oidc_handler.py b/synapse/handlers/oidc_handler.py
index f5d1821127..6d8551a6d6 100644
--- a/synapse/handlers/oidc_handler.py
+++ b/synapse/handlers/oidc_handler.py
@@ -29,6 +29,7 @@ from authlib.oidc.discovery import OpenIDProviderMetadata, get_well_known_url
from jinja2 import Environment, Template
from pymacaroons.exceptions import (
MacaroonDeserializationException,
+ MacaroonInitException,
MacaroonInvalidSignatureException,
)
from typing_extensions import TypedDict
@@ -217,7 +218,7 @@ class OidcHandler:
session_data = self._token_generator.verify_oidc_session_token(
session, state
)
- except (MacaroonDeserializationException, KeyError) as e:
+ except (MacaroonInitException, MacaroonDeserializationException, KeyError) as e:
logger.exception("Invalid session for OIDC callback")
self._sso_handler.render_error(request, "invalid_session", str(e))
return
@@ -330,6 +331,9 @@ class OidcProvider:
# optional brand identifier for this auth provider
self.idp_brand = provider.idp_brand
+ # Optional brand identifier for the unstable API (see MSC2858).
+ self.unstable_idp_brand = provider.unstable_idp_brand
+
self._sso_handler = hs.get_sso_handler()
self._sso_handler.register_identity_provider(self)
diff --git a/synapse/handlers/register.py b/synapse/handlers/register.py
index ac004ca7b9..e348d8be3b 100644
--- a/synapse/handlers/register.py
+++ b/synapse/handlers/register.py
@@ -213,8 +213,7 @@ class RegistrationHandler(BaseHandler):
admin api, otherwise False.
user_agent_ips: Tuples of IP addresses and user-agents used
during the registration process.
- auth_provider_id: The SSO IdP the user used, if any (just used for the
- prometheus metrics).
+ auth_provider_id: The SSO IdP the user used, if any.
Returns:
The registered user_id.
Raises:
@@ -226,6 +225,7 @@ class RegistrationHandler(BaseHandler):
threepid,
localpart,
user_agent_ips or [],
+ auth_provider_id=auth_provider_id,
)
if result == RegistrationBehaviour.DENY:
diff --git a/synapse/handlers/saml_handler.py b/synapse/handlers/saml_handler.py
index a9645b77d8..ec2ba11c75 100644
--- a/synapse/handlers/saml_handler.py
+++ b/synapse/handlers/saml_handler.py
@@ -81,6 +81,7 @@ class SamlHandler(BaseHandler):
# the SsoIdentityProvider protocol type.
self.idp_icon = None
self.idp_brand = None
+ self.unstable_idp_brand = None
# a map from saml session id to Saml2SessionData object
self._outstanding_requests_dict = {} # type: Dict[str, Saml2SessionData]
diff --git a/synapse/handlers/sso.py b/synapse/handlers/sso.py
index 6ef459acff..415b1c2d17 100644
--- a/synapse/handlers/sso.py
+++ b/synapse/handlers/sso.py
@@ -98,6 +98,11 @@ class SsoIdentityProvider(Protocol):
"""Optional branding identifier"""
return None
+ @property
+ def unstable_idp_brand(self) -> Optional[str]:
+ """Optional brand identifier for the unstable API (see MSC2858)."""
+ return None
+
@abc.abstractmethod
async def handle_redirect_request(
self,
diff --git a/synapse/rest/admin/rooms.py b/synapse/rest/admin/rooms.py
index f2c42a0f30..263d8ec076 100644
--- a/synapse/rest/admin/rooms.py
+++ b/synapse/rest/admin/rooms.py
@@ -685,7 +685,10 @@ class RoomEventContextServlet(RestServlet):
results["events_after"], time_now
)
results["state"] = await self._event_serializer.serialize_events(
- results["state"], time_now
+ results["state"],
+ time_now,
+ # No need to bundle aggregations for state events
+ bundle_aggregations=False,
)
return 200, results
diff --git a/synapse/rest/client/v1/login.py b/synapse/rest/client/v1/login.py
index 34bc1bd49b..e4c352f572 100644
--- a/synapse/rest/client/v1/login.py
+++ b/synapse/rest/client/v1/login.py
@@ -14,10 +14,12 @@
# limitations under the License.
import logging
+import re
from typing import TYPE_CHECKING, Awaitable, Callable, Dict, Optional
from synapse.api.errors import Codes, LoginError, SynapseError
from synapse.api.ratelimiting import Ratelimiter
+from synapse.api.urls import CLIENT_API_PREFIX
from synapse.appservice import ApplicationService
from synapse.handlers.sso import SsoIdentityProvider
from synapse.http import get_request_uri
@@ -94,11 +96,21 @@ class LoginRestServlet(RestServlet):
flows.append({"type": LoginRestServlet.CAS_TYPE})
if self.cas_enabled or self.saml2_enabled or self.oidc_enabled:
- sso_flow = {"type": LoginRestServlet.SSO_TYPE} # type: JsonDict
+ sso_flow = {
+ "type": LoginRestServlet.SSO_TYPE,
+ "identity_providers": [
+ _get_auth_flow_dict_for_idp(
+ idp,
+ )
+ for idp in self._sso_handler.get_identity_providers().values()
+ ],
+ } # type: JsonDict
if self._msc2858_enabled:
+ # backwards-compatibility support for clients which don't
+ # support the stable API yet
sso_flow["org.matrix.msc2858.identity_providers"] = [
- _get_auth_flow_dict_for_idp(idp)
+ _get_auth_flow_dict_for_idp(idp, use_unstable_brands=True)
for idp in self._sso_handler.get_identity_providers().values()
]
@@ -331,22 +343,38 @@ class LoginRestServlet(RestServlet):
return result
-def _get_auth_flow_dict_for_idp(idp: SsoIdentityProvider) -> JsonDict:
+def _get_auth_flow_dict_for_idp(
+ idp: SsoIdentityProvider, use_unstable_brands: bool = False
+) -> JsonDict:
"""Return an entry for the login flow dict
Returns an entry suitable for inclusion in "identity_providers" in the
response to GET /_matrix/client/r0/login
+
+ Args:
+ idp: the identity provider to describe
+ use_unstable_brands: whether we should use brand identifiers suitable
+ for the unstable API
"""
e = {"id": idp.idp_id, "name": idp.idp_name} # type: JsonDict
if idp.idp_icon:
e["icon"] = idp.idp_icon
if idp.idp_brand:
e["brand"] = idp.idp_brand
+ # use the stable brand identifier if the unstable identifier isn't defined.
+ if use_unstable_brands and idp.unstable_idp_brand:
+ e["brand"] = idp.unstable_idp_brand
return e
class SsoRedirectServlet(RestServlet):
- PATTERNS = client_patterns("/login/(cas|sso)/redirect$", v1=True)
+ PATTERNS = list(client_patterns("/login/(cas|sso)/redirect$", v1=True)) + [
+ re.compile(
+ "^"
+ + CLIENT_API_PREFIX
+ + "/r0/login/sso/redirect/(?P<idp_id>[A-Za-z0-9_.~-]+)$"
+ )
+ ]
def __init__(self, hs: "HomeServer"):
# make sure that the relevant handlers are instantiated, so that they
@@ -364,7 +392,8 @@ class SsoRedirectServlet(RestServlet):
def register(self, http_server: HttpServer) -> None:
super().register(http_server)
if self._msc2858_enabled:
- # expose additional endpoint for MSC2858 support
+ # expose additional endpoint for MSC2858 support: backwards-compat support
+ # for clients which don't yet support the stable endpoints.
http_server.register_paths(
"GET",
client_patterns(
diff --git a/synapse/rest/client/v1/room.py b/synapse/rest/client/v1/room.py
index d2612fd067..95aaf51d23 100644
--- a/synapse/rest/client/v1/room.py
+++ b/synapse/rest/client/v1/room.py
@@ -669,7 +669,10 @@ class RoomEventContextServlet(RestServlet):
results["events_after"], time_now
)
results["state"] = await self._event_serializer.serialize_events(
- results["state"], time_now
+ results["state"],
+ time_now,
+ # No need to bundle aggregations for state events
+ bundle_aggregations=False,
)
return 200, results
diff --git a/tests/handlers/test_register.py b/tests/handlers/test_register.py
index aaf5b92f41..00a0bc5274 100644
--- a/tests/handlers/test_register.py
+++ b/tests/handlers/test_register.py
@@ -526,6 +526,37 @@ class RegistrationTestCase(unittest.HomeserverTestCase):
self.assertTrue(requester.shadow_banned)
+ def test_spam_checker_receives_sso_type(self):
+ """Test rejecting registration based on SSO type"""
+
+ class BanBadIdPUser:
+ def check_registration_for_spam(
+ self, email_threepid, username, request_info, auth_provider_id=None
+ ):
+ # Reject any user coming from CAS and whose username contains profanity
+ if auth_provider_id == "cas" and "flimflob" in username:
+ return RegistrationBehaviour.DENY
+ return RegistrationBehaviour.ALLOW
+
+ # Configure a spam checker that denies a certain user on a specific IdP
+ spam_checker = self.hs.get_spam_checker()
+ spam_checker.spam_checkers = [BanBadIdPUser()]
+
+ f = self.get_failure(
+ self.handler.register_user(localpart="bobflimflob", auth_provider_id="cas"),
+ SynapseError,
+ )
+ exception = f.value
+
+ # We return 429 from the spam checker for denied registrations
+ self.assertIsInstance(exception, SynapseError)
+ self.assertEqual(exception.code, 429)
+
+ # Check the same username can register using SAML
+ self.get_success(
+ self.handler.register_user(localpart="bobflimflob", auth_provider_id="saml")
+ )
+
def test_email_to_displayname_mapping(self):
"""Test that custom emails are mapped to new user displaynames correctly"""
self._check_mapping(
diff --git a/tests/rest/client/v1/test_login.py b/tests/rest/client/v1/test_login.py
index 20af3285bd..988821b16f 100644
--- a/tests/rest/client/v1/test_login.py
+++ b/tests/rest/client/v1/test_login.py
@@ -437,14 +437,16 @@ class MultiSSOTestCase(unittest.HomeserverTestCase):
channel = self.make_request("GET", "/_matrix/client/r0/login")
self.assertEqual(channel.code, 200, channel.result)
- expected_flows = [
- {"type": "m.login.cas"},
- {"type": "m.login.sso"},
- {"type": "m.login.token"},
- {"type": "m.login.password"},
- ] + ADDITIONAL_LOGIN_FLOWS
+ expected_flow_types = [
+ "m.login.cas",
+ "m.login.sso",
+ "m.login.token",
+ "m.login.password",
+ ] + [f["type"] for f in ADDITIONAL_LOGIN_FLOWS]
- self.assertCountEqual(channel.json_body["flows"], expected_flows)
+ self.assertCountEqual(
+ [f["type"] for f in channel.json_body["flows"]], expected_flow_types
+ )
@override_config({"experimental_features": {"msc2858_enabled": True}})
def test_get_msc2858_login_flows(self):
@@ -636,22 +638,25 @@ class MultiSSOTestCase(unittest.HomeserverTestCase):
)
self.assertEqual(channel.code, 400, channel.result)
- def test_client_idp_redirect_msc2858_disabled(self):
- """If the client tries to pick an IdP but MSC2858 is disabled, return a 400"""
- channel = self._make_sso_redirect_request(True, "oidc")
- self.assertEqual(channel.code, 400, channel.result)
- self.assertEqual(channel.json_body["errcode"], "M_UNRECOGNIZED")
-
- @override_config({"experimental_features": {"msc2858_enabled": True}})
def test_client_idp_redirect_to_unknown(self):
"""If the client tries to pick an unknown IdP, return a 404"""
- channel = self._make_sso_redirect_request(True, "xxx")
+ channel = self._make_sso_redirect_request(False, "xxx")
self.assertEqual(channel.code, 404, channel.result)
self.assertEqual(channel.json_body["errcode"], "M_NOT_FOUND")
- @override_config({"experimental_features": {"msc2858_enabled": True}})
def test_client_idp_redirect_to_oidc(self):
"""If the client pick a known IdP, redirect to it"""
+ channel = self._make_sso_redirect_request(False, "oidc")
+ self.assertEqual(channel.code, 302, channel.result)
+ oidc_uri = channel.headers.getRawHeaders("Location")[0]
+ oidc_uri_path, oidc_uri_query = oidc_uri.split("?", 1)
+
+ # it should redirect us to the auth page of the OIDC server
+ self.assertEqual(oidc_uri_path, TEST_OIDC_AUTH_ENDPOINT)
+
+ @override_config({"experimental_features": {"msc2858_enabled": True}})
+ def test_client_msc2858_redirect_to_oidc(self):
+ """Test the unstable API"""
channel = self._make_sso_redirect_request(True, "oidc")
self.assertEqual(channel.code, 302, channel.result)
oidc_uri = channel.headers.getRawHeaders("Location")[0]
@@ -660,6 +665,12 @@ class MultiSSOTestCase(unittest.HomeserverTestCase):
# it should redirect us to the auth page of the OIDC server
self.assertEqual(oidc_uri_path, TEST_OIDC_AUTH_ENDPOINT)
+ def test_client_idp_redirect_msc2858_disabled(self):
+ """If the client tries to use the MSC2858 endpoint but MSC2858 is disabled, return a 400"""
+ channel = self._make_sso_redirect_request(True, "oidc")
+ self.assertEqual(channel.code, 400, channel.result)
+ self.assertEqual(channel.json_body["errcode"], "M_UNRECOGNIZED")
+
def _make_sso_redirect_request(
self, unstable_endpoint: bool = False, idp_prov: Optional[str] = None
):
|
