diff options
| -rw-r--r-- | AUTHORS.rst | 3 | ||||
| -rw-r--r-- | README.rst | 32 | ||||
| -rw-r--r-- | changelog.d/4276.misc | 1 | ||||
| -rw-r--r-- | changelog.d/5015.misc | 1 | ||||
| -rw-r--r-- | changelog.d/5493.misc | 1 | ||||
| -rw-r--r-- | synapse/handlers/federation.py | 46 | ||||
| -rw-r--r-- | synapse/storage/registration.py | 2 |
7 files changed, 70 insertions, 16 deletions
diff --git a/AUTHORS.rst b/AUTHORS.rst index 3ea18eefcb..d8b4a846d8 100644 --- a/AUTHORS.rst +++ b/AUTHORS.rst @@ -72,3 +72,6 @@ Jason Robinson <jasonr at matrix.org> Joseph Weston <joseph at weston.cloud> + Add admin API for querying HS version + +Benjamin Saunders <ben.e.saunders at gmail dot com> + * Documentation improvements diff --git a/README.rst b/README.rst index 5409f0c563..13e11a5773 100644 --- a/README.rst +++ b/README.rst @@ -340,8 +340,11 @@ log lines and looking for any 'Processed request' lines which take more than a few seconds to execute. Please let us know at #synapse:matrix.org if you see this failure mode so we can help debug it, however. -Help!! Synapse eats all my RAM! -------------------------------- +Help!! Synapse is slow and eats all my RAM/CPU! +----------------------------------------------- + +First, ensure you are running the latest version of Synapse, using Python 3 +with a PostgreSQL database. Synapse's architecture is quite RAM hungry currently - we deliberately cache a lot of recent room data and metadata in RAM in order to speed up @@ -352,14 +355,29 @@ variable. The default is 0.5, which can be decreased to reduce RAM usage in memory constrained enviroments, or increased if performance starts to degrade. +However, degraded performance due to a low cache factor, common on +machines with slow disks, often leads to explosions in memory use due +backlogged requests. In this case, reducing the cache factor will make +things worse. Instead, try increasing it drastically. 2.0 is a good +starting value. + Using `libjemalloc <http://jemalloc.net/>`_ can also yield a significant -improvement in overall amount, and especially in terms of giving back RAM -to the OS. To use it, the library must simply be put in the LD_PRELOAD -environment variable when launching Synapse. On Debian, this can be done -by installing the ``libjemalloc1`` package and adding this line to -``/etc/default/matrix-synapse``:: +improvement in overall memory use, and especially in terms of giving back +RAM to the OS. To use it, the library must simply be put in the +LD_PRELOAD environment variable when launching Synapse. On Debian, this +can be done by installing the ``libjemalloc1`` package and adding this +line to ``/etc/default/matrix-synapse``:: LD_PRELOAD=/usr/lib/x86_64-linux-gnu/libjemalloc.so.1 This can make a significant difference on Python 2.7 - it's unclear how much of an improvement it provides on Python 3.x. + +If you're encountering high CPU use by the Synapse process itself, you +may be affected by a bug with presence tracking that leads to a +massive excess of outgoing federation requests (see `discussion +<https://github.com/matrix-org/synapse/issues/3971>`_). If metrics +indicate that your server is also issuing far more outgoing federation +requests than can be accounted for by your users' activity, this is a +likely cause. The misbehavior can be worked around by setting +``use_presence: false`` in the Synapse config file. diff --git a/changelog.d/4276.misc b/changelog.d/4276.misc new file mode 100644 index 0000000000..285939a4b8 --- /dev/null +++ b/changelog.d/4276.misc @@ -0,0 +1 @@ +Improve README section on performance troubleshooting. diff --git a/changelog.d/5015.misc b/changelog.d/5015.misc new file mode 100644 index 0000000000..eeec85b92c --- /dev/null +++ b/changelog.d/5015.misc @@ -0,0 +1 @@ +Add logging to 3pid invite signature verification. diff --git a/changelog.d/5493.misc b/changelog.d/5493.misc new file mode 100644 index 0000000000..365e49d634 --- /dev/null +++ b/changelog.d/5493.misc @@ -0,0 +1 @@ +Track deactivated accounts in the database. diff --git a/synapse/handlers/federation.py b/synapse/handlers/federation.py index 93e064cda3..51d7eb274b 100644 --- a/synapse/handlers/federation.py +++ b/synapse/handlers/federation.py @@ -2744,25 +2744,55 @@ class FederationHandler(BaseHandler): if not invite_event: raise AuthError(403, "Could not find invite") + logger.debug("Checking auth on event %r", event.content) + last_exception = None + # for each public key in the 3pid invite event for public_key_object in self.hs.get_auth().get_public_keys(invite_event): try: + # for each sig on the third_party_invite block of the actual invite for server, signature_block in signed["signatures"].items(): for key_name, encoded_signature in signature_block.items(): if not key_name.startswith("ed25519:"): continue - public_key = public_key_object["public_key"] - verify_key = decode_verify_key_bytes( - key_name, - decode_base64(public_key) + logger.debug( + "Attempting to verify sig with key %s from %r " + "against pubkey %r", + key_name, server, public_key_object, ) - verify_signed_json(signed, server, verify_key) - if "key_validity_url" in public_key_object: - yield self._check_key_revocation( - public_key, + + try: + public_key = public_key_object["public_key"] + verify_key = decode_verify_key_bytes( + key_name, + decode_base64(public_key) + ) + verify_signed_json(signed, server, verify_key) + logger.debug( + "Successfully verified sig with key %s from %r " + "against pubkey %r", + key_name, server, public_key_object, + ) + except Exception: + logger.info( + "Failed to verify sig with key %s from %r " + "against pubkey %r", + key_name, server, public_key_object, + ) + raise + try: + if "key_validity_url" in public_key_object: + yield self._check_key_revocation( + public_key, + public_key_object["key_validity_url"] + ) + except Exception: + logger.info( + "Failed to query key_validity_url %s", public_key_object["key_validity_url"] ) + raise return except Exception as e: last_exception = e diff --git a/synapse/storage/registration.py b/synapse/storage/registration.py index d36917e4d6..0b3c656e90 100644 --- a/synapse/storage/registration.py +++ b/synapse/storage/registration.py @@ -662,7 +662,7 @@ class RegistrationStore( for user in rows: if not user["count_tokens"] and not user["count_threepids"]: - self.set_user_deactivated_status_txn(txn, user["user_id"], True) + self.set_user_deactivated_status_txn(txn, user["name"], True) rows_processed_nb += 1 logger.info("Marked %d rows as deactivated", rows_processed_nb) |