summary refs log tree commit diff
path: root/synapse
diff options
context:
space:
mode:
authorEric Eastwood <erice@element.io>2021-06-11 04:12:35 -0500
committerGitHub <noreply@github.com>2021-06-11 10:12:35 +0100
commitb31daac01c9cc757d7c73da2f23e1b7251c54b79 (patch)
treedd79e3444666c53b7c6a58f25e93eaae2d9670f1 /synapse
parentDocument how to see logger output when running the twisted tests (#10148) (diff)
downloadsynapse-b31daac01c9cc757d7c73da2f23e1b7251c54b79.tar.xz
Add metrics to track how often events are `soft_failed` (#10156)
Spawned from missing messages we were seeing on `matrix.org` from a
federated Gtiter bridged room, https://gitlab.com/gitterHQ/webapp/-/issues/2770.
The underlying issue in Synapse is tracked by https://github.com/matrix-org/synapse/issues/10066
where the message and join event race and the message is `soft_failed` before the
`join` event reaches the remote federated server.

Less soft_failed events = better and usually this should only trigger for events
where people are doing bad things and trying to fuzz and fake everything.
Diffstat (limited to 'synapse')
-rw-r--r--synapse/handlers/federation.py7
1 files changed, 7 insertions, 0 deletions
diff --git a/synapse/handlers/federation.py b/synapse/handlers/federation.py
index 6e40e2c216..6647063485 100644
--- a/synapse/handlers/federation.py
+++ b/synapse/handlers/federation.py
@@ -33,6 +33,7 @@ from typing import (
 )
 
 import attr
+from prometheus_client import Counter
 from signedjson.key import decode_verify_key_bytes
 from signedjson.sign import verify_signed_json
 from unpaddedbase64 import decode_base64
@@ -101,6 +102,11 @@ if TYPE_CHECKING:
 
 logger = logging.getLogger(__name__)
 
+soft_failed_event_counter = Counter(
+    "synapse_federation_soft_failed_events_total",
+    "Events received over federation that we marked as soft_failed",
+)
+
 
 @attr.s(slots=True)
 class _NewEventInfo:
@@ -2498,6 +2504,7 @@ class FederationHandler(BaseHandler):
             event_auth.check(room_version_obj, event, auth_events=current_auth_events)
         except AuthError as e:
             logger.warning("Soft-failing %r because %s", event, e)
+            soft_failed_event_counter.inc()
             event.internal_metadata.soft_failed = True
 
     async def on_get_missing_events(