From 33548f37aa7858c4d9ce01bf3ec931cc3f08833a Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Thu, 1 Apr 2021 17:08:21 +0100 Subject: Improve tracing for to device messages (#9686) --- synapse/notifier.py | 45 +++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 43 insertions(+), 2 deletions(-) (limited to 'synapse/notifier.py') diff --git a/synapse/notifier.py b/synapse/notifier.py index 1374aae490..d35c1f3f02 100644 --- a/synapse/notifier.py +++ b/synapse/notifier.py @@ -39,6 +39,7 @@ from synapse.api.errors import AuthError from synapse.events import EventBase from synapse.handlers.presence import format_user_presence_state from synapse.logging.context import PreserveLoggingContext +from synapse.logging.opentracing import log_kv, start_active_span from synapse.logging.utils import log_function from synapse.metrics import LaterGauge from synapse.streams.config import PaginationConfig @@ -136,6 +137,15 @@ class _NotifierUserStream: self.last_notified_ms = time_now_ms noify_deferred = self.notify_deferred + log_kv( + { + "notify": self.user_id, + "stream": stream_key, + "stream_id": stream_id, + "listeners": self.count_listeners(), + } + ) + users_woken_by_stream_counter.labels(stream_key).inc() with PreserveLoggingContext(): @@ -404,6 +414,13 @@ class Notifier: with Measure(self.clock, "on_new_event"): user_streams = set() + log_kv( + { + "waking_up_explicit_users": len(users), + "waking_up_explicit_rooms": len(rooms), + } + ) + for user in users: user_stream = self.user_to_user_stream.get(str(user)) if user_stream is not None: @@ -476,12 +493,34 @@ class Notifier: (end_time - now) / 1000.0, self.hs.get_reactor(), ) - with PreserveLoggingContext(): - await listener.deferred + + with start_active_span("wait_for_events.deferred"): + log_kv( + { + "wait_for_events": "sleep", + "token": prev_token, + } + ) + + with PreserveLoggingContext(): + await listener.deferred + + log_kv( + { + "wait_for_events": "woken", + "token": user_stream.current_token, + } + ) current_token = user_stream.current_token result = await callback(prev_token, current_token) + log_kv( + { + "wait_for_events": "result", + "result": bool(result), + } + ) if result: break @@ -489,8 +528,10 @@ class Notifier: # has happened between the old prev_token and the current_token prev_token = current_token except defer.TimeoutError: + log_kv({"wait_for_events": "timeout"}) break except defer.CancelledError: + log_kv({"wait_for_events": "cancelled"}) break if result is None: -- cgit 1.5.1 From e2b8a90897e137fd118768a3bf35b70642916eb7 Mon Sep 17 00:00:00 2001 From: Jonathan de Jong Date: Mon, 5 Apr 2021 15:10:18 +0200 Subject: Update mypy configuration: `no_implicit_optional = True` (#9742) --- changelog.d/9742.misc | 1 + mypy.ini | 1 + synapse/handlers/account_validity.py | 7 +++++-- synapse/handlers/e2e_keys.py | 2 +- synapse/http/client.py | 2 +- synapse/notifier.py | 2 +- synapse/replication/tcp/redis.py | 2 +- synapse/storage/databases/main/group_server.py | 4 ++-- synapse/util/caches/deferred_cache.py | 4 +++- tests/rest/client/v2_alpha/test_auth.py | 7 +++++-- 10 files changed, 21 insertions(+), 11 deletions(-) create mode 100644 changelog.d/9742.misc (limited to 'synapse/notifier.py') diff --git a/changelog.d/9742.misc b/changelog.d/9742.misc new file mode 100644 index 0000000000..681ab04df8 --- /dev/null +++ b/changelog.d/9742.misc @@ -0,0 +1 @@ +Start linting mypy with `no_implicit_optional`. \ No newline at end of file diff --git a/mypy.ini b/mypy.ini index 3ae5d45787..32e6197409 100644 --- a/mypy.ini +++ b/mypy.ini @@ -8,6 +8,7 @@ show_traceback = True mypy_path = stubs warn_unreachable = True local_partial_types = True +no_implicit_optional = True # To find all folders that pass mypy you run: # diff --git a/synapse/handlers/account_validity.py b/synapse/handlers/account_validity.py index d781bb251d..bee1447c2e 100644 --- a/synapse/handlers/account_validity.py +++ b/synapse/handlers/account_validity.py @@ -18,7 +18,7 @@ import email.utils import logging from email.mime.multipart import MIMEMultipart from email.mime.text import MIMEText -from typing import TYPE_CHECKING, List +from typing import TYPE_CHECKING, List, Optional from synapse.api.errors import StoreError, SynapseError from synapse.logging.context import make_deferred_yieldable @@ -241,7 +241,10 @@ class AccountValidityHandler: return True async def renew_account_for_user( - self, user_id: str, expiration_ts: int = None, email_sent: bool = False + self, + user_id: str, + expiration_ts: Optional[int] = None, + email_sent: bool = False, ) -> int: """Renews the account attached to a given user by pushing back the expiration date by the current validity period in the server's diff --git a/synapse/handlers/e2e_keys.py b/synapse/handlers/e2e_keys.py index 2ad9b6d930..739653a3fa 100644 --- a/synapse/handlers/e2e_keys.py +++ b/synapse/handlers/e2e_keys.py @@ -1008,7 +1008,7 @@ class E2eKeysHandler: return signature_list, failures async def _get_e2e_cross_signing_verify_key( - self, user_id: str, key_type: str, from_user_id: str = None + self, user_id: str, key_type: str, from_user_id: Optional[str] = None ) -> Tuple[JsonDict, str, VerifyKey]: """Fetch locally or remotely query for a cross-signing public key. diff --git a/synapse/http/client.py b/synapse/http/client.py index a0caba84e4..e691ba6d88 100644 --- a/synapse/http/client.py +++ b/synapse/http/client.py @@ -590,7 +590,7 @@ class SimpleHttpClient: uri: str, json_body: Any, args: Optional[QueryParams] = None, - headers: RawHeaders = None, + headers: Optional[RawHeaders] = None, ) -> Any: """Puts some json to the given URI. diff --git a/synapse/notifier.py b/synapse/notifier.py index d35c1f3f02..c178db57e3 100644 --- a/synapse/notifier.py +++ b/synapse/notifier.py @@ -548,7 +548,7 @@ class Notifier: pagination_config: PaginationConfig, timeout: int, is_guest: bool = False, - explicit_room_id: str = None, + explicit_room_id: Optional[str] = None, ) -> EventStreamResult: """For the given user and rooms, return any new events for them. If there are no new events wait for up to `timeout` milliseconds for any diff --git a/synapse/replication/tcp/redis.py b/synapse/replication/tcp/redis.py index 2f4d407f94..98bdeb0ec6 100644 --- a/synapse/replication/tcp/redis.py +++ b/synapse/replication/tcp/redis.py @@ -60,7 +60,7 @@ class ConstantProperty(Generic[T, V]): constant = attr.ib() # type: V - def __get__(self, obj: Optional[T], objtype: Type[T] = None) -> V: + def __get__(self, obj: Optional[T], objtype: Optional[Type[T]] = None) -> V: return self.constant def __set__(self, obj: Optional[T], value: V): diff --git a/synapse/storage/databases/main/group_server.py b/synapse/storage/databases/main/group_server.py index ac07e0197b..8f462dfc31 100644 --- a/synapse/storage/databases/main/group_server.py +++ b/synapse/storage/databases/main/group_server.py @@ -1027,8 +1027,8 @@ class GroupServerStore(GroupServerWorkerStore): user_id: str, is_admin: bool = False, is_public: bool = True, - local_attestation: dict = None, - remote_attestation: dict = None, + local_attestation: Optional[dict] = None, + remote_attestation: Optional[dict] = None, ) -> None: """Add a user to the group server. diff --git a/synapse/util/caches/deferred_cache.py b/synapse/util/caches/deferred_cache.py index 1adc92eb90..dd392cf694 100644 --- a/synapse/util/caches/deferred_cache.py +++ b/synapse/util/caches/deferred_cache.py @@ -283,7 +283,9 @@ class DeferredCache(Generic[KT, VT]): # we return a new Deferred which will be called before any subsequent observers. return observable.observe() - def prefill(self, key: KT, value: VT, callback: Callable[[], None] = None): + def prefill( + self, key: KT, value: VT, callback: Optional[Callable[[], None]] = None + ): callbacks = [callback] if callback else [] self.cache.set(key, value, callbacks=callbacks) diff --git a/tests/rest/client/v2_alpha/test_auth.py b/tests/rest/client/v2_alpha/test_auth.py index 9734a2159a..ed433d9333 100644 --- a/tests/rest/client/v2_alpha/test_auth.py +++ b/tests/rest/client/v2_alpha/test_auth.py @@ -13,7 +13,7 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -from typing import Union +from typing import Optional, Union from twisted.internet.defer import succeed @@ -74,7 +74,10 @@ class FallbackAuthTests(unittest.HomeserverTestCase): return channel def recaptcha( - self, session: str, expected_post_response: int, post_session: str = None + self, + session: str, + expected_post_response: int, + post_session: Optional[str] = None, ) -> None: """Get and respond to a fallback recaptcha. Returns the second request.""" if post_session is None: -- cgit 1.5.1