From 8d6dc106d188308f687db1f696cad7ad1b9ae83d Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Mon, 26 Mar 2018 12:02:44 +0100 Subject: Don't use _cursor_to_dict in find_next_generated_user_id_localpart --- synapse/storage/registration.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) (limited to 'synapse/storage') diff --git a/synapse/storage/registration.py b/synapse/storage/registration.py index d809b2ba46..6b557ca0cf 100644 --- a/synapse/storage/registration.py +++ b/synapse/storage/registration.py @@ -460,14 +460,12 @@ class RegistrationStore(RegistrationWorkerStore, """ def _find_next_generated_user_id(txn): txn.execute("SELECT name FROM users") - rows = self.cursor_to_dict(txn) regex = re.compile("^@(\d+):") found = set() - for r in rows: - user_id = r["name"] + for user_id, in txn: match = regex.search(user_id) if match: found.add(int(match.group(1))) -- cgit 1.5.1 From fecb45e0c38912f72655d53faa953e7713511e7f Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Mon, 26 Mar 2018 13:31:49 +0100 Subject: Remove last usage of ujson --- synapse/python_dependencies.py | 1 - synapse/storage/group_server.py | 2 +- 2 files changed, 1 insertion(+), 2 deletions(-) (limited to 'synapse/storage') diff --git a/synapse/python_dependencies.py b/synapse/python_dependencies.py index 91179ce532..40eedb63cb 100644 --- a/synapse/python_dependencies.py +++ b/synapse/python_dependencies.py @@ -34,7 +34,6 @@ REQUIREMENTS = { "bcrypt": ["bcrypt>=3.1.0"], "pillow": ["PIL"], "pydenticon": ["pydenticon"], - "ujson": ["ujson"], "blist": ["blist"], "pysaml2>=3.0.0": ["saml2>=3.0.0"], "pymacaroons-pynacl": ["pymacaroons"], diff --git a/synapse/storage/group_server.py b/synapse/storage/group_server.py index 8fde1aab8e..d03858234b 100644 --- a/synapse/storage/group_server.py +++ b/synapse/storage/group_server.py @@ -19,7 +19,7 @@ from synapse.api.errors import SynapseError from ._base import SQLBaseStore -import ujson as json +import simplejson as json # The category ID for the "default" category. We don't store as null in the -- cgit 1.5.1 From 3f49e131d9a31d5bf198f2aa6b102562b2e747d0 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 27 Mar 2018 10:30:03 +0100 Subject: Add counter metrics for calculating state delta This will allow us to measure how often we calculate state deltas in event persistence that we would have been able to calculate at the same time we calculated the state for the event. --- synapse/storage/events.py | 31 ++++++++++++++++++++++++++++++- 1 file changed, 30 insertions(+), 1 deletion(-) (limited to 'synapse/storage') diff --git a/synapse/storage/events.py b/synapse/storage/events.py index 85ce6bea1a..f3a362db20 100644 --- a/synapse/storage/events.py +++ b/synapse/storage/events.py @@ -52,6 +52,15 @@ persist_event_counter = metrics.register_counter("persisted_events") event_counter = metrics.register_counter( "persisted_events_sep", labels=["type", "origin_type", "origin_entity"] ) +state_delta_counter = metrics.register_counter( + "state_delta", +) +state_delta_single_event_counter = metrics.register_counter( + "state_delta_single_event", +) +state_delta_reuse_delta_counter = metrics.register_counter( + "state_delta_reuse_delta", +) def encode_json(json_object): @@ -368,7 +377,8 @@ class EventsStore(EventsWorkerStore): room_id, ev_ctx_rm, latest_event_ids ) - if new_latest_event_ids == set(latest_event_ids): + latest_event_ids = set(latest_event_ids) + if new_latest_event_ids == latest_event_ids: # No change in extremities, so no change in state continue @@ -389,6 +399,25 @@ class EventsStore(EventsWorkerStore): if all_single_prev_not_state: continue + state_delta_counter.inc() + if len(new_latest_event_ids) == 1: + state_delta_single_event_counter.inc() + + # This is a fairly handwavey check to see if we could + # have guessed what the delta would have been when + # processing one of these events. + # What we're interested in is if the latest extremities + # were the same when we created the event as they are + # now. We guess this by looking at the prev events and + # checking if they match up, as when this server creates + # a new event it will use the extremities as the prev + # events. + for ev, _ in ev_ctx_rm: + prev_event_ids = set(e for e, _ in ev.prev_events) + if latest_event_ids == prev_event_ids: + state_delta_reuse_delta_counter.inc() + break + logger.info( "Calculating state delta for room %s", room_id, ) -- cgit 1.5.1 From e70287cff3adeeeff2a7a1e95fc5845feb365710 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 27 Mar 2018 13:13:38 +0100 Subject: Comment --- synapse/storage/events.py | 7 +++++++ 1 file changed, 7 insertions(+) (limited to 'synapse/storage') diff --git a/synapse/storage/events.py b/synapse/storage/events.py index f3a362db20..ee8b3c46d3 100644 --- a/synapse/storage/events.py +++ b/synapse/storage/events.py @@ -52,12 +52,19 @@ persist_event_counter = metrics.register_counter("persisted_events") event_counter = metrics.register_counter( "persisted_events_sep", labels=["type", "origin_type", "origin_entity"] ) + +# The number of times we are recalculating the current state state_delta_counter = metrics.register_counter( "state_delta", ) +# The number of times we are recalculating state when there is only a +# single forward extremity state_delta_single_event_counter = metrics.register_counter( "state_delta_single_event", ) +# The number of times we are reculating state when we could have resonably +# calculated the delta when we calculated the state for an event we were +# persisting. state_delta_reuse_delta_counter = metrics.register_counter( "state_delta_reuse_delta", ) -- cgit 1.5.1 From 152c2ac19e6af373a9e79a8f4da80d9e77a5396f Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 27 Mar 2018 13:13:46 +0100 Subject: Fix indent --- synapse/storage/events.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'synapse/storage') diff --git a/synapse/storage/events.py b/synapse/storage/events.py index ee8b3c46d3..95b36d669b 100644 --- a/synapse/storage/events.py +++ b/synapse/storage/events.py @@ -423,7 +423,7 @@ class EventsStore(EventsWorkerStore): prev_event_ids = set(e for e, _ in ev.prev_events) if latest_event_ids == prev_event_ids: state_delta_reuse_delta_counter.inc() - break + break logger.info( "Calculating state delta for room %s", room_id, -- cgit 1.5.1 From 800cfd5774d9726a00c53f2d84ce0286ab64c17b Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 27 Mar 2018 13:30:39 +0100 Subject: Comment --- synapse/storage/events.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) (limited to 'synapse/storage') diff --git a/synapse/storage/events.py b/synapse/storage/events.py index 95b36d669b..f3d65f4338 100644 --- a/synapse/storage/events.py +++ b/synapse/storage/events.py @@ -415,10 +415,11 @@ class EventsStore(EventsWorkerStore): # processing one of these events. # What we're interested in is if the latest extremities # were the same when we created the event as they are - # now. We guess this by looking at the prev events and - # checking if they match up, as when this server creates - # a new event it will use the extremities as the prev - # events. + # now. When this server creates a new event (as opposed + # to receiving it over federation) it will use the + # forward extremities as the prev_events, so we can + # guess this by looking at the prev_events and checking + # if they match the current forward extremities. for ev, _ in ev_ctx_rm: prev_event_ids = set(e for e, _ in ev.prev_events) if latest_event_ids == prev_event_ids: -- cgit 1.5.1