diff --git a/synapse/storage/__init__.py b/synapse/storage/__init__.py
index a63c59a8a2..02b1f06854 100644
--- a/synapse/storage/__init__.py
+++ b/synapse/storage/__init__.py
@@ -44,7 +44,6 @@ from syutil.jsonutil import encode_canonical_json
from synapse.crypto.event_signing import compute_event_reference_hash
-import json
import logging
import os
@@ -164,6 +163,9 @@ class DataStore(RoomMemberStore, RoomStore,
stream_ordering=None, is_new_state=True,
current_state=None):
+ # Remove the any existing cache entries for the event_id
+ self._get_event_cache.pop(event.event_id)
+
# We purposefully do this first since if we include a `current_state`
# key, we *want* to update the `current_state_events` table
if current_state:
@@ -298,12 +300,16 @@ class DataStore(RoomMemberStore, RoomStore,
or_replace=True,
)
+ content = encode_canonical_json(
+ event.content
+ ).decode("UTF-8")
+
vals = {
"topological_ordering": event.depth,
"event_id": event.event_id,
"type": event.type,
"room_id": event.room_id,
- "content": json.dumps(event.get_dict()["content"]),
+ "content": content,
"processed": True,
"outlier": outlier,
"depth": event.depth,
@@ -323,7 +329,10 @@ class DataStore(RoomMemberStore, RoomStore,
"prev_events",
]
}
- vals["unrecognized_keys"] = json.dumps(unrec)
+
+ vals["unrecognized_keys"] = encode_canonical_json(
+ unrec
+ ).decode("UTF-8")
try:
self._simple_insert_txn(
@@ -420,6 +429,8 @@ class DataStore(RoomMemberStore, RoomStore,
)
def _store_redaction(self, txn, event):
+ # invalidate the cache for the redacted event
+ self._get_event_cache.pop(event.redacts)
txn.execute(
"INSERT OR IGNORE INTO redactions "
"(event_id, redacts) VALUES (?,?)",
diff --git a/synapse/storage/_base.py b/synapse/storage/_base.py
index 3e1ab0a159..be9934c66f 100644
--- a/synapse/storage/_base.py
+++ b/synapse/storage/_base.py
@@ -19,11 +19,12 @@ from synapse.events import FrozenEvent
from synapse.events.utils import prune_event
from synapse.util.logutils import log_function
from synapse.util.logcontext import PreserveLoggingContext, LoggingContext
+from synapse.util.lrucache import LruCache
from twisted.internet import defer
import collections
-import json
+import simplejson as json
import sys
import time
@@ -128,6 +129,8 @@ class SQLBaseStore(object):
self._txn_perf_counters = PerformanceCounters()
self._get_event_counters = PerformanceCounters()
+ self._get_event_cache = LruCache(hs.config.event_cache_size)
+
def start_profiling(self):
self._previous_loop_ts = self._clock.time_msec()
@@ -579,6 +582,19 @@ class SQLBaseStore(object):
def _get_event_txn(self, txn, event_id, check_redacted=True,
get_prev_content=False, allow_rejected=False):
+
+ start_time = time.time() * 1000
+ update_counter = self._get_event_counters.update
+
+ try:
+ cache = self._get_event_cache.setdefault(event_id, {})
+ # Separate cache entries for each way to invoke _get_event_txn
+ return cache[(check_redacted, get_prev_content, allow_rejected)]
+ except KeyError:
+ pass
+ finally:
+ start_time = update_counter("event_cache", start_time)
+
sql = (
"SELECT e.internal_metadata, e.json, r.event_id, rej.reason "
"FROM event_json as e "
@@ -588,8 +604,6 @@ class SQLBaseStore(object):
"LIMIT 1 "
)
- start_time = time.time() * 1000
-
txn.execute(sql, (event_id,))
res = txn.fetchone()
@@ -599,14 +613,16 @@ class SQLBaseStore(object):
internal_metadata, js, redacted, rejected_reason = res
- self._get_event_counters.update("select_event", start_time)
+ start_time = update_counter("select_event", start_time)
if allow_rejected or not rejected_reason:
- return self._get_event_from_row_txn(
+ result = self._get_event_from_row_txn(
txn, internal_metadata, js, redacted,
check_redacted=check_redacted,
get_prev_content=get_prev_content,
)
+ cache[(check_redacted, get_prev_content, allow_rejected)] = result
+ return result
else:
return None
diff --git a/synapse/storage/filtering.py b/synapse/storage/filtering.py
index e86eeced45..457a11fd02 100644
--- a/synapse/storage/filtering.py
+++ b/synapse/storage/filtering.py
@@ -17,7 +17,7 @@ from twisted.internet import defer
from ._base import SQLBaseStore
-import json
+import simplejson as json
class FilteringStore(SQLBaseStore):
diff --git a/synapse/storage/push_rule.py b/synapse/storage/push_rule.py
index 620de71398..ae46b39cc1 100644
--- a/synapse/storage/push_rule.py
+++ b/synapse/storage/push_rule.py
@@ -20,7 +20,7 @@ from twisted.internet import defer
import logging
import copy
-import json
+import simplejson as json
logger = logging.getLogger(__name__)
diff --git a/synapse/storage/roommember.py b/synapse/storage/roommember.py
index c69dd995ce..779f9ce544 100644
--- a/synapse/storage/roommember.py
+++ b/synapse/storage/roommember.py
@@ -35,6 +35,11 @@ RoomsForUser = namedtuple(
class RoomMemberStore(SQLBaseStore):
+ def __init__(self, *args, **kw):
+ super(RoomMemberStore, self).__init__(*args, **kw)
+
+ self._user_rooms_cache = {}
+
def _store_room_member_txn(self, txn, event):
"""Store a room member in the database.
"""
@@ -98,6 +103,8 @@ class RoomMemberStore(SQLBaseStore):
txn.execute(sql, (event.room_id, domain))
+ self.invalidate_rooms_for_user(target_user_id)
+
@defer.inlineCallbacks
def get_room_member(self, user_id, room_id):
"""Retrieve the current state of a room member.
@@ -240,28 +247,53 @@ class RoomMemberStore(SQLBaseStore):
results = self._parse_events_txn(txn, rows)
return results
+ # TODO(paul): Create a nice @cached decorator to do this
+ # @cached
+ # def get_foo(...)
+ # ...
+ # invalidate_foo = get_foo.invalidator
+
+ @defer.inlineCallbacks
+ def get_rooms_for_user(self, user_id):
+ # TODO(paul): put some performance counters in here so we can easily
+ # track what impact this cache is having
+ if user_id in self._user_rooms_cache:
+ defer.returnValue(self._user_rooms_cache[user_id])
+
+ rooms = yield self.get_rooms_for_user_where_membership_is(
+ user_id, membership_list=[Membership.JOIN],
+ )
+
+ # TODO(paul): Consider applying a maximum size; just evict things at
+ # random, or consider LRU?
+
+ self._user_rooms_cache[user_id] = rooms
+ defer.returnValue(rooms)
+
+ def invalidate_rooms_for_user(self, user_id):
+ if user_id in self._user_rooms_cache:
+ del self._user_rooms_cache[user_id]
+
+ @defer.inlineCallbacks
def user_rooms_intersect(self, user_id_list):
""" Checks whether all the users whose IDs are given in a list share a
room.
+
+ This is a "hot path" function that's called a lot, e.g. by presence for
+ generating the event stream. As such, it is implemented locally by
+ wrapping logic around heavily-cached database queries.
"""
- def interaction(txn):
- user_list_clause = " OR ".join(["m.user_id = ?"] * len(user_id_list))
- sql = (
- "SELECT m.room_id FROM room_memberships as m "
- "INNER JOIN current_state_events as c "
- "ON m.event_id = c.event_id "
- "WHERE m.membership = 'join' "
- "AND (%(clause)s) "
- # TODO(paul): We've got duplicate rows in the database somewhere
- # so we have to DISTINCT m.user_id here
- "GROUP BY m.room_id HAVING COUNT(DISTINCT m.user_id) = ?"
- ) % {"clause": user_list_clause}
-
- args = list(user_id_list)
- args.append(len(user_id_list))
+ if len(user_id_list) < 2:
+ defer.returnValue(True)
- txn.execute(sql, args)
+ deferreds = [self.get_rooms_for_user(u) for u in user_id_list]
+
+ results = yield defer.DeferredList(deferreds)
+
+ # A list of sets of strings giving room IDs for each user
+ room_id_lists = [set([r.room_id for r in result[1]]) for result in results]
- return len(txn.fetchall()) > 0
+ # There isn't a setintersection(*list_of_sets)
+ ret = len(room_id_lists.pop(0).intersection(*room_id_lists)) > 0
- return self.runInteraction("user_rooms_intersect", interaction)
+ defer.returnValue(ret)
|
