From 3598c11c8dcb5fb0df16276298760e0de31d0a84 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Wed, 26 Nov 2014 10:41:08 +0000 Subject: Correctly handle the case where we get an event for an unknown room, which turns out we are actually in --- synapse/storage/__init__.py | 54 ++++++++++++++++++++++++++++++++------------- synapse/storage/state.py | 2 +- 2 files changed, 40 insertions(+), 16 deletions(-) (limited to 'synapse/storage') diff --git a/synapse/storage/__init__.py b/synapse/storage/__init__.py index 1231794de0..a5ee923bc3 100644 --- a/synapse/storage/__init__.py +++ b/synapse/storage/__init__.py @@ -93,7 +93,8 @@ class DataStore(RoomMemberStore, RoomStore, @defer.inlineCallbacks @log_function - def persist_event(self, event, backfilled=False, is_new_state=True): + def persist_event(self, event, backfilled=False, is_new_state=True, + current_state=None): stream_ordering = None if backfilled: if not self.min_token_deferred.called: @@ -109,6 +110,7 @@ class DataStore(RoomMemberStore, RoomStore, backfilled=backfilled, stream_ordering=stream_ordering, is_new_state=is_new_state, + current_state=current_state, ) except _RollbackButIsFineException: pass @@ -137,7 +139,7 @@ class DataStore(RoomMemberStore, RoomStore, @log_function def _persist_event_txn(self, txn, event, backfilled, stream_ordering=None, - is_new_state=True): + is_new_state=True, current_state=None): if event.type == RoomMemberEvent.TYPE: self._store_room_member_txn(txn, event) elif event.type == FeedbackEvent.TYPE: @@ -206,8 +208,24 @@ class DataStore(RoomMemberStore, RoomStore, self._store_state_groups_txn(txn, event) + if current_state: + txn.execute("DELETE FROM current_state_events") + + for s in current_state: + self._simple_insert_txn( + txn, + "current_state_events", + { + "event_id": s.event_id, + "room_id": s.room_id, + "type": s.type, + "state_key": s.state_key, + }, + or_replace=True, + ) + is_state = hasattr(event, "state_key") and event.state_key is not None - if is_new_state and is_state: + if is_state: vals = { "event_id": event.event_id, "room_id": event.room_id, @@ -225,17 +243,18 @@ class DataStore(RoomMemberStore, RoomStore, or_replace=True, ) - self._simple_insert_txn( - txn, - "current_state_events", - { - "event_id": event.event_id, - "room_id": event.room_id, - "type": event.type, - "state_key": event.state_key, - }, - or_replace=True, - ) + if is_new_state: + self._simple_insert_txn( + txn, + "current_state_events", + { + "event_id": event.event_id, + "room_id": event.room_id, + "type": event.type, + "state_key": event.state_key, + }, + or_replace=True, + ) for e_id, h in event.prev_state: self._simple_insert_txn( @@ -312,7 +331,12 @@ class DataStore(RoomMemberStore, RoomStore, txn, event.event_id, ref_alg, ref_hash_bytes ) - self._update_min_depth_for_room_txn(txn, event.room_id, event.depth) + if not outlier: + self._update_min_depth_for_room_txn( + txn, + event.room_id, + event.depth + ) def _store_redaction(self, txn, event): txn.execute( diff --git a/synapse/storage/state.py b/synapse/storage/state.py index 55ea567793..e0f44b3e59 100644 --- a/synapse/storage/state.py +++ b/synapse/storage/state.py @@ -87,7 +87,7 @@ class StateStore(SQLBaseStore): ) def _store_state_groups_txn(self, txn, event): - if not event.state_events: + if event.state_events is None: return state_group = event.state_group -- cgit 1.4.1 From 87538711b6fe26b1c1c708ed67345a91fd64c8fb Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Wed, 26 Nov 2014 11:14:30 +0000 Subject: Update schema to support multiple signatures --- synapse/storage/schema/event_signatures.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'synapse/storage') diff --git a/synapse/storage/schema/event_signatures.sql b/synapse/storage/schema/event_signatures.sql index 4efa8a3e63..b6b56b47a2 100644 --- a/synapse/storage/schema/event_signatures.sql +++ b/synapse/storage/schema/event_signatures.sql @@ -42,7 +42,7 @@ CREATE TABLE IF NOT EXISTS event_signatures ( signature_name TEXT, key_id TEXT, signature BLOB, - CONSTRAINT uniqueness UNIQUE (event_id, key_id) + CONSTRAINT uniqueness UNIQUE (event_id, signature_name, key_id) ); CREATE INDEX IF NOT EXISTS event_signatures_id ON event_signatures ( -- cgit 1.4.1 From cb76945688e8f594392ba4e1b5ba8ee14abb1234 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Wed, 26 Nov 2014 11:17:19 +0000 Subject: Add update delta for schema change --- synapse/storage/__init__.py | 2 +- synapse/storage/schema/delta/v8.sql | 34 ++++++++++++++++++++++++++++++++++ 2 files changed, 35 insertions(+), 1 deletion(-) create mode 100644 synapse/storage/schema/delta/v8.sql (limited to 'synapse/storage') diff --git a/synapse/storage/__init__.py b/synapse/storage/__init__.py index a5ee923bc3..1fb33171e8 100644 --- a/synapse/storage/__init__.py +++ b/synapse/storage/__init__.py @@ -67,7 +67,7 @@ SCHEMAS = [ # Remember to update this number every time an incompatible change is made to # database schema files, so the users will be informed on server restarts. -SCHEMA_VERSION = 7 +SCHEMA_VERSION = 8 class _RollbackButIsFineException(Exception): diff --git a/synapse/storage/schema/delta/v8.sql b/synapse/storage/schema/delta/v8.sql new file mode 100644 index 0000000000..daf6646ed5 --- /dev/null +++ b/synapse/storage/schema/delta/v8.sql @@ -0,0 +1,34 @@ +/* Copyright 2014 OpenMarket Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + + CREATE TABLE IF NOT EXISTS event_signatures_2 ( + event_id TEXT, + signature_name TEXT, + key_id TEXT, + signature BLOB, + CONSTRAINT uniqueness UNIQUE (event_id, signature_name, key_id) +); + +INSERT INTO event_signatures_2 (event_id, signature_name, key_id, signature) +SELECT event_id, signature_name, key_id, signature FROM event_signatures; + +DROP TABLE event_signatures; +ALTER TABLE event_signatures_2 RENAME TO event_signatures; + +CREATE INDEX IF NOT EXISTS event_signatures_id ON event_signatures ( + event_id +); + +PRAGMA user_version = 8; \ No newline at end of file -- cgit 1.4.1 From 4e2ffe79a4406616990eac1a00682f08d668218a Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Wed, 26 Nov 2014 15:17:08 +0000 Subject: Don't delete the entire current_state_events table --- synapse/storage/__init__.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) (limited to 'synapse/storage') diff --git a/synapse/storage/__init__.py b/synapse/storage/__init__.py index 1fb33171e8..f15e3dfe62 100644 --- a/synapse/storage/__init__.py +++ b/synapse/storage/__init__.py @@ -209,7 +209,10 @@ class DataStore(RoomMemberStore, RoomStore, self._store_state_groups_txn(txn, event) if current_state: - txn.execute("DELETE FROM current_state_events") + txn.execute( + "DELETE FROM current_state_events WHERE room_id = ?", + (event.room_id,) + ) for s in current_state: self._simple_insert_txn( -- cgit 1.4.1 From 1505055334ecff6516c0b388efe4c5759e59fad0 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Thu, 27 Nov 2014 16:38:50 +0000 Subject: Don't return outliers when we get recent events for rooms. --- synapse/storage/stream.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'synapse/storage') diff --git a/synapse/storage/stream.py b/synapse/storage/stream.py index b84735e61c..3405cb365e 100644 --- a/synapse/storage/stream.py +++ b/synapse/storage/stream.py @@ -283,7 +283,7 @@ class StreamStore(SQLBaseStore): sql = ( "SELECT *, (%(redacted)s) AS redacted FROM events " - "WHERE room_id = ? AND stream_ordering <= ? " + "WHERE room_id = ? AND stream_ordering <= ? AND outlier = 0 " "ORDER BY topological_ordering DESC, stream_ordering DESC LIMIT ? " ) % { "redacted": del_sql, -- cgit 1.4.1 From 10eb8f070cc5e4fc5067c98ef699724716e33f74 Mon Sep 17 00:00:00 2001 From: "Paul \"LeoNerd\" Evans" Date: Tue, 2 Dec 2014 21:36:53 +0000 Subject: Workaround for non-uniqueness of room member events in the database confusing HAVING COUNT() test of room membership intersection (with thanks to Tom Molesworth) --- synapse/storage/roommember.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) (limited to 'synapse/storage') diff --git a/synapse/storage/roommember.py b/synapse/storage/roommember.py index c37df59d45..05b275663e 100644 --- a/synapse/storage/roommember.py +++ b/synapse/storage/roommember.py @@ -218,7 +218,9 @@ class RoomMemberStore(SQLBaseStore): "ON m.event_id = c.event_id " "WHERE m.membership = 'join' " "AND (%(clause)s) " - "GROUP BY m.room_id HAVING COUNT(m.room_id) = ?" + # TODO(paul): We've got duplicate rows in the database somewhere + # so we have to DISTINCT m.user_id here + "GROUP BY m.room_id HAVING COUNT(DISTINCT m.user_id) = ?" ) % {"clause": user_list_clause} args = list(user_id_list) -- cgit 1.4.1