From c5b49eb7ca7327cba0e3658b1ec84cca823c8b54 Mon Sep 17 00:00:00 2001 From: David Baker Date: Thu, 8 Sep 2016 09:40:10 +0100 Subject: Fix /notifications API when used with `from` param --- synapse/storage/event_push_actions.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'synapse/storage/event_push_actions.py') diff --git a/synapse/storage/event_push_actions.py b/synapse/storage/event_push_actions.py index eb15fb751b..56dce4b616 100644 --- a/synapse/storage/event_push_actions.py +++ b/synapse/storage/event_push_actions.py @@ -342,7 +342,7 @@ class EventPushActionsStore(SQLBaseStore): def f(txn): before_clause = "" if before: - before_clause = "AND stream_ordering < ?" + before_clause = "AND epa.stream_ordering < ?" args = [user_id, before, limit] else: args = [user_id, limit] -- cgit 1.5.1 From 4ef222ab6142942eb76d4e0cdea57c1114b0c28a Mon Sep 17 00:00:00 2001 From: David Baker Date: Thu, 8 Sep 2016 13:43:35 +0100 Subject: Implement `only=highlight` on `/notifications` --- synapse/rest/client/v2_alpha/notifications.py | 3 ++- synapse/storage/event_push_actions.py | 9 ++++++++- 2 files changed, 10 insertions(+), 2 deletions(-) (limited to 'synapse/storage/event_push_actions.py') diff --git a/synapse/rest/client/v2_alpha/notifications.py b/synapse/rest/client/v2_alpha/notifications.py index f1a48acf07..fd2a3d69d4 100644 --- a/synapse/rest/client/v2_alpha/notifications.py +++ b/synapse/rest/client/v2_alpha/notifications.py @@ -45,11 +45,12 @@ class NotificationsServlet(RestServlet): from_token = parse_string(request, "from", required=False) limit = parse_integer(request, "limit", default=50) + only = parse_string(request, "only", required=False) limit = min(limit, 500) push_actions = yield self.store.get_push_actions_for_user( - user_id, from_token, limit + user_id, from_token, limit, only_highlight=(only == "highlight") ) receipts_by_room = yield self.store.get_receipts_for_user_with_orderings( diff --git a/synapse/storage/event_push_actions.py b/synapse/storage/event_push_actions.py index eb15fb751b..dedf517cfa 100644 --- a/synapse/storage/event_push_actions.py +++ b/synapse/storage/event_push_actions.py @@ -338,7 +338,8 @@ class EventPushActionsStore(SQLBaseStore): defer.returnValue(notifs[:limit]) @defer.inlineCallbacks - def get_push_actions_for_user(self, user_id, before=None, limit=50): + def get_push_actions_for_user(self, user_id, before=None, limit=50, + only_highlight=False): def f(txn): before_clause = "" if before: @@ -346,6 +347,12 @@ class EventPushActionsStore(SQLBaseStore): args = [user_id, before, limit] else: args = [user_id, limit] + + if only_highlight: + if len(before_clause) > 0: + before_clause += " " + before_clause += "AND epa.highlight = 1" + sql = ( "SELECT epa.event_id, epa.room_id," " epa.stream_ordering, epa.topological_ordering," -- cgit 1.5.1 From f2acc3dcf9cf213948ce3d2ebf12c3202abe97fd Mon Sep 17 00:00:00 2001 From: David Baker Date: Fri, 9 Sep 2016 18:54:54 +0100 Subject: Add index to event_push_actions and remove room_id caluse so it uses it Mostly from @negativemjark --- synapse/storage/event_push_actions.py | 4 +++- .../schema/delta/35/event_push_actions_index.sql | 18 ++++++++++++++++++ 2 files changed, 21 insertions(+), 1 deletion(-) create mode 100644 synapse/storage/schema/delta/35/event_push_actions_index.sql (limited to 'synapse/storage/event_push_actions.py') diff --git a/synapse/storage/event_push_actions.py b/synapse/storage/event_push_actions.py index dedf517cfa..a67c886f9a 100644 --- a/synapse/storage/event_push_actions.py +++ b/synapse/storage/event_push_actions.py @@ -353,12 +353,14 @@ class EventPushActionsStore(SQLBaseStore): before_clause += " " before_clause += "AND epa.highlight = 1" + # NB. This assumes event_ids are globally unique since + # it makes the query easier to index sql = ( "SELECT epa.event_id, epa.room_id," " epa.stream_ordering, epa.topological_ordering," " epa.actions, epa.profile_tag, e.received_ts" " FROM event_push_actions epa, events e" - " WHERE epa.room_id = e.room_id AND epa.event_id = e.event_id" + " WHERE epa.event_id = e.event_id" " AND epa.user_id = ? %s" " ORDER BY epa.stream_ordering DESC" " LIMIT ?" diff --git a/synapse/storage/schema/delta/35/event_push_actions_index.sql b/synapse/storage/schema/delta/35/event_push_actions_index.sql new file mode 100644 index 0000000000..4fc32c351a --- /dev/null +++ b/synapse/storage/schema/delta/35/event_push_actions_index.sql @@ -0,0 +1,18 @@ +/* Copyright 2016 OpenMarket Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + + CREATE INDEX event_push_actions_user_id_highlight_stream_ordering on event_push_actions( + user_id, highlight, stream_ordering + ); -- cgit 1.5.1 From 54417999b692a8dd0f8f4edd62598c80835a4212 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Mon, 12 Sep 2016 10:39:55 +0100 Subject: Revert "Add index to event_push_actions" --- synapse/storage/event_push_actions.py | 4 +--- .../schema/delta/35/event_push_actions_index.sql | 18 ------------------ 2 files changed, 1 insertion(+), 21 deletions(-) delete mode 100644 synapse/storage/schema/delta/35/event_push_actions_index.sql (limited to 'synapse/storage/event_push_actions.py') diff --git a/synapse/storage/event_push_actions.py b/synapse/storage/event_push_actions.py index a87d90741a..10e9305f7b 100644 --- a/synapse/storage/event_push_actions.py +++ b/synapse/storage/event_push_actions.py @@ -353,14 +353,12 @@ class EventPushActionsStore(SQLBaseStore): before_clause += " " before_clause += "AND epa.highlight = 1" - # NB. This assumes event_ids are globally unique since - # it makes the query easier to index sql = ( "SELECT epa.event_id, epa.room_id," " epa.stream_ordering, epa.topological_ordering," " epa.actions, epa.profile_tag, e.received_ts" " FROM event_push_actions epa, events e" - " WHERE epa.event_id = e.event_id" + " WHERE epa.room_id = e.room_id AND epa.event_id = e.event_id" " AND epa.user_id = ? %s" " ORDER BY epa.stream_ordering DESC" " LIMIT ?" diff --git a/synapse/storage/schema/delta/35/event_push_actions_index.sql b/synapse/storage/schema/delta/35/event_push_actions_index.sql deleted file mode 100644 index 4fc32c351a..0000000000 --- a/synapse/storage/schema/delta/35/event_push_actions_index.sql +++ /dev/null @@ -1,18 +0,0 @@ -/* Copyright 2016 OpenMarket Ltd - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - - CREATE INDEX event_push_actions_user_id_highlight_stream_ordering on event_push_actions( - user_id, highlight, stream_ordering - ); -- cgit 1.5.1 From 15ca0c6a4d4b7ddbff8e3348ba949177e1562108 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Mon, 12 Sep 2016 12:36:36 +0100 Subject: Make reindex happen in bg --- synapse/storage/event_push_actions.py | 30 ++++++++++++++++++++++ .../schema/delta/35/event_push_actions_index.sql | 5 ++-- 2 files changed, 32 insertions(+), 3 deletions(-) (limited to 'synapse/storage/event_push_actions.py') diff --git a/synapse/storage/event_push_actions.py b/synapse/storage/event_push_actions.py index a87d90741a..40bfe754b5 100644 --- a/synapse/storage/event_push_actions.py +++ b/synapse/storage/event_push_actions.py @@ -17,6 +17,7 @@ from ._base import SQLBaseStore from twisted.internet import defer from synapse.util.caches.descriptors import cachedInlineCallbacks from synapse.types import RoomStreamToken +from synapse.storage.engines import PostgresEngine from .stream import lower_bound import logging @@ -26,10 +27,17 @@ logger = logging.getLogger(__name__) class EventPushActionsStore(SQLBaseStore): + EPA_HIGHLIGHT_INDEX = "epa_highlight_index" + def __init__(self, hs): self.stream_ordering_month_ago = None super(EventPushActionsStore, self).__init__(hs) + self.register_background_update_handler( + self.EPA_HIGHLIGHT_INDEX, + self._background_index_epa_highlight, + ) + def _set_push_actions_for_event_and_users_txn(self, txn, event, tuples): """ Args: @@ -500,6 +508,28 @@ class EventPushActionsStore(SQLBaseStore): return range_end + @defer.inlineCallbacks + def _background_index_epa_highlight(self, progress, batch_size): + def reindex_txn(txn): + if isinstance(self.database_engine, PostgresEngine): + txn.execute( + "CREATE INDEX CONCURRENTLY event_push_actions_u_highlight" + " on event_push_actions(user_id, highlight, stream_ordering)" + ) + else: + txn.execute( + "CREATE INDEX event_push_actions_u_highlight" + " on event_push_actions(user_id, highlight, stream_ordering)" + ) + + yield self.runInteraction( + self.EPA_HIGHLIGHT_INDEX, reindex_txn + ) + + yield self._end_background_update(self.EPA_HIGHLIGHT_INDEX) + + defer.returnValue(1) + def _action_has_highlight(actions): for action in actions: diff --git a/synapse/storage/schema/delta/35/event_push_actions_index.sql b/synapse/storage/schema/delta/35/event_push_actions_index.sql index 4fc32c351a..2e836d8e9c 100644 --- a/synapse/storage/schema/delta/35/event_push_actions_index.sql +++ b/synapse/storage/schema/delta/35/event_push_actions_index.sql @@ -13,6 +13,5 @@ * limitations under the License. */ - CREATE INDEX event_push_actions_user_id_highlight_stream_ordering on event_push_actions( - user_id, highlight, stream_ordering - ); + INSERT into background_updates (update_name, progress_json) + VALUES ('epa_highlight_index', '{}'); -- cgit 1.5.1 From 0294c14ec43bed0b116c7ff531482539fb713443 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Mon, 12 Sep 2016 12:43:56 +0100 Subject: Add back in query change --- synapse/storage/event_push_actions.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) (limited to 'synapse/storage/event_push_actions.py') diff --git a/synapse/storage/event_push_actions.py b/synapse/storage/event_push_actions.py index 8632b2f936..40bfe754b5 100644 --- a/synapse/storage/event_push_actions.py +++ b/synapse/storage/event_push_actions.py @@ -361,12 +361,14 @@ class EventPushActionsStore(SQLBaseStore): before_clause += " " before_clause += "AND epa.highlight = 1" + # NB. This assumes event_ids are globally unique since + # it makes the query easier to index sql = ( "SELECT epa.event_id, epa.room_id," " epa.stream_ordering, epa.topological_ordering," " epa.actions, epa.profile_tag, e.received_ts" " FROM event_push_actions epa, events e" - " WHERE epa.room_id = e.room_id AND epa.event_id = e.event_id" + " WHERE epa.event_id = e.event_id" " AND epa.user_id = ? %s" " ORDER BY epa.stream_ordering DESC" " LIMIT ?" -- cgit 1.5.1 From 7cd6edb9470ed949b2c63317624dbc9e38950c95 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Mon, 12 Sep 2016 12:54:38 +0100 Subject: Use register_background_index_update --- synapse/storage/event_push_actions.py | 28 ++++------------------------ 1 file changed, 4 insertions(+), 24 deletions(-) (limited to 'synapse/storage/event_push_actions.py') diff --git a/synapse/storage/event_push_actions.py b/synapse/storage/event_push_actions.py index 40bfe754b5..7974a108ad 100644 --- a/synapse/storage/event_push_actions.py +++ b/synapse/storage/event_push_actions.py @@ -33,9 +33,11 @@ class EventPushActionsStore(SQLBaseStore): self.stream_ordering_month_ago = None super(EventPushActionsStore, self).__init__(hs) - self.register_background_update_handler( + self.register_background_index_update( self.EPA_HIGHLIGHT_INDEX, - self._background_index_epa_highlight, + index_name="event_push_actions_u_highlight", + table="event_push_actions", + columns=["user_id", "highlight", "stream_ordering"], ) def _set_push_actions_for_event_and_users_txn(self, txn, event, tuples): @@ -508,28 +510,6 @@ class EventPushActionsStore(SQLBaseStore): return range_end - @defer.inlineCallbacks - def _background_index_epa_highlight(self, progress, batch_size): - def reindex_txn(txn): - if isinstance(self.database_engine, PostgresEngine): - txn.execute( - "CREATE INDEX CONCURRENTLY event_push_actions_u_highlight" - " on event_push_actions(user_id, highlight, stream_ordering)" - ) - else: - txn.execute( - "CREATE INDEX event_push_actions_u_highlight" - " on event_push_actions(user_id, highlight, stream_ordering)" - ) - - yield self.runInteraction( - self.EPA_HIGHLIGHT_INDEX, reindex_txn - ) - - yield self._end_background_update(self.EPA_HIGHLIGHT_INDEX) - - defer.returnValue(1) - def _action_has_highlight(actions): for action in actions: -- cgit 1.5.1 From 5ef5435529778d6bf0bf19f36b7fb6febc26a718 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Mon, 12 Sep 2016 13:32:58 +0100 Subject: Remove unused import --- synapse/storage/event_push_actions.py | 1 - 1 file changed, 1 deletion(-) (limited to 'synapse/storage/event_push_actions.py') diff --git a/synapse/storage/event_push_actions.py b/synapse/storage/event_push_actions.py index 7974a108ad..e02afdb1df 100644 --- a/synapse/storage/event_push_actions.py +++ b/synapse/storage/event_push_actions.py @@ -17,7 +17,6 @@ from ._base import SQLBaseStore from twisted.internet import defer from synapse.util.caches.descriptors import cachedInlineCallbacks from synapse.types import RoomStreamToken -from synapse.storage.engines import PostgresEngine from .stream import lower_bound import logging -- cgit 1.5.1 From fa20c9ce9410d71d7144932c33156b9dacd554f5 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Mon, 12 Sep 2016 14:04:08 +0100 Subject: Change the index to be stream_ordering, highlight --- synapse/storage/event_push_actions.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'synapse/storage/event_push_actions.py') diff --git a/synapse/storage/event_push_actions.py b/synapse/storage/event_push_actions.py index e02afdb1df..51b13e9498 100644 --- a/synapse/storage/event_push_actions.py +++ b/synapse/storage/event_push_actions.py @@ -36,7 +36,7 @@ class EventPushActionsStore(SQLBaseStore): self.EPA_HIGHLIGHT_INDEX, index_name="event_push_actions_u_highlight", table="event_push_actions", - columns=["user_id", "highlight", "stream_ordering"], + columns=["user_id", "stream_ordering", "highlight"], ) def _set_push_actions_for_event_and_users_txn(self, txn, event, tuples): -- cgit 1.5.1 From c94de0ab606e5bbe7331a58456204a5775df779b Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Mon, 12 Sep 2016 16:55:01 +0100 Subject: Add WHERE clause support to index creation --- synapse/storage/background_updates.py | 24 +++++++++++++++--------- synapse/storage/event_push_actions.py | 3 ++- 2 files changed, 17 insertions(+), 10 deletions(-) (limited to 'synapse/storage/event_push_actions.py') diff --git a/synapse/storage/background_updates.py b/synapse/storage/background_updates.py index 003f5ba203..94b2bcc54a 100644 --- a/synapse/storage/background_updates.py +++ b/synapse/storage/background_updates.py @@ -219,7 +219,7 @@ class BackgroundUpdateStore(SQLBaseStore): self._background_update_handlers[update_name] = update_handler def register_background_index_update(self, update_name, index_name, - table, columns): + table, columns, where_clause=None): """Helper for store classes to do a background index addition To use: @@ -243,14 +243,20 @@ class BackgroundUpdateStore(SQLBaseStore): conc = True else: conc = False - - sql = "CREATE INDEX %(conc)s %(name)s ON %(table)s (%(columns)s)" \ - % { - "conc": "CONCURRENTLY" if conc else "", - "name": index_name, - "table": table, - "columns": ", ".join(columns), - } + # We don't use partial indices on SQLite as it wasn't introduced + # until 3.8, and wheezy has 3.7 + where_clause = None + + sql = ( + "CREATE INDEX %(conc)s %(name)s ON %(table)s (%(columns)s)" + " %(where_clause)s" + ) % { + "conc": "CONCURRENTLY" if conc else "", + "name": index_name, + "table": table, + "columns": ", ".join(columns), + "where_clause": "WHERE " + where_clause if where_clause else "" + } def create_index_concurrently(conn): conn.rollback() diff --git a/synapse/storage/event_push_actions.py b/synapse/storage/event_push_actions.py index 51b13e9498..efa1db3b94 100644 --- a/synapse/storage/event_push_actions.py +++ b/synapse/storage/event_push_actions.py @@ -36,7 +36,8 @@ class EventPushActionsStore(SQLBaseStore): self.EPA_HIGHLIGHT_INDEX, index_name="event_push_actions_u_highlight", table="event_push_actions", - columns=["user_id", "stream_ordering", "highlight"], + columns=["user_id", "stream_ordering"], + where_clause="highlight = 1", ) def _set_push_actions_for_event_and_users_txn(self, txn, event, tuples): -- cgit 1.5.1 From b17af156c71b9859edbd27516eacd2c55f488e31 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Mon, 12 Sep 2016 17:05:54 +0100 Subject: Remove where clause --- synapse/storage/event_push_actions.py | 1 - 1 file changed, 1 deletion(-) (limited to 'synapse/storage/event_push_actions.py') diff --git a/synapse/storage/event_push_actions.py b/synapse/storage/event_push_actions.py index efa1db3b94..9cd923eb93 100644 --- a/synapse/storage/event_push_actions.py +++ b/synapse/storage/event_push_actions.py @@ -37,7 +37,6 @@ class EventPushActionsStore(SQLBaseStore): index_name="event_push_actions_u_highlight", table="event_push_actions", columns=["user_id", "stream_ordering"], - where_clause="highlight = 1", ) def _set_push_actions_for_event_and_users_txn(self, txn, event, tuples): -- cgit 1.5.1