From e993925279b9e4cc00d52e7331cc5fa3cd4ae59f Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Thu, 25 Aug 2016 17:35:37 +0100 Subject: Add store-and-forward direct-to-device messaging --- synapse/rest/client/v2_alpha/sendtodevice.py | 69 ++++++++++++++++++++++++++++ 1 file changed, 69 insertions(+) create mode 100644 synapse/rest/client/v2_alpha/sendtodevice.py (limited to 'synapse/rest/client/v2_alpha/sendtodevice.py') diff --git a/synapse/rest/client/v2_alpha/sendtodevice.py b/synapse/rest/client/v2_alpha/sendtodevice.py new file mode 100644 index 0000000000..6180266499 --- /dev/null +++ b/synapse/rest/client/v2_alpha/sendtodevice.py @@ -0,0 +1,69 @@ +# -*- coding: utf-8 -*- +# Copyright 2016 OpenMarket Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import logging + +from twisted.internet import defer +from synapse.http.servlet import parse_json_object_from_request + +from synapse.http import servlet +from ._base import client_v2_patterns + +logger = logging.getLogger(__name__) + + +class SendToDeviceRestServlet(servlet.RestServlet): + PATTERNS = client_v2_patterns( + "/sendToDevice/(?P[^/]*)/(?P[^/]*)$", + releases=[], v2_alpha=False + ) + + def __init__(self, hs): + """ + Args: + hs (synapse.server.HomeServer): server + """ + super(SendToDeviceRestServlet, self).__init__() + self.hs = hs + self.auth = hs.get_auth() + self.store = hs.get_datastore() + self.is_mine_id = hs.is_mine_id + + @defer.inlineCallbacks + def on_PUT(self, request, message_type, txn_id): + requester = yield self.auth.get_user_by_req(request) + + content = parse_json_object_from_request(request) + + local_messages = {} + for user_id, by_device in content["messages"].items(): + if self.is_mine_id(user_id): + messages_by_device = { + device_id: { + "content": message_content, + "type": message_type, + "sender": requester.user.to_string(), + } + for device_id, message_content in by_device.items() + } + local_messages[user_id] = messages_by_device + + yield self.store.add_messages_to_device_inbox(local_messages) + + defer.returnValue((200, {})) + + +def register_servlets(hs, http_server): + SendToDeviceRestServlet(hs).register(http_server) -- cgit 1.5.1 From 641efb6a39f0d913895cec73b1bda8dbb548c2a3 Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Thu, 25 Aug 2016 18:14:02 +0100 Subject: Fix the deduplication of incoming direct-to-device messages --- synapse/rest/client/v2_alpha/sendtodevice.py | 13 ++++++++++++- synapse/storage/deviceinbox.py | 4 ++++ 2 files changed, 16 insertions(+), 1 deletion(-) (limited to 'synapse/rest/client/v2_alpha/sendtodevice.py') diff --git a/synapse/rest/client/v2_alpha/sendtodevice.py b/synapse/rest/client/v2_alpha/sendtodevice.py index 6180266499..42754e8774 100644 --- a/synapse/rest/client/v2_alpha/sendtodevice.py +++ b/synapse/rest/client/v2_alpha/sendtodevice.py @@ -19,6 +19,7 @@ from twisted.internet import defer from synapse.http.servlet import parse_json_object_from_request from synapse.http import servlet +from synapse.rest.client.v1.transactions import HttpTransactionStore from ._base import client_v2_patterns logger = logging.getLogger(__name__) @@ -40,9 +41,17 @@ class SendToDeviceRestServlet(servlet.RestServlet): self.auth = hs.get_auth() self.store = hs.get_datastore() self.is_mine_id = hs.is_mine_id + self.txns = HttpTransactionStore() @defer.inlineCallbacks def on_PUT(self, request, message_type, txn_id): + try: + defer.returnValue( + self.txns.get_client_transaction(request, txn_id) + ) + except KeyError: + pass + requester = yield self.auth.get_user_by_req(request) content = parse_json_object_from_request(request) @@ -62,7 +71,9 @@ class SendToDeviceRestServlet(servlet.RestServlet): yield self.store.add_messages_to_device_inbox(local_messages) - defer.returnValue((200, {})) + response = (200, {}) + self.txns.store_client_transaction(request, txn_id, response) + defer.returnValue(response) def register_servlets(hs, http_server): diff --git a/synapse/storage/deviceinbox.py b/synapse/storage/deviceinbox.py index 0b71a5cd90..e682fe1bd1 100644 --- a/synapse/storage/deviceinbox.py +++ b/synapse/storage/deviceinbox.py @@ -132,5 +132,9 @@ class DeviceInboxStore(SQLBaseStore): ) txn.execute(sql, (user_id, device_id, up_to_stream_id)) + return self.runInteraction( + "delete_messages_for_device", delete_messages_for_device_txn + ) + def get_to_device_stream_token(self): return self._device_inbox_id_gen.get_current_token() -- cgit 1.5.1 From b162cb2e41d2b55803d4371e1c978a74acc16b64 Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Thu, 25 Aug 2016 18:18:53 +0100 Subject: Add some TODOs --- synapse/rest/client/v2_alpha/sendtodevice.py | 4 ++++ 1 file changed, 4 insertions(+) (limited to 'synapse/rest/client/v2_alpha/sendtodevice.py') diff --git a/synapse/rest/client/v2_alpha/sendtodevice.py b/synapse/rest/client/v2_alpha/sendtodevice.py index 42754e8774..00533741af 100644 --- a/synapse/rest/client/v2_alpha/sendtodevice.py +++ b/synapse/rest/client/v2_alpha/sendtodevice.py @@ -56,6 +56,10 @@ class SendToDeviceRestServlet(servlet.RestServlet): content = parse_json_object_from_request(request) + # TODO: Prod the notifier to wake up sync streams. + # TODO: Implement replication for the messages. + # TODO: Send the messages to remote servers if needed. + local_messages = {} for user_id, by_device in content["messages"].items(): if self.is_mine_id(user_id): -- cgit 1.5.1 From 1aa3e1d2874370fbb6fa9f4fd2b8a110d81981fc Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Wed, 31 Aug 2016 10:38:58 +0100 Subject: Add a replication stream for direct to device messages --- synapse/replication/resource.py | 19 +++++++++++- synapse/replication/slave/storage/deviceinbox.py | 12 ++++++++ synapse/rest/client/v2_alpha/sendtodevice.py | 10 ++++-- synapse/storage/deviceinbox.py | 39 ++++++++++++++++++++++++ 4 files changed, 77 insertions(+), 3 deletions(-) (limited to 'synapse/rest/client/v2_alpha/sendtodevice.py') diff --git a/synapse/replication/resource.py b/synapse/replication/resource.py index 84993b33b3..533676814a 100644 --- a/synapse/replication/resource.py +++ b/synapse/replication/resource.py @@ -42,6 +42,7 @@ STREAM_NAMES = ( ("pushers",), ("state",), ("caches",), + ("to_device",), ) @@ -144,6 +145,7 @@ class ReplicationResource(Resource): pushers_token, state_token, caches_token, + int(stream_token.to_device_key), )) @request_handler() @@ -193,6 +195,7 @@ class ReplicationResource(Resource): yield self.pushers(writer, current_token, limit, request_streams) yield self.state(writer, current_token, limit, request_streams) yield self.caches(writer, current_token, limit, request_streams) + yield self.to_device(writer, current_token, limit, request_streams) self.streams(writer, current_token, request_streams) logger.info("Replicated %d rows", writer.total) @@ -398,6 +401,20 @@ class ReplicationResource(Resource): "position", "cache_func", "keys", "invalidation_ts" )) + @defer.inlineCallbacks + def to_device(self, writer, current_token, limit, request_streams): + current_position = current_token.to_device + + to_device = request_streams.get("to_device") + + if to_device is not None: + to_device_rows = yield self.store.get_all_new_device_messages( + to_device, current_position, limit + ) + writer.write_header_and_rows("to_device", to_device_rows, ( + "position", "user_id", "device_id", "message_json" + )) + class _Writer(object): """Writes the streams as a JSON object as the response to the request""" @@ -426,7 +443,7 @@ class _Writer(object): class _ReplicationToken(collections.namedtuple("_ReplicationToken", ( "events", "presence", "typing", "receipts", "account_data", "backfill", - "push_rules", "pushers", "state", "caches", + "push_rules", "pushers", "state", "caches", "to_device", ))): __slots__ = [] diff --git a/synapse/replication/slave/storage/deviceinbox.py b/synapse/replication/slave/storage/deviceinbox.py index 7583d23708..64d8eb2af1 100644 --- a/synapse/replication/slave/storage/deviceinbox.py +++ b/synapse/replication/slave/storage/deviceinbox.py @@ -28,3 +28,15 @@ class SlavedDeviceInboxStore(BaseSlavedStore): get_to_device_stream_token = DataStore.get_to_device_stream_token.__func__ get_new_messages_for_device = DataStore.get_new_messages_for_device.__func__ delete_messages_for_device = DataStore.delete_messages_for_device.__func__ + + def stream_positions(self): + result = super(SlavedDeviceInboxStore, self).stream_positions() + result["to_device"] = self._device_inbox_id_gen.get_current_token() + return result + + def process_replication(self, result): + stream = result.get("to_device") + if stream: + self._device_inbox_id_gen.advance(int(stream["position"])) + + return super(SlavedDeviceInboxStore, self).process_replication(result) diff --git a/synapse/rest/client/v2_alpha/sendtodevice.py b/synapse/rest/client/v2_alpha/sendtodevice.py index 00533741af..7c0991ca55 100644 --- a/synapse/rest/client/v2_alpha/sendtodevice.py +++ b/synapse/rest/client/v2_alpha/sendtodevice.py @@ -40,6 +40,7 @@ class SendToDeviceRestServlet(servlet.RestServlet): self.hs = hs self.auth = hs.get_auth() self.store = hs.get_datastore() + self.notifier = hs.get_notifier() self.is_mine_id = hs.is_mine_id self.txns = HttpTransactionStore() @@ -71,9 +72,14 @@ class SendToDeviceRestServlet(servlet.RestServlet): } for device_id, message_content in by_device.items() } - local_messages[user_id] = messages_by_device + if messages_by_device: + local_messages[user_id] = messages_by_device - yield self.store.add_messages_to_device_inbox(local_messages) + stream_id = yield self.store.add_messages_to_device_inbox(local_messages) + + self.notifier.on_new_event( + "to_device", stream_id, users=local_messages.keys() + ) response = (200, {}) self.txns.store_client_transaction(request, txn_id, response) diff --git a/synapse/storage/deviceinbox.py b/synapse/storage/deviceinbox.py index e682fe1bd1..27ed1004da 100644 --- a/synapse/storage/deviceinbox.py +++ b/synapse/storage/deviceinbox.py @@ -136,5 +136,44 @@ class DeviceInboxStore(SQLBaseStore): "delete_messages_for_device", delete_messages_for_device_txn ) + def get_all_new_device_messages(self, last_pos, current_pos, limit): + """ + Args: + last_pos(int): + current_pos(int): + limit(int): + Returns: + A deferred list of rows from the device inbox + """ + if last_pos == current_pos: + return defer.succeed([]) + + def get_all_new_device_messages_txn(txn): + sql = ( + "SELECT stream_id FROM device_inbox" + " WHERE ? < stream_id AND stream_id <= ?" + " GROUP BY stream_id" + " ORDER BY stream_id ASC" + " LIMIT ?" + ) + txn.execute(sql, (last_pos, current_pos, limit)) + stream_ids = txn.fetchall() + if not stream_ids: + return [] + max_stream_id_in_limit = stream_ids[-1] + + sql = ( + "SELECT stream_id, user_id, device_id, message_json" + " FROM device_inbox" + " WHERE ? < stream_id AND stream_id <= ?" + " ORDER BY stream_id ASC" + ) + txn.execute(sql, (last_pos, max_stream_id_in_limit)) + return txn.fetchall() + + return self.runInteraction( + "get_all_new_device_messages", get_all_new_device_messages_txn + ) + def get_to_device_stream_token(self): return self._device_inbox_id_gen.get_current_token() -- cgit 1.5.1 From 7ed5acacf4eeba4e5c0fb03f0ed433c0461c968b Mon Sep 17 00:00:00 2001 From: Mark Haines Date: Thu, 1 Sep 2016 18:08:40 +0100 Subject: Fix up the calls to the notifier for device messages --- synapse/app/synchrotron.py | 3 +++ synapse/rest/client/v2_alpha/sendtodevice.py | 2 +- 2 files changed, 4 insertions(+), 1 deletion(-) (limited to 'synapse/rest/client/v2_alpha/sendtodevice.py') diff --git a/synapse/app/synchrotron.py b/synapse/app/synchrotron.py index b8cedc7af5..07d3d047c6 100644 --- a/synapse/app/synchrotron.py +++ b/synapse/app/synchrotron.py @@ -399,6 +399,9 @@ class SynchrotronServer(HomeServer): notify_from_stream( result, "typing", "typing_key", room="room_id" ) + notify_from_stream( + result, "to_device", "to_device_key", user="user_id" + ) while True: try: diff --git a/synapse/rest/client/v2_alpha/sendtodevice.py b/synapse/rest/client/v2_alpha/sendtodevice.py index 7c0991ca55..9c10a99acf 100644 --- a/synapse/rest/client/v2_alpha/sendtodevice.py +++ b/synapse/rest/client/v2_alpha/sendtodevice.py @@ -78,7 +78,7 @@ class SendToDeviceRestServlet(servlet.RestServlet): stream_id = yield self.store.add_messages_to_device_inbox(local_messages) self.notifier.on_new_event( - "to_device", stream_id, users=local_messages.keys() + "to_device_key", stream_id, users=local_messages.keys() ) response = (200, {}) -- cgit 1.5.1