GET /devices endpoint

implement a GET /devices endpoint which lists all of the user's devices.

It also returns the last IP where we saw that device, so there is some dancing
to fish that out of the user_ips table.

10 files changed, 397 insertions, 17 deletions

diff --git a/synapse/handlers/device.py b/synapse/handlers/device.py
index 8d7d9874f8..6bbbf59e52 100644
--- a/synapse/handlers/device.py
+++ b/synapse/handlers/device.py
@@ -69,3 +69,30 @@ class DeviceHandler(BaseHandler):
                 attempts += 1
 
         raise StoreError(500, "Couldn't generate a device ID.")
+
+    @defer.inlineCallbacks
+    def get_devices_by_user(self, user_id):
+        """
+        Retrieve the given user's devices
+
+        Args:
+            user_id (str):
+        Returns:
+            defer.Deferred: dict[str, dict[str, X]]: map from device_id to
+            info on the device
+        """
+
+        devices = yield self.store.get_devices_by_user(user_id)
+
+        ips = yield self.store.get_last_client_ip_by_device(
+            devices=((user_id, device_id) for device_id in devices.keys())
+        )
+
+        for device_id in devices.keys():
+            ip = ips.get((user_id, device_id), {})
+            devices[device_id].update({
+                "last_seen_ts": ip.get("last_seen"),
+                "last_seen_ip": ip.get("ip"),
+            })
+
+        defer.returnValue(devices)
diff --git a/synapse/rest/__init__.py b/synapse/rest/__init__.py
index 8b223e032b..14227f1cdb 100644
--- a/synapse/rest/__init__.py
+++ b/synapse/rest/__init__.py
@@ -46,6 +46,7 @@ from synapse.rest.client.v2_alpha import (
     account_data,
     report_event,
     openid,
+    devices,
 )
 
 from synapse.http.server import JsonResource
@@ -90,3 +91,4 @@ class ClientRestResource(JsonResource):
         account_data.register_servlets(hs, client_resource)
         report_event.register_servlets(hs, client_resource)
         openid.register_servlets(hs, client_resource)
+        devices.register_servlets(hs, client_resource)
diff --git a/synapse/rest/client/v2_alpha/_base.py b/synapse/rest/client/v2_alpha/_base.py
index b6faa2b0e6..20e765f48f 100644
--- a/synapse/rest/client/v2_alpha/_base.py
+++ b/synapse/rest/client/v2_alpha/_base.py
@@ -25,7 +25,9 @@ import logging
 logger = logging.getLogger(__name__)
 
 
-def client_v2_patterns(path_regex, releases=(0,)):
+def client_v2_patterns(path_regex, releases=(0,),
+                       v2_alpha=True,
+                       unstable=True):
     """Creates a regex compiled client path with the correct client path
     prefix.
 
@@ -35,9 +37,12 @@ def client_v2_patterns(path_regex, releases=(0,)):
     Returns:
         SRE_Pattern
     """
-    patterns = [re.compile("^" + CLIENT_V2_ALPHA_PREFIX + path_regex)]
-    unstable_prefix = CLIENT_V2_ALPHA_PREFIX.replace("/v2_alpha", "/unstable")
-    patterns.append(re.compile("^" + unstable_prefix + path_regex))
+    patterns = []
+    if v2_alpha:
+        patterns.append(re.compile("^" + CLIENT_V2_ALPHA_PREFIX + path_regex))
+    if unstable:
+        unstable_prefix = CLIENT_V2_ALPHA_PREFIX.replace("/v2_alpha", "/unstable")
+        patterns.append(re.compile("^" + unstable_prefix + path_regex))
     for release in releases:
         new_prefix = CLIENT_V2_ALPHA_PREFIX.replace("/v2_alpha", "/r%d" % release)
         patterns.append(re.compile("^" + new_prefix + path_regex))
diff --git a/synapse/rest/client/v2_alpha/devices.py b/synapse/rest/client/v2_alpha/devices.py
new file mode 100644
index 0000000000..5cf8bd1afa
--- /dev/null
+++ b/synapse/rest/client/v2_alpha/devices.py
@@ -0,0 +1,51 @@
+# -*- coding: utf-8 -*-
+# Copyright 2015, 2016 OpenMarket Ltd
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from twisted.internet import defer
+
+from synapse.http.servlet import RestServlet
+
+from ._base import client_v2_patterns
+
+import logging
+
+
+logger = logging.getLogger(__name__)
+
+
+class DevicesRestServlet(RestServlet):
+    PATTERNS = client_v2_patterns("/devices$", releases=[], v2_alpha=False)
+
+    def __init__(self, hs):
+        """
+        Args:
+            hs (synapse.server.HomeServer): server
+        """
+        super(DevicesRestServlet, self).__init__()
+        self.hs = hs
+        self.auth = hs.get_auth()
+        self.device_handler = hs.get_device_handler()
+
+    @defer.inlineCallbacks
+    def on_GET(self, request):
+        requester = yield self.auth.get_user_by_req(request)
+        devices = yield self.device_handler.get_devices_by_user(
+            requester.user.to_string()
+        )
+        defer.returnValue((200, {"devices": devices}))
+
+
+def register_servlets(hs, http_server):
+    DevicesRestServlet(hs).register(http_server)
diff --git a/synapse/storage/client_ips.py b/synapse/storage/client_ips.py
index a90990e006..07161496ca 100644
--- a/synapse/storage/client_ips.py
+++ b/synapse/storage/client_ips.py
@@ -13,10 +13,13 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+import logging
+
 from ._base import SQLBaseStore, Cache
 
 from twisted.internet import defer
 
+logger = logging.getLogger(__name__)
 
 # Number of msec of granularity to store the user IP 'last seen' time. Smaller
 # times give more inserts into the database even for readonly API hits
@@ -66,3 +69,72 @@ class ClientIpStore(SQLBaseStore):
             desc="insert_client_ip",
             lock=False,
         )
+
+    @defer.inlineCallbacks
+    def get_last_client_ip_by_device(self, devices):
+        """For each device_id listed, give the user_ip it was last seen on
+
+        Args:
+            devices (iterable[(str, str)]):  list of (user_id, device_id) pairs
+
+        Returns:
+            defer.Deferred: resolves to a dict, where the keys
+            are (user_id, device_id) tuples. The values are also dicts, with
+            keys giving the column names
+        """
+
+        res = yield self.runInteraction(
+            "get_last_client_ip_by_device",
+            self._get_last_client_ip_by_device_txn,
+            retcols=(
+                "user_id",
+                "access_token",
+                "ip",
+                "user_agent",
+                "device_id",
+                "last_seen",
+            ),
+            devices=devices
+        )
+
+        ret = {(d["user_id"], d["device_id"]): d for d in res}
+        defer.returnValue(ret)
+
+    @classmethod
+    def _get_last_client_ip_by_device_txn(cls, txn, devices, retcols):
+        def where_clause_for_device(d):
+                return
+
+        where_clauses = []
+        bindings = []
+        for (user_id, device_id) in devices:
+            if device_id is None:
+                where_clauses.append("(user_id = ? AND device_id IS NULL)")
+                bindings.extend((user_id, ))
+            else:
+                where_clauses.append("(user_id = ? AND device_id = ?)")
+                bindings.extend((user_id, device_id))
+
+        inner_select = (
+            "SELECT MAX(last_seen) mls, user_id, device_id FROM user_ips "
+            "WHERE %(where)s "
+            "GROUP BY user_id, device_id"
+        ) % {
+            "where": " OR ".join(where_clauses),
+        }
+
+        sql = (
+            "SELECT %(retcols)s FROM user_ips "
+            "JOIN (%(inner_select)s) ips ON"
+            "    user_ips.last_seen = ips.mls AND"
+            "    user_ips.user_id = ips.user_id AND"
+            "    (user_ips.device_id = ips.device_id OR"
+            "         (user_ips.device_id IS NULL AND ips.device_id IS NULL)"
+            "    )"
+        ) % {
+            "retcols": ",".join("user_ips." + c for c in retcols),
+            "inner_select": inner_select,
+        }
+
+        txn.execute(sql, bindings)
+        return cls.cursor_to_dict(txn)
diff --git a/synapse/storage/devices.py b/synapse/storage/devices.py
index 9065e96d28..1cc6e07f2b 100644
--- a/synapse/storage/devices.py
+++ b/synapse/storage/devices.py
@@ -65,7 +65,7 @@ class DeviceStore(SQLBaseStore):
             user_id (str): The ID of the user which owns the device
             device_id (str): The ID of the device to retrieve
         Returns:
-            defer.Deferred for a namedtuple containing the device information
+            defer.Deferred for a dict containing the device information
         Raises:
             StoreError: if the device is not found
         """
@@ -75,3 +75,23 @@ class DeviceStore(SQLBaseStore):
             retcols=("user_id", "device_id", "display_name"),
             desc="get_device",
         )
+
+    @defer.inlineCallbacks
+    def get_devices_by_user(self, user_id):
+        """Retrieve all of a user's registered devices.
+
+        Args:
+            user_id (str):
+        Returns:
+            defer.Deferred: resolves to a dict from device_id to a dict
+            containing "device_id", "user_id" and "display_name" for each
+            device.
+        """
+        devices = yield self._simple_select_list(
+            table="devices",
+            keyvalues={"user_id": user_id},
+            retcols=("user_id", "device_id", "display_name"),
+            desc="get_devices_by_user"
+        )
+
+        defer.returnValue({d["device_id"]: d for d in devices})
diff --git a/synapse/storage/schema/delta/33/user_ips_index.sql b/synapse/storage/schema/delta/33/user_ips_index.sql
new file mode 100644
index 0000000000..8a05677d42
--- /dev/null
+++ b/synapse/storage/schema/delta/33/user_ips_index.sql
@@ -0,0 +1,16 @@
+/* Copyright 2016 OpenMarket Ltd
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+CREATE INDEX user_ips_device_id ON user_ips(user_id, device_id, last_seen);
diff --git a/tests/handlers/test_device.py b/tests/handlers/test_device.py
index cc6512ccc7..c2e12135d6 100644
--- a/tests/handlers/test_device.py
+++ b/tests/handlers/test_device.py
@@ -12,25 +12,27 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-
+from synapse import types
 from twisted.internet import defer
 
-from synapse.handlers.device import DeviceHandler
-from tests import unittest
-from tests.utils import setup_test_homeserver
-
-
-class DeviceHandlers(object):
-    def __init__(self, hs):
-        self.device_handler = DeviceHandler(hs)
+import synapse.handlers.device
+import synapse.storage
+from tests import unittest, utils
 
 
 class DeviceTestCase(unittest.TestCase):
+    def __init__(self, *args, **kwargs):
+        super(DeviceTestCase, self).__init__(*args, **kwargs)
+        self.store = None    # type: synapse.storage.DataStore
+        self.handler = None  # type: device.DeviceHandler
+        self.clock = None    # type: utils.MockClock
+
     @defer.inlineCallbacks
     def setUp(self):
-        self.hs = yield setup_test_homeserver(handlers=None)
-        self.hs.handlers = handlers = DeviceHandlers(self.hs)
-        self.handler = handlers.device_handler
+        hs = yield utils.setup_test_homeserver(handlers=None)
+        self.handler = synapse.handlers.device.DeviceHandler(hs)
+        self.store = hs.get_datastore()
+        self.clock = hs.get_clock()
 
     @defer.inlineCallbacks
     def test_device_is_created_if_doesnt_exist(self):
@@ -73,3 +75,55 @@ class DeviceTestCase(unittest.TestCase):
 
         dev = yield self.handler.store.get_device("theresa", device_id)
         self.assertEqual(dev["display_name"], "display")
+
+    @defer.inlineCallbacks
+    def test_get_devices_by_user(self):
+        # check this works for both devices which have a recorded client_ip,
+        # and those which don't.
+        user1 = "@boris:aaa"
+        user2 = "@theresa:bbb"
+        yield self._record_user(user1, "xyz", "display 0")
+        yield self._record_user(user1, "fco", "display 1", "token1", "ip1")
+        yield self._record_user(user1, "abc", "display 2", "token2", "ip2")
+        yield self._record_user(user1, "abc", "display 2", "token3", "ip3")
+
+        yield self._record_user(user2, "def", "dispkay", "token4", "ip4")
+
+        res = yield self.handler.get_devices_by_user(user1)
+        self.assertEqual(3, len(res.keys()))
+        self.assertDictContainsSubset({
+            "user_id": user1,
+            "device_id": "xyz",
+            "display_name": "display 0",
+            "last_seen_ip": None,
+            "last_seen_ts": None,
+        }, res["xyz"])
+        self.assertDictContainsSubset({
+            "user_id": user1,
+            "device_id": "fco",
+            "display_name": "display 1",
+            "last_seen_ip": "ip1",
+            "last_seen_ts": 1000000,
+        }, res["fco"])
+        self.assertDictContainsSubset({
+            "user_id": user1,
+            "device_id": "abc",
+            "display_name": "display 2",
+            "last_seen_ip": "ip3",
+            "last_seen_ts": 3000000,
+        }, res["abc"])
+
+    @defer.inlineCallbacks
+    def _record_user(self, user_id, device_id, display_name,
+                     access_token=None, ip=None):
+        device_id = yield self.handler.check_device_registered(
+            user_id=user_id,
+            device_id=device_id,
+            initial_device_display_name=display_name
+        )
+
+        if ip is not None:
+            yield self.store.insert_client_ip(
+                types.UserID.from_string(user_id),
+                access_token, ip, "user_agent", device_id)
+            self.clock.advance_time(1000)
\ No newline at end of file
diff --git a/tests/storage/test_client_ips.py b/tests/storage/test_client_ips.py
new file mode 100644
index 0000000000..1f0c0e7c37
--- /dev/null
+++ b/tests/storage/test_client_ips.py
@@ -0,0 +1,62 @@
+# -*- coding: utf-8 -*-
+# Copyright 2016 OpenMarket Ltd
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from twisted.internet import defer
+
+import synapse.server
+import synapse.storage
+import synapse.types
+import tests.unittest
+import tests.utils
+
+
+class ClientIpStoreTestCase(tests.unittest.TestCase):
+    def __init__(self, *args, **kwargs):
+        super(ClientIpStoreTestCase, self).__init__(*args, **kwargs)
+        self.store = None  # type: synapse.storage.DataStore
+        self.clock = None  # type: tests.utils.MockClock
+
+    @defer.inlineCallbacks
+    def setUp(self):
+        hs = yield tests.utils.setup_test_homeserver()
+        self.store = hs.get_datastore()
+        self.clock = hs.get_clock()
+
+    @defer.inlineCallbacks
+    def test_insert_new_client_ip(self):
+        self.clock.now = 12345678
+        user_id = "@user:id"
+        yield self.store.insert_client_ip(
+            synapse.types.UserID.from_string(user_id),
+            "access_token", "ip", "user_agent", "device_id",
+        )
+
+        # deliberately use an iterable here to make sure that the lookup
+        # method doesn't iterate it twice
+        device_list = iter(((user_id, "device_id"),))
+        result = yield self.store.get_last_client_ip_by_device(device_list)
+
+        r = result[(user_id, "device_id")]
+        self.assertDictContainsSubset(
+            {
+                "user_id": user_id,
+                "device_id": "device_id",
+                "access_token": "access_token",
+                "ip": "ip",
+                "user_agent": "user_agent",
+                "last_seen": 12345678000,
+            },
+            r
+        )
diff --git a/tests/storage/test_devices.py b/tests/storage/test_devices.py
new file mode 100644
index 0000000000..d3e9d97a9a
--- /dev/null
+++ b/tests/storage/test_devices.py
@@ -0,0 +1,71 @@
+# -*- coding: utf-8 -*-
+# Copyright 2016 OpenMarket Ltd
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from twisted.internet import defer
+
+import synapse.server
+import synapse.types
+import tests.unittest
+import tests.utils
+
+
+class DeviceStoreTestCase(tests.unittest.TestCase):
+    def __init__(self, *args, **kwargs):
+        super(DeviceStoreTestCase, self).__init__(*args, **kwargs)
+        self.store = None  # type: synapse.storage.DataStore
+
+    @defer.inlineCallbacks
+    def setUp(self):
+        hs = yield tests.utils.setup_test_homeserver()
+
+        self.store = hs.get_datastore()
+
+    @defer.inlineCallbacks
+    def test_store_new_device(self):
+        yield self.store.store_device(
+            "user_id", "device_id", "display_name"
+        )
+
+        res = yield self.store.get_device("user_id", "device_id")
+        self.assertDictContainsSubset({
+            "user_id": "user_id",
+            "device_id": "device_id",
+            "display_name": "display_name",
+        }, res)
+
+    @defer.inlineCallbacks
+    def test_get_devices_by_user(self):
+        yield self.store.store_device(
+            "user_id", "device1", "display_name 1"
+        )
+        yield self.store.store_device(
+            "user_id", "device2", "display_name 2"
+        )
+        yield self.store.store_device(
+            "user_id2", "device3", "display_name 3"
+        )
+
+        res = yield self.store.get_devices_by_user("user_id")
+        self.assertEqual(2, len(res.keys()))
+        self.assertDictContainsSubset({
+            "user_id": "user_id",
+            "device_id": "device1",
+            "display_name": "display_name 1",
+        }, res["device1"])
+        self.assertDictContainsSubset({
+            "user_id": "user_id",
+            "device_id": "device2",
+            "display_name": "display_name 2",
+        }, res["device2"])