Merge branch 'release-v0.4.0' of github.com:matrix-org/synapse

11 files changed, 211 insertions, 62 deletions

diff --git a/synapse/storage/__init__.py b/synapse/storage/__init__.py
index 15919eb580..3aa6345a7f 100644
--- a/synapse/storage/__init__.py
+++ b/synapse/storage/__init__.py
@@ -57,13 +57,14 @@ SCHEMAS = [
     "presence",
     "im",
     "room_aliases",
+    "keys",
     "redactions",
 ]
 
 
 # Remember to update this number every time an incompatible change is made to
 # database schema files, so the users will be informed on server restarts.
-SCHEMA_VERSION = 4
+SCHEMA_VERSION = 6
 
 
 class _RollbackButIsFineException(Exception):
@@ -105,7 +106,7 @@ class DataStore(RoomMemberStore, RoomStore,
                 stream_ordering=stream_ordering,
                 is_new_state=is_new_state,
             )
-        except _RollbackButIsFineException as e:
+        except _RollbackButIsFineException:
             pass
 
     @defer.inlineCallbacks
@@ -154,6 +155,8 @@ class DataStore(RoomMemberStore, RoomStore,
 
         cols["unrecognized_keys"] = json.dumps(unrec_keys)
 
+        cols["ts"] = cols.pop("origin_server_ts")
+
         logger.debug("Persisting: %s", repr(cols))
 
         if pdu.is_state:
@@ -294,6 +297,28 @@ class DataStore(RoomMemberStore, RoomStore,
 
         defer.returnValue(self.min_token)
 
+    def insert_client_ip(self, user, access_token, device_id, ip, user_agent):
+        return self._simple_insert(
+            "user_ips",
+            {
+                "user": user.to_string(),
+                "access_token": access_token,
+                "device_id": device_id,
+                "ip": ip,
+                "user_agent": user_agent,
+                "last_seen": int(self._clock.time_msec()),
+            }
+        )
+
+    def get_user_ip_and_agents(self, user):
+        return self._simple_select_list(
+            table="user_ips",
+            keyvalues={"user": user.to_string()},
+            retcols=[
+                "device_id", "access_token", "ip", "user_agent", "last_seen"
+            ],
+        )
+
     def snapshot_room(self, room_id, user_id, state_type=None, state_key=None):
         """Snapshot the room for an update by a user
         Args:
diff --git a/synapse/storage/_base.py b/synapse/storage/_base.py
index 889de2bedc..65a86e9056 100644
--- a/synapse/storage/_base.py
+++ b/synapse/storage/_base.py
@@ -121,7 +121,7 @@ class SQLBaseStore(object):
     # "Simple" SQL API methods that operate on a single table with no JOINs,
     # no complex WHERE clauses, just a dict of values for columns.
 
-    def _simple_insert(self, table, values, or_replace=False):
+    def _simple_insert(self, table, values, or_replace=False, or_ignore=False):
         """Executes an INSERT query on the named table.
 
         Args:
@@ -130,13 +130,16 @@ class SQLBaseStore(object):
             or_replace : bool; if True performs an INSERT OR REPLACE
         """
         return self.runInteraction(
-            self._simple_insert_txn, table, values, or_replace=or_replace
+            self._simple_insert_txn, table, values, or_replace=or_replace,
+            or_ignore=or_ignore,
         )
 
     @log_function
-    def _simple_insert_txn(self, txn, table, values, or_replace=False):
+    def _simple_insert_txn(self, txn, table, values, or_replace=False,
+                           or_ignore=False):
         sql = "%s INTO %s (%s) VALUES(%s)" % (
-            ("INSERT OR REPLACE" if or_replace else "INSERT"),
+            ("INSERT OR REPLACE" if or_replace else
+             "INSERT OR IGNORE" if or_ignore else "INSERT"),
             table,
             ", ".join(k for k in values),
             ", ".join("?" for k in values)
@@ -351,6 +354,7 @@ class SQLBaseStore(object):
         d.pop("stream_ordering", None)
         d.pop("topological_ordering", None)
         d.pop("processed", None)
+        d["origin_server_ts"] = d.pop("ts", 0)
 
         d.update(json.loads(row_dict["unrecognized_keys"]))
         d["content"] = json.loads(d["content"])
@@ -358,7 +362,7 @@ class SQLBaseStore(object):
 
         if "age_ts" not in d:
             # For compatibility
-            d["age_ts"] = d["ts"] if "ts" in d else 0
+            d["age_ts"] = d.get("origin_server_ts", 0)
 
         return self.event_factory.create_event(
             etype=d["type"],
diff --git a/synapse/storage/directory.py b/synapse/storage/directory.py
index 540eb4c2c4..52373a28a6 100644
--- a/synapse/storage/directory.py
+++ b/synapse/storage/directory.py
@@ -93,6 +93,36 @@ class DirectoryStore(SQLBaseStore):
                 }
             )
 
+    def delete_room_alias(self, room_alias):
+        return self.runInteraction(
+            self._delete_room_alias_txn,
+            room_alias,
+        )
+
+    def _delete_room_alias_txn(self, txn, room_alias):
+        cursor = txn.execute(
+            "SELECT room_id FROM room_aliases WHERE room_alias = ?",
+            (room_alias.to_string(),)
+        )
+
+        res = cursor.fetchone()
+        if res:
+            room_id = res[0]
+        else:
+            return None
+
+        txn.execute(
+            "DELETE FROM room_aliases WHERE room_alias = ?",
+            (room_alias.to_string(),)
+        )
+
+        txn.execute(
+            "DELETE FROM room_alias_servers WHERE room_alias = ?",
+            (room_alias.to_string(),)
+        )
+
+        return room_id
+
     def get_aliases_for_room(self, room_id):
         return self._simple_select_onecol(
             "room_aliases",
diff --git a/synapse/storage/keys.py b/synapse/storage/keys.py
index 5a38c3e8f2..8189e071a3 100644
--- a/synapse/storage/keys.py
+++ b/synapse/storage/keys.py
@@ -18,7 +18,8 @@ from _base import SQLBaseStore
 from twisted.internet import defer
 
 import OpenSSL
-import nacl.signing
+from  syutil.crypto.signing_key import decode_verify_key_bytes
+import hashlib
 
 class KeyStore(SQLBaseStore):
     """Persistence for signature verification keys and tls X.509 certificates
@@ -42,62 +43,76 @@ class KeyStore(SQLBaseStore):
         )
         defer.returnValue(tls_certificate)
 
-    def store_server_certificate(self, server_name, key_server, ts_now_ms,
+    def store_server_certificate(self, server_name, from_server, time_now_ms,
                                  tls_certificate):
         """Stores the TLS X.509 certificate for the given server
         Args:
-            server_name (bytes): The name of the server.
-            key_server (bytes): Where the certificate was looked up
-            ts_now_ms (int): The time now in milliseconds
+            server_name (str): The name of the server.
+            from_server (str): Where the certificate was looked up
+            time_now_ms (int): The time now in milliseconds
             tls_certificate (OpenSSL.crypto.X509): The X.509 certificate.
         """
         tls_certificate_bytes = OpenSSL.crypto.dump_certificate(
             OpenSSL.crypto.FILETYPE_ASN1, tls_certificate
         )
+        fingerprint = hashlib.sha256(tls_certificate_bytes).hexdigest()
         return self._simple_insert(
             table="server_tls_certificates",
-            keyvalues={
+            values={
                 "server_name": server_name,
-                "key_server": key_server,
-                "ts_added_ms": ts_now_ms,
-                "tls_certificate": tls_certificate_bytes,
+                "fingerprint": fingerprint,
+                "from_server": from_server,
+                "ts_added_ms": time_now_ms,
+                "tls_certificate": buffer(tls_certificate_bytes),
             },
+            or_ignore=True,
         )
 
     @defer.inlineCallbacks
-    def get_server_verification_key(self, server_name):
-        """Retrieve the NACL verification key for a given server
+    def get_server_verify_keys(self, server_name, key_ids):
+        """Retrieve the NACL verification key for a given server for the given
+        key_ids
         Args:
-            server_name (bytes): The name of the server.
+            server_name (str): The name of the server.
+            key_ids (list of str): List of key_ids to try and look up.
         Returns:
-            (nacl.signing.VerifyKey): The verification key.
+            (list of VerifyKey): The verification keys.
         """
-        verification_key_bytes, = yield self._simple_select_one(
-            table="server_signature_keys",
-            key_values={"server_name": server_name},
-            retcols=("tls_certificate",),
+        sql = (
+            "SELECT key_id, verify_key FROM server_signature_keys"
+            " WHERE server_name = ?"
+            " AND key_id in (" + ",".join("?" for key_id in key_ids) + ")"
         )
-        verification_key = nacl.signing.VerifyKey(verification_key_bytes)
-        defer.returnValue(verification_key)
 
-    def store_server_verification_key(self, server_name, key_version,
-                                      key_server, ts_now_ms, verification_key):
+        rows = yield self._execute_and_decode(sql, server_name, *key_ids)
+
+        keys = []
+        for row in rows:
+            key_id = row["key_id"]
+            key_bytes = row["verify_key"]
+            key = decode_verify_key_bytes(key_id, str(key_bytes))
+            keys.append(key)
+        defer.returnValue(keys)
+
+    def store_server_verify_key(self, server_name, from_server, time_now_ms,
+                                verify_key):
         """Stores a NACL verification key for the given server.
         Args:
-            server_name (bytes): The name of the server.
-            key_version (bytes): The version of the key for the server.
-            key_server (bytes): Where the verification key was looked up
+            server_name (str): The name of the server.
+            key_id (str): The version of the key for the server.
+            from_server (str): Where the verification key was looked up
             ts_now_ms (int): The time now in milliseconds
-            verification_key (nacl.signing.VerifyKey): The NACL verify key.
+            verification_key (VerifyKey): The NACL verify key.
         """
-        verification_key_bytes = verification_key.encode()
+        verify_key_bytes = verify_key.encode()
         return self._simple_insert(
             table="server_signature_keys",
-            key_values={
+            values={
                 "server_name": server_name,
-                "key_version": key_version,
-                "key_server": key_server,
-                "ts_added_ms": ts_now_ms,
-                "verification_key": verification_key_bytes,
+                "key_id": "%s:%s" % (verify_key.alg, verify_key.version),
+                "from_server": from_server,
+                "ts_added_ms": time_now_ms,
+                "verify_key": buffer(verify_key.encode()),
             },
+            or_ignore=True,
         )
diff --git a/synapse/storage/registration.py b/synapse/storage/registration.py
index db20b1daa0..719806f82b 100644
--- a/synapse/storage/registration.py
+++ b/synapse/storage/registration.py
@@ -88,27 +88,40 @@ class RegistrationStore(SQLBaseStore):
             query, user_id
         )
 
-    @defer.inlineCallbacks
     def get_user_by_token(self, token):
         """Get a user from the given access token.
 
         Args:
             token (str): The access token of a user.
         Returns:
-            str: The user ID of the user.
+            dict: Including the name (user_id), device_id and whether they are
+                an admin.
         Raises:
             StoreError if no user was found.
         """
-        user_id = yield self.runInteraction(self._query_for_auth,
-                                                     token)
-        defer.returnValue(user_id)
+        return self.runInteraction(
+            self._query_for_auth,
+            token
+        )
+
+    def is_server_admin(self, user):
+        return self._simple_select_one_onecol(
+            table="users",
+            keyvalues={"name": user.to_string()},
+            retcol="admin",
+        )
 
     def _query_for_auth(self, txn, token):
-        txn.execute("SELECT users.name FROM access_tokens LEFT JOIN users" +
-                    " ON users.id = access_tokens.user_id WHERE token = ?",
-                    [token])
-        row = txn.fetchone()
-        if row:
-            return row[0]
+        sql = (
+            "SELECT users.name, users.admin, access_tokens.device_id "
+            "FROM users "
+            "INNER JOIN access_tokens on users.id = access_tokens.user_id "
+            "WHERE token = ?"
+        )
+
+        cursor = txn.execute(sql, (token,))
+        rows = self.cursor_to_dict(cursor)
+        if rows:
+            return rows[0]
 
         raise StoreError(404, "Token not found.")
diff --git a/synapse/storage/roommember.py b/synapse/storage/roommember.py
index 958e730591..ceeef5880e 100644
--- a/synapse/storage/roommember.py
+++ b/synapse/storage/roommember.py
@@ -18,7 +18,6 @@ from twisted.internet import defer
 from ._base import SQLBaseStore
 
 from synapse.api.constants import Membership
-from synapse.util.logutils import log_function
 
 import logging
 
diff --git a/synapse/storage/schema/delta/v5.sql b/synapse/storage/schema/delta/v5.sql
new file mode 100644
index 0000000000..af9df11aa9
--- /dev/null
+++ b/synapse/storage/schema/delta/v5.sql
@@ -0,0 +1,16 @@
+
+CREATE TABLE IF NOT EXISTS user_ips (
+    user TEXT NOT NULL,
+    access_token TEXT NOT NULL,
+    device_id TEXT,
+    ip TEXT NOT NULL,
+    user_agent TEXT NOT NULL,
+    last_seen INTEGER NOT NULL,
+    CONSTRAINT user_ip UNIQUE (user, access_token, ip, user_agent) ON CONFLICT REPLACE
+);
+
+CREATE INDEX IF NOT EXISTS user_ips_user ON user_ips(user);
+
+ALTER TABLE users ADD COLUMN admin BOOL DEFAULT 0 NOT NULL;
+
+PRAGMA user_version = 5;
diff --git a/synapse/storage/schema/delta/v6.sql b/synapse/storage/schema/delta/v6.sql
new file mode 100644
index 0000000000..9bf2068d84
--- /dev/null
+++ b/synapse/storage/schema/delta/v6.sql
@@ -0,0 +1,31 @@
+/* Copyright 2014 OpenMarket Ltd
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+CREATE TABLE IF NOT EXISTS server_tls_certificates(
+  server_name TEXT, -- Server name.
+  fingerprint TEXT, -- Certificate fingerprint.
+  from_server TEXT, -- Which key server the certificate was fetched from.
+  ts_added_ms INTEGER, -- When the certifcate was added.
+  tls_certificate BLOB, -- DER encoded x509 certificate.
+  CONSTRAINT uniqueness UNIQUE (server_name, fingerprint)
+);
+
+CREATE TABLE IF NOT EXISTS server_signature_keys(
+  server_name TEXT, -- Server name.
+  key_id TEXT, -- Key version.
+  from_server TEXT, -- Which key server the key was fetched form.
+  ts_added_ms INTEGER, -- When the key was added.
+  verify_key BLOB, -- NACL verification key.
+  CONSTRAINT uniqueness UNIQUE (server_name, key_id)
+);
diff --git a/synapse/storage/schema/keys.sql b/synapse/storage/schema/keys.sql
index 706a1a03ff..9bf2068d84 100644
--- a/synapse/storage/schema/keys.sql
+++ b/synapse/storage/schema/keys.sql
@@ -14,17 +14,18 @@
  */
 CREATE TABLE IF NOT EXISTS server_tls_certificates(
   server_name TEXT, -- Server name.
-  key_server TEXT, -- Which key server the certificate was fetched from.
+  fingerprint TEXT, -- Certificate fingerprint.
+  from_server TEXT, -- Which key server the certificate was fetched from.
   ts_added_ms INTEGER, -- When the certifcate was added.
   tls_certificate BLOB, -- DER encoded x509 certificate.
-  CONSTRAINT uniqueness UNIQUE (server_name)
+  CONSTRAINT uniqueness UNIQUE (server_name, fingerprint)
 );
 
 CREATE TABLE IF NOT EXISTS server_signature_keys(
   server_name TEXT, -- Server name.
-  key_version TEXT, -- Key version.
-  key_server TEXT, -- Which key server the key was fetched form.
+  key_id TEXT, -- Key version.
+  from_server TEXT, -- Which key server the key was fetched form.
   ts_added_ms INTEGER, -- When the key was added.
-  verification_key BLOB, -- NACL verification key.
-  CONSTRAINT uniqueness UNIQUE (server_name, key_version)
+  verify_key BLOB, -- NACL verification key.
+  CONSTRAINT uniqueness UNIQUE (server_name, key_id)
 );
diff --git a/synapse/storage/schema/users.sql b/synapse/storage/schema/users.sql
index 2519702971..8244f733bd 100644
--- a/synapse/storage/schema/users.sql
+++ b/synapse/storage/schema/users.sql
@@ -17,6 +17,7 @@ CREATE TABLE IF NOT EXISTS users(
     name TEXT,
     password_hash TEXT,
     creation_ts INTEGER,
+    admin BOOL DEFAULT 0 NOT NULL,
     UNIQUE(name) ON CONFLICT ROLLBACK
 );
 
@@ -29,3 +30,16 @@ CREATE TABLE IF NOT EXISTS access_tokens(
     FOREIGN KEY(user_id) REFERENCES users(id),
     UNIQUE(token) ON CONFLICT ROLLBACK
 );
+
+CREATE TABLE IF NOT EXISTS user_ips (
+    user TEXT NOT NULL,
+    access_token TEXT NOT NULL,
+    device_id TEXT,
+    ip TEXT NOT NULL,
+    user_agent TEXT NOT NULL,
+    last_seen INTEGER NOT NULL,
+    CONSTRAINT user_ip UNIQUE (user, access_token, ip, user_agent) ON CONFLICT REPLACE
+);
+
+CREATE INDEX IF NOT EXISTS user_ips_user ON user_ips(user);
+
diff --git a/synapse/storage/transactions.py b/synapse/storage/transactions.py
index ab4599b468..2ba8e30efe 100644
--- a/synapse/storage/transactions.py
+++ b/synapse/storage/transactions.py
@@ -87,7 +87,8 @@ class TransactionStore(SQLBaseStore):
 
         txn.execute(query, (code, response_json, transaction_id, origin))
 
-    def prep_send_transaction(self, transaction_id, destination, ts, pdu_list):
+    def prep_send_transaction(self, transaction_id, destination,
+                              origin_server_ts, pdu_list):
         """Persists an outgoing transaction and calculates the values for the
         previous transaction id list.
 
@@ -97,7 +98,7 @@ class TransactionStore(SQLBaseStore):
         Args:
             transaction_id (str)
             destination (str)
-            ts (int)
+            origin_server_ts (int)
             pdu_list (list)
 
         Returns:
@@ -106,11 +107,11 @@ class TransactionStore(SQLBaseStore):
 
         return self.runInteraction(
             self._prep_send_transaction,
-            transaction_id, destination, ts, pdu_list
+            transaction_id, destination, origin_server_ts, pdu_list
         )
 
-    def _prep_send_transaction(self, txn, transaction_id, destination, ts,
-                               pdu_list):
+    def _prep_send_transaction(self, txn, transaction_id, destination,
+                               origin_server_ts, pdu_list):
 
         # First we find out what the prev_txs should be.
         # Since we know that we are only sending one transaction at a time,
@@ -131,7 +132,7 @@ class TransactionStore(SQLBaseStore):
             None,
             transaction_id=transaction_id,
             destination=destination,
-            ts=ts,
+            ts=origin_server_ts,
             response_code=0,
             response_json=None
         ))