Merge branch 'develop' into request_logging

Conflicts:
	setup.py
	synapse/storage/_base.py
	synapse/util/async.py

5 files changed, 160 insertions, 119 deletions

diff --git a/synapse/api/events/__init__.py b/synapse/api/events/__init__.py
index f66fea2904..1d8bed2906 100644
--- a/synapse/api/events/__init__.py
+++ b/synapse/api/events/__init__.py
@@ -13,7 +13,6 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-from synapse.api.errors import SynapseError, Codes
 from synapse.util.jsonobject import JsonEncodedObject
 
 
@@ -56,22 +55,26 @@ class SynapseEvent(JsonEncodedObject):
         "user_id",  # sender/initiator
         "content",  # HTTP body, JSON
         "state_key",
-        "required_power_level",
         "age_ts",
         "prev_content",
-        "prev_state",
+        "replaces_state",
         "redacted_because",
+        "origin_server_ts",
     ]
 
     internal_keys = [
         "is_state",
-        "prev_events",
         "depth",
         "destinations",
         "origin",
         "outlier",
-        "power_level",
         "redacted",
+        "prev_events",
+        "hashes",
+        "signatures",
+        "prev_state",
+        "auth_events",
+        "state_hash",
     ]
 
     required_keys = [
@@ -82,8 +85,8 @@ class SynapseEvent(JsonEncodedObject):
 
     def __init__(self, raises=True, **kwargs):
         super(SynapseEvent, self).__init__(**kwargs)
-        if "content" in kwargs:
-            self.check_json(self.content, raises=raises)
+        # if "content" in kwargs:
+        #     self.check_json(self.content, raises=raises)
 
     def get_content_template(self):
         """ Retrieve the JSON template for this event as a dict.
@@ -114,66 +117,6 @@ class SynapseEvent(JsonEncodedObject):
         """
         raise NotImplementedError("get_content_template not implemented.")
 
-    def check_json(self, content, raises=True):
-        """Checks the given JSON content abides by the rules of the template.
-
-        Args:
-            content : A JSON object to check.
-            raises: True to raise a SynapseError if the check fails.
-        Returns:
-            True if the content passes the template. Returns False if the check
-            fails and raises=False.
-        Raises:
-            SynapseError if the check fails and raises=True.
-        """
-        # recursively call to inspect each layer
-        err_msg = self._check_json(content, self.get_content_template())
-        if err_msg:
-            if raises:
-                raise SynapseError(400, err_msg, Codes.BAD_JSON)
-            else:
-                return False
-        else:
-            return True
-
-    def _check_json(self, content, template):
-        """Check content and template matches.
-
-        If the template is a dict, each key in the dict will be validated with
-        the content, else it will just compare the types of content and
-        template. This basic type check is required because this function will
-        be recursively called and could be called with just strs or ints.
-
-        Args:
-            content: The content to validate.
-            template: The validation template.
-        Returns:
-            str: An error message if the validation fails, else None.
-        """
-        if type(content) != type(template):
-            return "Mismatched types: %s" % template
-
-        if type(template) == dict:
-            for key in template:
-                if key not in content:
-                    return "Missing %s key" % key
-
-                if type(content[key]) != type(template[key]):
-                    return "Key %s is of the wrong type (got %s, want %s)" % (
-                        key, type(content[key]), type(template[key]))
-
-                if type(content[key]) == dict:
-                    # we must go deeper
-                    msg = self._check_json(content[key], template[key])
-                    if msg:
-                        return msg
-                elif type(content[key]) == list:
-                    # make sure each item type in content matches the template
-                    for entry in content[key]:
-                        msg = self._check_json(entry, template[key][0])
-                        if msg:
-                            return msg
-
 
 class SynapseStateEvent(SynapseEvent):
 
diff --git a/synapse/api/events/factory.py b/synapse/api/events/factory.py
index 74d0ef77f4..a1ec708a81 100644
--- a/synapse/api/events/factory.py
+++ b/synapse/api/events/factory.py
@@ -16,11 +16,13 @@
 from synapse.api.events.room import (
     RoomTopicEvent, MessageEvent, RoomMemberEvent, FeedbackEvent,
     InviteJoinEvent, RoomConfigEvent, RoomNameEvent, GenericEvent,
-    RoomPowerLevelsEvent, RoomJoinRulesEvent, RoomOpsPowerLevelsEvent,
-    RoomCreateEvent, RoomAddStateLevelEvent, RoomSendEventLevelEvent,
+    RoomPowerLevelsEvent, RoomJoinRulesEvent,
+    RoomCreateEvent,
     RoomRedactionEvent,
 )
 
+from synapse.types import EventID
+
 from synapse.util.stringutils import random_string
 
 
@@ -37,9 +39,6 @@ class EventFactory(object):
         RoomPowerLevelsEvent,
         RoomJoinRulesEvent,
         RoomCreateEvent,
-        RoomAddStateLevelEvent,
-        RoomSendEventLevelEvent,
-        RoomOpsPowerLevelsEvent,
         RoomRedactionEvent,
     ]
 
@@ -51,12 +50,26 @@ class EventFactory(object):
         self.clock = hs.get_clock()
         self.hs = hs
 
+        self.event_id_count = 0
+
+    def create_event_id(self):
+        i = str(self.event_id_count)
+        self.event_id_count += 1
+
+        local_part = str(int(self.clock.time())) + i + random_string(5)
+
+        e_id = EventID.create_local(local_part, self.hs)
+
+        return e_id.to_string()
+
     def create_event(self, etype=None, **kwargs):
         kwargs["type"] = etype
         if "event_id" not in kwargs:
-            kwargs["event_id"] = "%s@%s" % (
-                random_string(10), self.hs.hostname
-            )
+            kwargs["event_id"] = self.create_event_id()
+            kwargs["origin"] = self.hs.hostname
+        else:
+            ev_id = self.hs.parse_eventid(kwargs["event_id"])
+            kwargs["origin"] = ev_id.domain
 
         if "origin_server_ts" not in kwargs:
             kwargs["origin_server_ts"] = int(self.clock.time_msec())
diff --git a/synapse/api/events/room.py b/synapse/api/events/room.py
index cd936074fc..8c4ac45d02 100644
--- a/synapse/api/events/room.py
+++ b/synapse/api/events/room.py
@@ -154,27 +154,6 @@ class RoomPowerLevelsEvent(SynapseStateEvent):
         return {}
 
 
-class RoomAddStateLevelEvent(SynapseStateEvent):
-    TYPE = "m.room.add_state_level"
-
-    def get_content_template(self):
-        return {}
-
-
-class RoomSendEventLevelEvent(SynapseStateEvent):
-    TYPE = "m.room.send_event_level"
-
-    def get_content_template(self):
-        return {}
-
-
-class RoomOpsPowerLevelsEvent(SynapseStateEvent):
-    TYPE = "m.room.ops_levels"
-
-    def get_content_template(self):
-        return {}
-
-
 class RoomAliasesEvent(SynapseStateEvent):
     TYPE = "m.room.aliases"
 
diff --git a/synapse/api/events/utils.py b/synapse/api/events/utils.py
index c3a32be8c1..802648f8f7 100644
--- a/synapse/api/events/utils.py
+++ b/synapse/api/events/utils.py
@@ -15,21 +15,34 @@
 
 from .room import (
     RoomMemberEvent, RoomJoinRulesEvent, RoomPowerLevelsEvent,
-    RoomAddStateLevelEvent, RoomSendEventLevelEvent, RoomOpsPowerLevelsEvent,
     RoomAliasesEvent, RoomCreateEvent,
 )
 
+
 def prune_event(event):
-    """ Prunes the given event of all keys we don't know about or think could
-    potentially be dodgy.
+    """ Returns a pruned version of the given event, which removes all keys we
+    don't know about or think could potentially be dodgy.
 
     This is used when we "redact" an event. We want to remove all fields that
     the user has specified, but we do want to keep necessary information like
     type, state_key etc.
     """
+    event_type = event.type
 
-    # Remove all extraneous fields.
-    event.unrecognized_keys = {}
+    allowed_keys = [
+        "event_id",
+        "user_id",
+        "room_id",
+        "hashes",
+        "signatures",
+        "content",
+        "type",
+        "state_key",
+        "depth",
+        "prev_events",
+        "prev_state",
+        "auth_events",
+    ]
 
     new_content = {}
 
@@ -38,27 +51,33 @@ def prune_event(event):
             if field in event.content:
                 new_content[field] = event.content[field]
 
-    if event.type == RoomMemberEvent.TYPE:
+    if event_type == RoomMemberEvent.TYPE:
         add_fields("membership")
-    elif event.type == RoomCreateEvent.TYPE:
+    elif event_type == RoomCreateEvent.TYPE:
         add_fields("creator")
-    elif event.type == RoomJoinRulesEvent.TYPE:
+    elif event_type == RoomJoinRulesEvent.TYPE:
         add_fields("join_rule")
-    elif event.type == RoomPowerLevelsEvent.TYPE:
-        # TODO: Actually check these are valid user_ids etc.
-        add_fields("default")
-        for k, v in event.content.items():
-            if k.startswith("@") and isinstance(v, (int, long)):
-                new_content[k] = v
-    elif event.type == RoomAddStateLevelEvent.TYPE:
-        add_fields("level")
-    elif event.type == RoomSendEventLevelEvent.TYPE:
-        add_fields("level")
-    elif event.type == RoomOpsPowerLevelsEvent.TYPE:
-        add_fields("kick_level", "ban_level", "redact_level")
-    elif event.type == RoomAliasesEvent.TYPE:
+    elif event_type == RoomPowerLevelsEvent.TYPE:
+        add_fields(
+            "users",
+            "users_default",
+            "events",
+            "events_default",
+            "events_default",
+            "state_default",
+            "ban",
+            "kick",
+            "redact",
+        )
+    elif event_type == RoomAliasesEvent.TYPE:
         add_fields("aliases")
 
-    event.content = new_content
+    allowed_fields = {
+        k: v
+        for k, v in event.get_full_dict().items()
+        if k in allowed_keys
+    }
+
+    allowed_fields["content"] = new_content
 
-    return event
+    return type(event)(**allowed_fields)
diff --git a/synapse/api/events/validator.py b/synapse/api/events/validator.py
new file mode 100644
index 0000000000..2d4f2a3aa7
--- /dev/null
+++ b/synapse/api/events/validator.py
@@ -0,0 +1,87 @@
+# -*- coding: utf-8 -*-
+# Copyright 2014 OpenMarket Ltd
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from synapse.api.errors import SynapseError, Codes
+
+
+class EventValidator(object):
+    def __init__(self, hs):
+        pass
+
+    def validate(self, event):
+        """Checks the given JSON content abides by the rules of the template.
+
+        Args:
+            content : A JSON object to check.
+            raises: True to raise a SynapseError if the check fails.
+        Returns:
+            True if the content passes the template. Returns False if the check
+            fails and raises=False.
+        Raises:
+            SynapseError if the check fails and raises=True.
+        """
+        # recursively call to inspect each layer
+        err_msg = self._check_json_template(
+            event.content,
+            event.get_content_template()
+        )
+        if err_msg:
+            raise SynapseError(400, err_msg, Codes.BAD_JSON)
+        else:
+            return True
+
+    def _check_json_template(self, content, template):
+        """Check content and template matches.
+
+        If the template is a dict, each key in the dict will be validated with
+        the content, else it will just compare the types of content and
+        template. This basic type check is required because this function will
+        be recursively called and could be called with just strs or ints.
+
+        Args:
+            content: The content to validate.
+            template: The validation template.
+        Returns:
+            str: An error message if the validation fails, else None.
+        """
+        if type(content) != type(template):
+            return "Mismatched types: %s" % template
+
+        if type(template) == dict:
+            for key in template:
+                if key not in content:
+                    return "Missing %s key" % key
+
+                if type(content[key]) != type(template[key]):
+                    return "Key %s is of the wrong type (got %s, want %s)" % (
+                        key, type(content[key]), type(template[key]))
+
+                if type(content[key]) == dict:
+                    # we must go deeper
+                    msg = self._check_json_template(
+                        content[key],
+                        template[key]
+                    )
+                    if msg:
+                        return msg
+                elif type(content[key]) == list:
+                    # make sure each item type in content matches the template
+                    for entry in content[key]:
+                        msg = self._check_json_template(
+                            entry,
+                            template[key][0]
+                        )
+                        if msg:
+                            return msg
\ No newline at end of file