Merge pull request #66 from matrix-org/use-simplejson

Use simplejson

26 files changed, 43 insertions, 35 deletions

diff --git a/synapse/crypto/keyclient.py b/synapse/crypto/keyclient.py
index cd12349f67..74008347c3 100644
--- a/synapse/crypto/keyclient.py
+++ b/synapse/crypto/keyclient.py
@@ -19,7 +19,7 @@ from twisted.internet.protocol import Factory
 from twisted.internet import defer, reactor
 from synapse.http.endpoint import matrix_federation_endpoint
 from synapse.util.logcontext import PreserveLoggingContext
-import json
+import simplejson as json
 import logging
 
 
diff --git a/synapse/events/__init__.py b/synapse/events/__init__.py
index 8f0c6e959f..64e08223b0 100644
--- a/synapse/events/__init__.py
+++ b/synapse/events/__init__.py
@@ -13,7 +13,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-from synapse.util.frozenutils import freeze, unfreeze
+from synapse.util.frozenutils import freeze
 
 
 class _EventInternalMetadata(object):
@@ -140,10 +140,6 @@ class FrozenEvent(EventBase):
 
         return e
 
-    def get_dict(self):
-        # We need to unfreeze what we return
-        return unfreeze(super(FrozenEvent, self).get_dict())
-
     def __str__(self):
         return self.__repr__()
 
diff --git a/synapse/federation/persistence.py b/synapse/federation/persistence.py
index 85c82a4623..76a9dcd777 100644
--- a/synapse/federation/persistence.py
+++ b/synapse/federation/persistence.py
@@ -23,7 +23,8 @@ from twisted.internet import defer
 
 from synapse.util.logutils import log_function
 
-import json
+from syutil.jsonutil import encode_canonical_json
+
 import logging
 
 
@@ -70,7 +71,7 @@ class TransactionActions(object):
             transaction.transaction_id,
             transaction.origin,
             code,
-            json.dumps(response)
+            encode_canonical_json(response)
         )
 
     @defer.inlineCallbacks
@@ -100,5 +101,5 @@ class TransactionActions(object):
             transaction.transaction_id,
             transaction.destination,
             response_code,
-            json.dumps(response_dict)
+            encode_canonical_json(response_dict)
         )
diff --git a/synapse/federation/transport/server.py b/synapse/federation/transport/server.py
index 9c9f8d525b..2ffb37aa18 100644
--- a/synapse/federation/transport/server.py
+++ b/synapse/federation/transport/server.py
@@ -20,7 +20,7 @@ from synapse.api.errors import Codes, SynapseError
 from synapse.util.logutils import log_function
 
 import logging
-import json
+import simplejson as json
 import re
 
 
diff --git a/synapse/handlers/federation.py b/synapse/handlers/federation.py
index 0f9c82fd06..77c81fe2da 100644
--- a/synapse/handlers/federation.py
+++ b/synapse/handlers/federation.py
@@ -23,6 +23,7 @@ from synapse.api.errors import (
 from synapse.api.constants import EventTypes, Membership, RejectedReason
 from synapse.util.logutils import log_function
 from synapse.util.async import run_on_reactor
+from synapse.util.frozenutils import unfreeze
 from synapse.crypto.event_signing import (
     compute_event_signature, add_hashes_and_signatures,
 )
@@ -311,7 +312,7 @@ class FederationHandler(BaseHandler):
         self.room_queues[room_id] = []
 
         builder = self.event_builder_factory.new(
-            event.get_pdu_json()
+            unfreeze(event.get_pdu_json())
         )
 
         handled_events = set()
diff --git a/synapse/handlers/room.py b/synapse/handlers/room.py
index 0369b907a5..914742d913 100644
--- a/synapse/handlers/room.py
+++ b/synapse/handlers/room.py
@@ -462,7 +462,7 @@ class RoomMemberHandler(BaseHandler):
                 room_hosts,
                 room_id,
                 event.user_id,
-                event.get_dict()["content"],  # FIXME To get a non-frozen dict
+                event.content,  # FIXME To get a non-frozen dict
                 context
             )
         else:
diff --git a/synapse/http/client.py b/synapse/http/client.py
index 198f575cfa..d500e19c81 100644
--- a/synapse/http/client.py
+++ b/synapse/http/client.py
@@ -23,7 +23,7 @@ from twisted.web.http_headers import Headers
 
 from StringIO import StringIO
 
-import json
+import simplejson as json
 import logging
 import urllib
 
diff --git a/synapse/http/matrixfederationclient.py b/synapse/http/matrixfederationclient.py
index 056d446e42..406203acf2 100644
--- a/synapse/http/matrixfederationclient.py
+++ b/synapse/http/matrixfederationclient.py
@@ -33,7 +33,7 @@ from synapse.api.errors import (
 
 from syutil.crypto.jsonsign import sign_json
 
-import json
+import simplejson as json
 import logging
 import urllib
 import urlparse
diff --git a/synapse/push/__init__.py b/synapse/push/__init__.py
index 418a348a58..0659a1cb9b 100644
--- a/synapse/push/__init__.py
+++ b/synapse/push/__init__.py
@@ -22,7 +22,7 @@ import synapse.util.async
 import baserules
 
 import logging
-import json
+import simplejson as json
 import re
 
 logger = logging.getLogger(__name__)
diff --git a/synapse/push/pusherpool.py b/synapse/push/pusherpool.py
index 5a525befd7..7483d257bf 100644
--- a/synapse/push/pusherpool.py
+++ b/synapse/push/pusherpool.py
@@ -20,7 +20,7 @@ from httppusher import HttpPusher
 from synapse.push import PusherConfigException
 
 import logging
-import json
+import simplejson as json
 
 logger = logging.getLogger(__name__)
 
diff --git a/synapse/python_dependencies.py b/synapse/python_dependencies.py
index c7a685e71e..8164effeee 100644
--- a/synapse/python_dependencies.py
+++ b/synapse/python_dependencies.py
@@ -4,7 +4,7 @@ from distutils.version import LooseVersion
 logger = logging.getLogger(__name__)
 
 REQUIREMENTS = {
-    "syutil>=0.0.2": ["syutil"],
+    "syutil>=0.0.3": ["syutil"],
     "matrix_angular_sdk>=0.6.1": ["syweb>=0.6.1"],
     "Twisted==14.0.2": ["twisted==14.0.2"],
     "service_identity>=1.0.0": ["service_identity>=1.0.0"],
diff --git a/synapse/rest/client/v1/directory.py b/synapse/rest/client/v1/directory.py
index 8ed7e2d669..420aa89f38 100644
--- a/synapse/rest/client/v1/directory.py
+++ b/synapse/rest/client/v1/directory.py
@@ -20,7 +20,7 @@ from synapse.api.errors import AuthError, SynapseError, Codes
 from synapse.types import RoomAlias
 from .base import ClientV1RestServlet, client_path_pattern
 
-import json
+import simplejson as json
 import logging
 
 
diff --git a/synapse/rest/client/v1/login.py b/synapse/rest/client/v1/login.py
index 7116ac98e8..b2257b749d 100644
--- a/synapse/rest/client/v1/login.py
+++ b/synapse/rest/client/v1/login.py
@@ -19,7 +19,7 @@ from synapse.api.errors import SynapseError
 from synapse.types import UserID
 from base import ClientV1RestServlet, client_path_pattern
 
-import json
+import simplejson as json
 
 
 class LoginRestServlet(ClientV1RestServlet):
diff --git a/synapse/rest/client/v1/presence.py b/synapse/rest/client/v1/presence.py
index 7feb4aadb1..78d4f2b128 100644
--- a/synapse/rest/client/v1/presence.py
+++ b/synapse/rest/client/v1/presence.py
@@ -21,7 +21,7 @@ from synapse.api.errors import SynapseError
 from synapse.types import UserID
 from .base import ClientV1RestServlet, client_path_pattern
 
-import json
+import simplejson as json
 import logging
 
 logger = logging.getLogger(__name__)
diff --git a/synapse/rest/client/v1/profile.py b/synapse/rest/client/v1/profile.py
index 15d6f3fc6c..1e77eb49cf 100644
--- a/synapse/rest/client/v1/profile.py
+++ b/synapse/rest/client/v1/profile.py
@@ -19,7 +19,7 @@ from twisted.internet import defer
 from .base import ClientV1RestServlet, client_path_pattern
 from synapse.types import UserID
 
-import json
+import simplejson as json
 
 
 class ProfileDisplaynameRestServlet(ClientV1RestServlet):
diff --git a/synapse/rest/client/v1/push_rule.py b/synapse/rest/client/v1/push_rule.py
index c4e7dfcf0e..b012f31084 100644
--- a/synapse/rest/client/v1/push_rule.py
+++ b/synapse/rest/client/v1/push_rule.py
@@ -27,7 +27,7 @@ from synapse.push.rulekinds import (
     PRIORITY_CLASS_MAP, PRIORITY_CLASS_INVERSE_MAP
 )
 
-import json
+import simplejson as json
 
 
 class PushRuleRestServlet(ClientV1RestServlet):
diff --git a/synapse/rest/client/v1/pusher.py b/synapse/rest/client/v1/pusher.py
index 80e9939b79..6045e86f34 100644
--- a/synapse/rest/client/v1/pusher.py
+++ b/synapse/rest/client/v1/pusher.py
@@ -19,7 +19,7 @@ from synapse.api.errors import SynapseError, Codes
 from synapse.push import PusherConfigException
 from .base import ClientV1RestServlet, client_path_pattern
 
-import json
+import simplejson as json
 
 
 class PusherRestServlet(ClientV1RestServlet):
diff --git a/synapse/rest/client/v1/register.py b/synapse/rest/client/v1/register.py
index c0423c2d45..d3399c446b 100644
--- a/synapse/rest/client/v1/register.py
+++ b/synapse/rest/client/v1/register.py
@@ -25,7 +25,7 @@ from synapse.util.async import run_on_reactor
 
 from hashlib import sha1
 import hmac
-import json
+import simplejson as json
 import logging
 import urllib
 
diff --git a/synapse/rest/client/v1/room.py b/synapse/rest/client/v1/room.py
index 410f19ccf6..0346afb1b4 100644
--- a/synapse/rest/client/v1/room.py
+++ b/synapse/rest/client/v1/room.py
@@ -23,7 +23,7 @@ from synapse.api.constants import EventTypes, Membership
 from synapse.types import UserID, RoomID, RoomAlias
 from synapse.events.utils import serialize_event
 
-import json
+import simplejson as json
 import logging
 import urllib
 
diff --git a/synapse/rest/client/v2_alpha/filter.py b/synapse/rest/client/v2_alpha/filter.py
index 6ddc495d23..703250cea8 100644
--- a/synapse/rest/client/v2_alpha/filter.py
+++ b/synapse/rest/client/v2_alpha/filter.py
@@ -21,7 +21,7 @@ from synapse.types import UserID
 
 from ._base import client_v2_pattern
 
-import json
+import simplejson as json
 import logging
 
 
diff --git a/synapse/rest/media/v0/content_repository.py b/synapse/rest/media/v0/content_repository.py
index 22e26e3cd5..e77a20fb2e 100644
--- a/synapse/rest/media/v0/content_repository.py
+++ b/synapse/rest/media/v0/content_repository.py
@@ -25,7 +25,7 @@ from twisted.web import server, resource
 from twisted.internet import defer
 
 import base64
-import json
+import simplejson as json
 import logging
 import os
 import re
diff --git a/synapse/storage/__init__.py b/synapse/storage/__init__.py
index a33e2298f6..02b1f06854 100644
--- a/synapse/storage/__init__.py
+++ b/synapse/storage/__init__.py
@@ -44,7 +44,6 @@ from syutil.jsonutil import encode_canonical_json
 from synapse.crypto.event_signing import compute_event_reference_hash
 
 
-import json
 import logging
 import os
 
@@ -301,12 +300,16 @@ class DataStore(RoomMemberStore, RoomStore,
             or_replace=True,
         )
 
+        content = encode_canonical_json(
+            event.content
+        ).decode("UTF-8")
+
         vals = {
             "topological_ordering": event.depth,
             "event_id": event.event_id,
             "type": event.type,
             "room_id": event.room_id,
-            "content": json.dumps(event.get_dict()["content"]),
+            "content": content,
             "processed": True,
             "outlier": outlier,
             "depth": event.depth,
@@ -326,7 +329,10 @@ class DataStore(RoomMemberStore, RoomStore,
                 "prev_events",
             ]
         }
-        vals["unrecognized_keys"] = json.dumps(unrec)
+
+        vals["unrecognized_keys"] = encode_canonical_json(
+            unrec
+        ).decode("UTF-8")
 
         try:
             self._simple_insert_txn(
diff --git a/synapse/storage/_base.py b/synapse/storage/_base.py
index 29fc334f45..be9934c66f 100644
--- a/synapse/storage/_base.py
+++ b/synapse/storage/_base.py
@@ -24,7 +24,7 @@ from synapse.util.lrucache import LruCache
 from twisted.internet import defer
 
 import collections
-import json
+import simplejson as json
 import sys
 import time
 
diff --git a/synapse/storage/filtering.py b/synapse/storage/filtering.py
index e86eeced45..457a11fd02 100644
--- a/synapse/storage/filtering.py
+++ b/synapse/storage/filtering.py
@@ -17,7 +17,7 @@ from twisted.internet import defer
 
 from ._base import SQLBaseStore
 
-import json
+import simplejson as json
 
 
 class FilteringStore(SQLBaseStore):
diff --git a/synapse/storage/push_rule.py b/synapse/storage/push_rule.py
index 620de71398..ae46b39cc1 100644
--- a/synapse/storage/push_rule.py
+++ b/synapse/storage/push_rule.py
@@ -20,7 +20,7 @@ from twisted.internet import defer
 
 import logging
 import copy
-import json
+import simplejson as json
 
 logger = logging.getLogger(__name__)
 
diff --git a/synapse/util/frozenutils.py b/synapse/util/frozenutils.py
index a13a2015e4..9e10d37aec 100644
--- a/synapse/util/frozenutils.py
+++ b/synapse/util/frozenutils.py
@@ -21,6 +21,9 @@ def freeze(o):
     if t is dict:
         return frozendict({k: freeze(v) for k, v in o.items()})
 
+    if t is frozendict:
+        return o
+
     if t is str or t is unicode:
         return o
 
@@ -33,10 +36,11 @@ def freeze(o):
 
 
 def unfreeze(o):
-    if isinstance(o, frozendict) or isinstance(o, dict):
+    t = type(o)
+    if t is dict or t is frozendict:
         return dict({k: unfreeze(v) for k, v in o.items()})
 
-    if isinstance(o, basestring):
+    if t is str or t is unicode:
         return o
 
     try: