Go back to storing JSON in TEXT

11 files changed, 20 insertions, 41 deletions

diff --git a/synapse/storage/_base.py b/synapse/storage/_base.py
index 5ec1d2613e..f5952d1fc0 100644
--- a/synapse/storage/_base.py
+++ b/synapse/storage/_base.py
@@ -818,9 +818,6 @@ class SQLBaseStore(object):
 
         internal_metadata, js, redacted, rejected_reason = res
 
-        internal_metadata = self.database_engine.load_unicode(internal_metadata)
-        js = self.database_engine.load_unicode(js)
-
         start_time = update_counter("select_event", start_time)
 
         result = self._get_event_from_row_txn(
diff --git a/synapse/storage/appservice.py b/synapse/storage/appservice.py
index 40e05b3635..63d1af4e86 100644
--- a/synapse/storage/appservice.py
+++ b/synapse/storage/appservice.py
@@ -366,9 +366,7 @@ class ApplicationServiceTransactionStore(SQLBaseStore):
         new_txn_id = max(highest_txn_id, last_txn_id) + 1
 
         # Insert new txn into txn table
-        event_ids = buffer(
-            json.dumps([e.event_id for e in events]).encode("utf8")
-        )
+        event_ids = json.dumps([e.event_id for e in events])
         txn.execute(
             "INSERT INTO application_services_txns(as_id, txn_id, event_ids) "
             "VALUES(?,?,?)",
diff --git a/synapse/storage/engines/postgres.py b/synapse/storage/engines/postgres.py
index 457c1f70a5..6f75245fa7 100644
--- a/synapse/storage/engines/postgres.py
+++ b/synapse/storage/engines/postgres.py
@@ -39,6 +39,3 @@ class PostgresEngine(object):
         if isinstance(error, self.module.DatabaseError):
             return error.pgcode in ["40001", "40P01"]
         return False
-
-    def load_unicode(self, v):
-        return bytes(v).decode("UTF8")
diff --git a/synapse/storage/engines/sqlite3.py b/synapse/storage/engines/sqlite3.py
index 389df35eb5..dd0d8e0e0f 100644
--- a/synapse/storage/engines/sqlite3.py
+++ b/synapse/storage/engines/sqlite3.py
@@ -37,8 +37,3 @@ class Sqlite3Engine(object):
 
     def is_deadlock(self, error):
         return False
-
-    def load_unicode(self, v):
-        if isinstance(v, types.UnicodeType):
-            return v
-        return bytes(v).decode("UTF8")
diff --git a/synapse/storage/events.py b/synapse/storage/events.py
index 0373d152b2..7dbf7a396a 100644
--- a/synapse/storage/events.py
+++ b/synapse/storage/events.py
@@ -168,7 +168,7 @@ class EventsStore(SQLBaseStore):
 
         metadata_json = encode_canonical_json(
             event.internal_metadata.get_dict()
-        )
+        ).decode("UTF-8")
 
         # If we have already persisted this event, we don't need to do any
         # more processing.
@@ -184,7 +184,7 @@ class EventsStore(SQLBaseStore):
                 )
                 txn.execute(
                     sql,
-                    (buffer(metadata_json), event.event_id,)
+                    (metadata_json, event.event_id,)
                 )
 
                 sql = (
@@ -229,14 +229,14 @@ class EventsStore(SQLBaseStore):
             values={
                 "event_id": event.event_id,
                 "room_id": event.room_id,
-                "internal_metadata": buffer(metadata_json),
-                "json": buffer(encode_canonical_json(event_dict)),
+                "internal_metadata": metadata_json,
+                "json": encode_canonical_json(event_dict).decode("UTF-8"),
             },
         )
 
-        content = buffer(encode_canonical_json(
+        content = encode_canonical_json(
             event.content
-        ))
+        ).decode("UTF-8")
 
         vals = {
             "topological_ordering": event.depth,
@@ -261,9 +261,9 @@ class EventsStore(SQLBaseStore):
             ]
         }
 
-        vals["unrecognized_keys"] = buffer(encode_canonical_json(
+        vals["unrecognized_keys"] = encode_canonical_json(
             unrec
-        ))
+        ).decode("UTF-8")
 
         sql = (
             "INSERT INTO events"
diff --git a/synapse/storage/profile.py b/synapse/storage/profile.py
index e33963d0b4..047698aa13 100644
--- a/synapse/storage/profile.py
+++ b/synapse/storage/profile.py
@@ -35,16 +35,13 @@ class ProfileStore(SQLBaseStore):
             desc="get_profile_displayname",
         )
 
-        if name:
-            name = self.database_engine.load_unicode(name)
-
         defer.returnValue(name)
 
     def set_profile_displayname(self, user_localpart, new_displayname):
         return self._simple_update_one(
             table="profiles",
             keyvalues={"user_id": user_localpart},
-            updatevalues={"displayname": new_displayname.encode("utf8")},
+            updatevalues={"displayname": new_displayname},
             desc="set_profile_displayname",
         )
 
diff --git a/synapse/storage/registration.py b/synapse/storage/registration.py
index 8a63fe4691..2a5c5080e4 100644
--- a/synapse/storage/registration.py
+++ b/synapse/storage/registration.py
@@ -98,11 +98,6 @@ class RegistrationStore(SQLBaseStore):
             allow_none=True,
         )
 
-        if user_info:
-            user_info["password_hash"] = self.database_engine.load_unicode(
-                user_info["password_hash"]
-            )
-
         defer.returnValue(user_info)
 
     @cached()
diff --git a/synapse/storage/schema/delta/15/appservice_txns.sql b/synapse/storage/schema/delta/15/appservice_txns.sql
index ddea8fc693..1c3324f415 100644
--- a/synapse/storage/schema/delta/15/appservice_txns.sql
+++ b/synapse/storage/schema/delta/15/appservice_txns.sql
@@ -22,7 +22,7 @@ CREATE TABLE IF NOT EXISTS application_services_state(
 CREATE TABLE IF NOT EXISTS application_services_txns(
     as_id VARCHAR(150) NOT NULL,
     txn_id INTEGER NOT NULL,
-    event_ids LONGBLOB NOT NULL,
+    event_ids TEXT NOT NULL,
     UNIQUE(as_id, txn_id)
 );
 
diff --git a/synapse/storage/schema/full_schemas/11/im.sql b/synapse/storage/schema/full_schemas/11/im.sql
index e9e09214d7..addbec5885 100644
--- a/synapse/storage/schema/full_schemas/11/im.sql
+++ b/synapse/storage/schema/full_schemas/11/im.sql
@@ -19,8 +19,8 @@ CREATE TABLE IF NOT EXISTS events(
     event_id VARCHAR(150) NOT NULL,
     type VARCHAR(150) NOT NULL,
     room_id VARCHAR(150) NOT NULL,
-    content bytea NOT NULL,
-    unrecognized_keys bytea,
+    content TEXT NOT NULL,
+    unrecognized_keys TEXT,
     processed BOOL NOT NULL,
     outlier BOOL NOT NULL,
     depth BIGINT DEFAULT 0 NOT NULL,
@@ -35,8 +35,8 @@ CREATE INDEX events_room_id ON events (room_id);
 CREATE TABLE IF NOT EXISTS event_json(
     event_id VARCHAR(150) NOT NULL,
     room_id VARCHAR(150) NOT NULL,
-    internal_metadata bytea NOT NULL,
-    json bytea NOT NULL,
+    internal_metadata TEXT NOT NULL,
+    json TEXT NOT NULL,
     UNIQUE (event_id)
 );
 
diff --git a/synapse/storage/schema/full_schemas/16/application_services.sql b/synapse/storage/schema/full_schemas/16/application_services.sql
index f08c5bcf76..5d63d57d59 100644
--- a/synapse/storage/schema/full_schemas/16/application_services.sql
+++ b/synapse/storage/schema/full_schemas/16/application_services.sql
@@ -39,7 +39,7 @@ CREATE TABLE IF NOT EXISTS application_services_state(
 CREATE TABLE IF NOT EXISTS application_services_txns(
     as_id VARCHAR(150) NOT NULL,
     txn_id INTEGER NOT NULL,
-    event_ids bytea NOT NULL,
+    event_ids TEXT NOT NULL,
     UNIQUE(as_id, txn_id)
 );
 
diff --git a/synapse/storage/schema/full_schemas/16/im.sql b/synapse/storage/schema/full_schemas/16/im.sql
index 17e4c949b9..5b4b494484 100644
--- a/synapse/storage/schema/full_schemas/16/im.sql
+++ b/synapse/storage/schema/full_schemas/16/im.sql
@@ -19,8 +19,8 @@ CREATE TABLE IF NOT EXISTS events(
     event_id VARCHAR(150) NOT NULL,
     type VARCHAR(150) NOT NULL,
     room_id VARCHAR(150) NOT NULL,
-    content bytea NOT NULL,
-    unrecognized_keys bytea,
+    content TEXT NOT NULL,
+    unrecognized_keys TEXT,
     processed BOOL NOT NULL,
     outlier BOOL NOT NULL,
     depth BIGINT DEFAULT 0 NOT NULL,
@@ -39,8 +39,8 @@ CREATE INDEX events_order_room ON events (
 CREATE TABLE IF NOT EXISTS event_json(
     event_id VARCHAR(150) NOT NULL,
     room_id VARCHAR(150) NOT NULL,
-    internal_metadata bytea NOT NULL,
-    json bytea NOT NULL,
+    internal_metadata TEXT NOT NULL,
+    json TEXT NOT NULL,
     UNIQUE (event_id)
 );