From c8dd3314d673fce90a53520475cdb19d5358dd34 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Mon, 15 Dec 2014 13:55:22 +0000 Subject: Fix bug where we ignored event_edge_hashes table --- scripts/check_event_hash.py | 3 +++ 1 file changed, 3 insertions(+) (limited to 'scripts') diff --git a/scripts/check_event_hash.py b/scripts/check_event_hash.py index 7c32f8102a..679afbd268 100644 --- a/scripts/check_event_hash.py +++ b/scripts/check_event_hash.py @@ -18,6 +18,9 @@ class dictobj(dict): def get_full_dict(self): return dict(self) + def get_pdu_json(self): + return dict(self) + def main(): parser = argparse.ArgumentParser() -- cgit 1.5.1 From 616f88027cbd6f75af987215ad03074caf9fc05a Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Mon, 15 Dec 2014 13:55:41 +0000 Subject: Add beginnings of upgrade script --- scripts/upgrade_db_to_v0.5.5.py | 223 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 223 insertions(+) create mode 100644 scripts/upgrade_db_to_v0.5.5.py (limited to 'scripts') diff --git a/scripts/upgrade_db_to_v0.5.5.py b/scripts/upgrade_db_to_v0.5.5.py new file mode 100644 index 0000000000..5898341d6e --- /dev/null +++ b/scripts/upgrade_db_to_v0.5.5.py @@ -0,0 +1,223 @@ +from synapse.storage._base import SQLBaseStore +from synapse.storage.signatures import SignatureStore +from synapse.storage.event_federation import EventFederationStore + +from syutil.base64util import encode_base64, decode_base64 + +from synapse.events import FrozenEvent +from synapse.events.builder import EventBuilder +from synapse.events.utils import prune_event + +from synapse.crypto.event_signing import check_event_content_hash + +from syutil.crypto.jsonsign import verify_signed_json, SignatureVerifyException +from syutil.crypto.signing_key import ( + decode_verify_key_bytes, write_signing_keys +) + +import dns.resolver +import hashlib +import json +import sqlite3 +import sys +import urllib2 + + +class Store(object): + _get_event_signatures_txn = SignatureStore.__dict__["_get_event_signatures_txn"] + _get_event_content_hashes_txn = SignatureStore.__dict__["_get_event_content_hashes_txn"] + _get_event_reference_hashes_txn = SignatureStore.__dict__["_get_event_reference_hashes_txn"] + _get_prev_event_hashes_txn = SignatureStore.__dict__["_get_prev_event_hashes_txn"] + _get_prev_events_and_state = EventFederationStore.__dict__["_get_prev_events_and_state"] + _get_auth_events = EventFederationStore.__dict__["_get_auth_events"] + cursor_to_dict = SQLBaseStore.__dict__["cursor_to_dict"] + _simple_select_onecol_txn = SQLBaseStore.__dict__["_simple_select_onecol_txn"] + _simple_select_list_txn = SQLBaseStore.__dict__["_simple_select_list_txn"] + + def _generate_event_json(self, txn, rows): + sql = "SELECT * FROM events WHERE event_id = ? ORDER BY rowid asc" + + events = [] + for row in rows: + d = dict(row) + + d.pop("stream_ordering", None) + d.pop("topological_ordering", None) + d.pop("processed", None) + + if "origin_server_ts" not in d: + d["origin_server_ts"] = d.pop("ts", 0) + else: + d.pop("ts", 0) + + d.pop("prev_state", None) + d.update(json.loads(d.pop("unrecognized_keys"))) + + d["sender"] = d.pop("user_id") + + d["content"] = json.loads(d["content"]) + + if "age_ts" not in d: + # For compatibility + d["age_ts"] = d.get("origin_server_ts", 0) + + d.setdefault("unsigned", {})["age_ts"] = d.pop("age_ts") + + d.pop("outlier", None) + + # d.pop("membership", None) + + d.pop("state_hash", None) + + d.pop("replaces_state", None) + + events.append(EventBuilder(d)) + + for i, ev in enumerate(events): + signatures = self._get_event_signatures_txn( + txn, ev.event_id, + ) + + ev.signatures = { + n: { + k: encode_base64(v) for k, v in s.items() + } + for n, s in signatures.items() + } + + hashes = self._get_event_content_hashes_txn( + txn, ev.event_id, + ) + + ev.hashes = { + k: encode_base64(v) for k, v in hashes.items() + } + + prevs = self._get_prev_events_and_state(txn, ev.event_id) + + ev.prev_events = [ + (e_id, h) + for e_id, h, is_state in prevs + if is_state == 0 + ] + + # ev.auth_events = self._get_auth_events(txn, ev.event_id) + + hashes = dict(ev.auth_events) + + for e_id, hash in ev.prev_events: + if e_id in hashes and not hash: + hash.update(hashes[e_id]) + # + # if hasattr(ev, "state_key"): + # ev.prev_state = [ + # (e_id, h) + # for e_id, h, is_state in prevs + # if is_state == 1 + # ] + + return [e.build() for e in events] + + +store = Store() + + +def get_key(server_name): + print "Getting keys for: %s" % (server_name,) + targets = [] + if ":" in server_name: + target, port = server_name.split(":") + targets.append((target, int(port))) + return + try: + answers = dns.resolver.query("_matrix._tcp." + server_name, "SRV") + for srv in answers: + targets.append((srv.target, srv.port)) + except dns.resolver.NXDOMAIN: + targets.append((server_name, 8448)) + except: + print "Failed to lookup keys for %s" % (server_name,) + return {} + + for target, port in targets: + url = "https://%s:%i/_matrix/key/v1" % (target, port) + try: + keys = json.load(urllib2.urlopen(url, timeout=2)) + verify_keys = {} + for key_id, key_base64 in keys["verify_keys"].items(): + verify_key = decode_verify_key_bytes(key_id, decode_base64(key_base64)) + verify_signed_json(keys, server_name, verify_key) + verify_keys[key_id] = verify_key + print "Got keys for: %s" % (server_name,) + return verify_keys + except urllib2.URLError: + pass + + print "Failed to get keys for %s" % (server_name,) + return {} + + +def get_events(cursor): + # cursor.execute( + # "SELECT * FROM events WHERE event_id = ? ORDER BY rowid DESC", + # ("$14182049031533SMfTT:matrix.org",) + # ) + + # cursor.execute( + # "SELECT * FROM events ORDER BY rowid DESC LIMIT 10000" + # ) + + cursor.execute( + "SELECT * FROM events ORDER BY rowid DESC" + ) + + rows = store.cursor_to_dict(cursor) + + events = store._generate_event_json(cursor, rows) + + print "Got events from DB." + + algorithms = { + "sha256": hashlib.sha256, + } + + server_keys = {} + + for event in events: + for alg_name in event.hashes: + if check_event_content_hash(event, algorithms[alg_name]): + # print "PASS content hash %s" % (alg_name,) + pass + else: + pass + print "FAIL content hash %s %s" % (alg_name, event.event_id, ) + # print "%s %d" % (event.event_id, event.origin_server_ts) + # print json.dumps(event.get_pdu_json(), indent=4, sort_keys=True) + + for host, sigs in event.signatures.items(): + pruned = prune_event(event) + + for key_id in sigs: + if host not in server_keys: + server_keys[host] = get_key(host) + if key_id in server_keys[host]: + try: + verify_signed_json( + pruned.get_pdu_json(), + host, + server_keys[host][key_id] + ) + except SignatureVerifyException as e: + # print e + print "FAIL signature check %s %s" % (key_id, event.event_id) + # print json.dumps(pruned.get_pdu_json(), indent=4, sort_keys=True) + +def main(): + conn = sqlite3.connect(sys.argv[1]) + cursor = conn.cursor() + get_events(cursor) + conn.commit() + + +if __name__ == "__main__": + main() \ No newline at end of file -- cgit 1.5.1 From b75adaedcaa0f153557557217844cd06f92635ec Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Mon, 15 Dec 2014 16:14:34 +0000 Subject: Finish up upgrade script --- scripts/upgrade_db_to_v0.5.5.py | 124 ++++++++++++++++++++++++++++++---------- synapse/events/__init__.py | 2 +- 2 files changed, 95 insertions(+), 31 deletions(-) (limited to 'scripts') diff --git a/scripts/upgrade_db_to_v0.5.5.py b/scripts/upgrade_db_to_v0.5.5.py index 5898341d6e..be9d07b2df 100644 --- a/scripts/upgrade_db_to_v0.5.5.py +++ b/scripts/upgrade_db_to_v0.5.5.py @@ -4,25 +4,42 @@ from synapse.storage.event_federation import EventFederationStore from syutil.base64util import encode_base64, decode_base64 -from synapse.events import FrozenEvent +from synapse.crypto.event_signing import compute_event_signature + from synapse.events.builder import EventBuilder from synapse.events.utils import prune_event from synapse.crypto.event_signing import check_event_content_hash -from syutil.crypto.jsonsign import verify_signed_json, SignatureVerifyException -from syutil.crypto.signing_key import ( - decode_verify_key_bytes, write_signing_keys +from syutil.crypto.jsonsign import ( + verify_signed_json, SignatureVerifyException, ) +from syutil.crypto.signing_key import decode_verify_key_bytes + +from syutil.jsonutil import encode_canonical_json +import argparse import dns.resolver import hashlib import json import sqlite3 -import sys +import syutil import urllib2 +delta_sql = """ +CREATE TABLE IF NOT EXISTS event_json( + event_id TEXT NOT NULL, + room_id TEXT NOT NULL, + json BLOB NOT NULL, + CONSTRAINT ev_j_uniq UNIQUE (event_id) +); + +CREATE INDEX IF NOT EXISTS event_json_id ON event_json(event_id); +CREATE INDEX IF NOT EXISTS event_json_room_id ON event_json(room_id); +""" + + class Store(object): _get_event_signatures_txn = SignatureStore.__dict__["_get_event_signatures_txn"] _get_event_content_hashes_txn = SignatureStore.__dict__["_get_event_content_hashes_txn"] @@ -33,10 +50,9 @@ class Store(object): cursor_to_dict = SQLBaseStore.__dict__["cursor_to_dict"] _simple_select_onecol_txn = SQLBaseStore.__dict__["_simple_select_onecol_txn"] _simple_select_list_txn = SQLBaseStore.__dict__["_simple_select_list_txn"] + _simple_insert_txn = SQLBaseStore.__dict__["_simple_insert_txn"] def _generate_event_json(self, txn, rows): - sql = "SELECT * FROM events WHERE event_id = ? ORDER BY rowid asc" - events = [] for row in rows: d = dict(row) @@ -145,7 +161,9 @@ def get_key(server_name): keys = json.load(urllib2.urlopen(url, timeout=2)) verify_keys = {} for key_id, key_base64 in keys["verify_keys"].items(): - verify_key = decode_verify_key_bytes(key_id, decode_base64(key_base64)) + verify_key = decode_verify_key_bytes( + key_id, decode_base64(key_base64) + ) verify_signed_json(keys, server_name, verify_key) verify_keys[key_id] = verify_key print "Got keys for: %s" % (server_name,) @@ -157,18 +175,11 @@ def get_key(server_name): return {} -def get_events(cursor): - # cursor.execute( - # "SELECT * FROM events WHERE event_id = ? ORDER BY rowid DESC", - # ("$14182049031533SMfTT:matrix.org",) - # ) - - # cursor.execute( - # "SELECT * FROM events ORDER BY rowid DESC LIMIT 10000" - # ) +def reinsert_events(cursor, server_name, signing_key): + cursor.executescript(delta_sql) cursor.execute( - "SELECT * FROM events ORDER BY rowid DESC" + "SELECT * FROM events ORDER BY rowid ASC" ) rows = store.cursor_to_dict(cursor) @@ -181,19 +192,26 @@ def get_events(cursor): "sha256": hashlib.sha256, } - server_keys = {} + key_id = "%s:%s" % (signing_key.alg, signing_key.version) + verify_key = signing_key.verify_key + verify_key.alg = signing_key.alg + verify_key.version = signing_key.version + + server_keys = { + server_name: { + key_id: verify_key + } + } for event in events: for alg_name in event.hashes: if check_event_content_hash(event, algorithms[alg_name]): - # print "PASS content hash %s" % (alg_name,) pass else: pass print "FAIL content hash %s %s" % (alg_name, event.event_id, ) - # print "%s %d" % (event.event_id, event.origin_server_ts) - # print json.dumps(event.get_pdu_json(), indent=4, sort_keys=True) + have_own_correctly_signed = False for host, sigs in event.signatures.items(): pruned = prune_event(event) @@ -207,17 +225,63 @@ def get_events(cursor): host, server_keys[host][key_id] ) - except SignatureVerifyException as e: - # print e - print "FAIL signature check %s %s" % (key_id, event.event_id) - # print json.dumps(pruned.get_pdu_json(), indent=4, sort_keys=True) -def main(): - conn = sqlite3.connect(sys.argv[1]) + if host == server_name: + have_own_correctly_signed = True + except SignatureVerifyException: + print "FAIL signature check %s %s" % ( + key_id, event.event_id + ) + + # TODO: Re sign with our own server key + if not have_own_correctly_signed: + sigs = compute_event_signature(event, server_name, signing_key) + event.signatures.update(sigs) + + pruned = prune_event(event) + + for key_id in event.signatures[server_name]: + verify_signed_json( + pruned.get_pdu_json(), + server_name, + server_keys[server_name][key_id] + ) + + event_json = encode_canonical_json( + event.get_dict() + ).decode("UTF-8") + + store._simple_insert_txn( + cursor, + table="event_json", + values={ + "event_id": event.event_id, + "room_id": event.room_id, + "json": event_json, + }, + or_replace=True, + ) + + +def main(database, server_name, signing_key): + conn = sqlite3.connect(database) cursor = conn.cursor() - get_events(cursor) + reinsert_events(cursor, server_name, signing_key) conn.commit() if __name__ == "__main__": - main() \ No newline at end of file + parser = argparse.ArgumentParser() + + parser.add_argument("database") + parser.add_argument("server_name") + parser.add_argument( + "signing_key", type=argparse.FileType('r'), + ) + args = parser.parse_args() + + signing_key = syutil.crypto.signing_key.read_signing_keys( + args.signing_key + ) + + main(args.database, args.server_name, signing_key[0]) diff --git a/synapse/events/__init__.py b/synapse/events/__init__.py index 98d7f0e324..d9dfe5e3f3 100644 --- a/synapse/events/__init__.py +++ b/synapse/events/__init__.py @@ -175,4 +175,4 @@ class FrozenEvent(EventBase): def __repr__(self): return "" % ( self.event_id, self.type, self.get("state_key", None), - ) \ No newline at end of file + ) -- cgit 1.5.1 From 35f4f6b07019a9815fb06529412c35a65c3cd285 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 16 Dec 2014 13:27:53 +0000 Subject: Update upgrade script --- scripts/upgrade_db_to_v0.5.5.py | 13 +++++++++++-- synapse/storage/__init__.py | 2 +- 2 files changed, 12 insertions(+), 3 deletions(-) (limited to 'scripts') diff --git a/scripts/upgrade_db_to_v0.5.5.py b/scripts/upgrade_db_to_v0.5.5.py index be9d07b2df..aa0d7667ac 100644 --- a/scripts/upgrade_db_to_v0.5.5.py +++ b/scripts/upgrade_db_to_v0.5.5.py @@ -31,6 +31,7 @@ delta_sql = """ CREATE TABLE IF NOT EXISTS event_json( event_id TEXT NOT NULL, room_id TEXT NOT NULL, + internal_metadata NOT NULL, json BLOB NOT NULL, CONSTRAINT ev_j_uniq UNIQUE (event_id) ); @@ -79,7 +80,7 @@ class Store(object): d.setdefault("unsigned", {})["age_ts"] = d.pop("age_ts") - d.pop("outlier", None) + outlier = d.pop("outlier", False) # d.pop("membership", None) @@ -87,7 +88,10 @@ class Store(object): d.pop("replaces_state", None) - events.append(EventBuilder(d)) + b = EventBuilder(d) + b.internal_metadata.outlier = outlier + + events.append(b) for i, ev in enumerate(events): signatures = self._get_event_signatures_txn( @@ -251,12 +255,17 @@ def reinsert_events(cursor, server_name, signing_key): event.get_dict() ).decode("UTF-8") + metadata_json = encode_canonical_json( + event.internal_metadata.get_dict() + ).decode("UTF-8") + store._simple_insert_txn( cursor, table="event_json", values={ "event_id": event.event_id, "room_id": event.room_id, + "internal_metadata": metadata_json, "json": event_json, }, or_replace=True, diff --git a/synapse/storage/__init__.py b/synapse/storage/__init__.py index 26f205ae8f..cc1dcc2e74 100644 --- a/synapse/storage/__init__.py +++ b/synapse/storage/__init__.py @@ -156,7 +156,7 @@ class DataStore(RoomMemberStore, RoomStore, ] } - metadata_json = encode_canonical_json( + metadata_json = encode_canonical_json( event.internal_metadata.get_dict() ) -- cgit 1.5.1