diff options
| -rw-r--r-- | .gitignore | 3 | ||||
| -rw-r--r-- | docs/postgres.rst | 11 | ||||
| -rwxr-xr-x | scripts/database-prepare-for-0.0.1.sh | 21 | ||||
| -rwxr-xr-x | scripts/database-prepare-for-0.5.0.sh | 21 | ||||
| -rwxr-xr-x | scripts/synapse_port_db (renamed from scripts/port_from_sqlite_to_postgres.py) | 0 | ||||
| -rwxr-xr-x | scripts/upgrade_db_to_v0.6.0.py | 326 | ||||
| -rwxr-xr-x | synapse/app/homeserver.py | 4 | ||||
| -rw-r--r-- | synapse/config/_base.py | 2 | ||||
| -rw-r--r-- | synapse/handlers/auth.py | 27 | ||||
| -rw-r--r-- | synapse/handlers/register.py | 3 | ||||
| -rw-r--r-- | synapse/python_dependencies.py | 5 | ||||
| -rw-r--r-- | synapse/storage/_base.py | 2 | ||||
| -rw-r--r-- | tests/utils.py | 13 | ||||
| -rw-r--r-- | tox.ini | 2 |
14 files changed, 51 insertions, 389 deletions
diff --git a/.gitignore b/.gitignore index 4c336b710d..f8c4000134 100644 --- a/.gitignore +++ b/.gitignore @@ -43,3 +43,6 @@ build/ localhost-800*/ static/client/register/register_config.js .tox + +env/ +*.config diff --git a/docs/postgres.rst b/docs/postgres.rst index 19d8391115..b5027fefb0 100644 --- a/docs/postgres.rst +++ b/docs/postgres.rst @@ -55,9 +55,8 @@ Porting from SQLite Overview ~~~~~~~~ -The script ``port_from_sqlite_to_postgres.py`` allows porting an existing -synapse server backed by SQLite to using PostgreSQL. This is done in as a two -phase process: +The script ``synapse_port_db`` allows porting an existing synapse server +backed by SQLite to using PostgreSQL. This is done in as a two phase process: 1. Copy the existing SQLite database to a separate location (while the server is down) and running the port script against that offline database. @@ -86,8 +85,7 @@ Assuming your new config file (as described in the section *Synapse config*) is named ``homeserver-postgres.yaml`` and the SQLite snapshot is at ``homeserver.db.snapshot`` then simply run:: - python scripts/port_from_sqlite_to_postgres.py \ - --sqlite-database homeserver.db.snapshot \ + synapse_port_db --sqlite-database homeserver.db.snapshot \ --postgres-config homeserver-postgres.yaml The flag ``--curses`` displays a coloured curses progress UI. @@ -100,8 +98,7 @@ To complete the conversion shut down the synapse server and run the port script one last time, e.g. if the SQLite database is at ``homeserver.db`` run:: - python scripts/port_from_sqlite_to_postgres.py \ - --sqlite-database homeserver.db \ + synapse_port_db --sqlite-database homeserver.db \ --postgres-config database_config.yaml Once that has completed, change the synapse config to point at the PostgreSQL diff --git a/scripts/database-prepare-for-0.0.1.sh b/scripts/database-prepare-for-0.0.1.sh deleted file mode 100755 index 43d759a5cd..0000000000 --- a/scripts/database-prepare-for-0.0.1.sh +++ /dev/null @@ -1,21 +0,0 @@ -#!/bin/bash - -# This is will prepare a synapse database for running with v0.0.1 of synapse. -# It will store all the user information, but will *delete* all messages and -# room data. - -set -e - -cp "$1" "$1.bak" - -DUMP=$(sqlite3 "$1" << 'EOF' -.dump users -.dump access_tokens -.dump presence -.dump profiles -EOF -) - -rm "$1" - -sqlite3 "$1" <<< "$DUMP" diff --git a/scripts/database-prepare-for-0.5.0.sh b/scripts/database-prepare-for-0.5.0.sh deleted file mode 100755 index e824cb583e..0000000000 --- a/scripts/database-prepare-for-0.5.0.sh +++ /dev/null @@ -1,21 +0,0 @@ -#!/bin/bash - -# This is will prepare a synapse database for running with v0.5.0 of synapse. -# It will store all the user information, but will *delete* all messages and -# room data. - -set -e - -cp "$1" "$1.bak" - -DUMP=$(sqlite3 "$1" << 'EOF' -.dump users -.dump access_tokens -.dump presence -.dump profiles -EOF -) - -rm "$1" - -sqlite3 "$1" <<< "$DUMP" diff --git a/scripts/port_from_sqlite_to_postgres.py b/scripts/synapse_port_db index e7ed4c309b..e7ed4c309b 100755 --- a/scripts/port_from_sqlite_to_postgres.py +++ b/scripts/synapse_port_db diff --git a/scripts/upgrade_db_to_v0.6.0.py b/scripts/upgrade_db_to_v0.6.0.py deleted file mode 100755 index cd4be28b86..0000000000 --- a/scripts/upgrade_db_to_v0.6.0.py +++ /dev/null @@ -1,326 +0,0 @@ -#!/usr/bin/env python -from synapse.storage import SCHEMA_VERSION, read_schema -from synapse.storage._base import SQLBaseStore -from synapse.storage.signatures import SignatureStore -from synapse.storage.event_federation import EventFederationStore - -from unpaddedbase64 import encode_base64, decode_base64 - -from synapse.crypto.event_signing import compute_event_signature - -from synapse.events.builder import EventBuilder -from synapse.events.utils import prune_event - -from synapse.crypto.event_signing import check_event_content_hash - -from signedjson.sign import verify_signed_json, SignatureVerifyException -from signedjson.key import decode_verify_key_bytes - -from canonicaljson import encode_canonical_json - -import argparse -# import dns.resolver -import hashlib -import httplib -import json -import sqlite3 -import urllib2 - - -delta_sql = """ -CREATE TABLE IF NOT EXISTS event_json( - event_id TEXT NOT NULL, - room_id TEXT NOT NULL, - internal_metadata NOT NULL, - json BLOB NOT NULL, - CONSTRAINT ev_j_uniq UNIQUE (event_id) -); - -CREATE INDEX IF NOT EXISTS event_json_id ON event_json(event_id); -CREATE INDEX IF NOT EXISTS event_json_room_id ON event_json(room_id); - -PRAGMA user_version = 10; -""" - - -class Store(object): - _get_event_signatures_txn = SignatureStore.__dict__["_get_event_signatures_txn"] - _get_event_content_hashes_txn = SignatureStore.__dict__["_get_event_content_hashes_txn"] - _get_event_reference_hashes_txn = SignatureStore.__dict__["_get_event_reference_hashes_txn"] - _get_prev_event_hashes_txn = SignatureStore.__dict__["_get_prev_event_hashes_txn"] - _get_prev_events_and_state = EventFederationStore.__dict__["_get_prev_events_and_state"] - _get_auth_events = EventFederationStore.__dict__["_get_auth_events"] - cursor_to_dict = SQLBaseStore.__dict__["cursor_to_dict"] - _simple_select_onecol_txn = SQLBaseStore.__dict__["_simple_select_onecol_txn"] - _simple_select_list_txn = SQLBaseStore.__dict__["_simple_select_list_txn"] - _simple_insert_txn = SQLBaseStore.__dict__["_simple_insert_txn"] - - def _generate_event_json(self, txn, rows): - events = [] - for row in rows: - d = dict(row) - - d.pop("stream_ordering", None) - d.pop("topological_ordering", None) - d.pop("processed", None) - - if "origin_server_ts" not in d: - d["origin_server_ts"] = d.pop("ts", 0) - else: - d.pop("ts", 0) - - d.pop("prev_state", None) - d.update(json.loads(d.pop("unrecognized_keys"))) - - d["sender"] = d.pop("user_id") - - d["content"] = json.loads(d["content"]) - - if "age_ts" not in d: - # For compatibility - d["age_ts"] = d.get("origin_server_ts", 0) - - d.setdefault("unsigned", {})["age_ts"] = d.pop("age_ts") - - outlier = d.pop("outlier", False) - - # d.pop("membership", None) - - d.pop("state_hash", None) - - d.pop("replaces_state", None) - - b = EventBuilder(d) - b.internal_metadata.outlier = outlier - - events.append(b) - - for i, ev in enumerate(events): - signatures = self._get_event_signatures_txn( - txn, ev.event_id, - ) - - ev.signatures = { - n: { - k: encode_base64(v) for k, v in s.items() - } - for n, s in signatures.items() - } - - hashes = self._get_event_content_hashes_txn( - txn, ev.event_id, - ) - - ev.hashes = { - k: encode_base64(v) for k, v in hashes.items() - } - - prevs = self._get_prev_events_and_state(txn, ev.event_id) - - ev.prev_events = [ - (e_id, h) - for e_id, h, is_state in prevs - if is_state == 0 - ] - - # ev.auth_events = self._get_auth_events(txn, ev.event_id) - - hashes = dict(ev.auth_events) - - for e_id, hash in ev.prev_events: - if e_id in hashes and not hash: - hash.update(hashes[e_id]) - # - # if hasattr(ev, "state_key"): - # ev.prev_state = [ - # (e_id, h) - # for e_id, h, is_state in prevs - # if is_state == 1 - # ] - - return [e.build() for e in events] - - -store = Store() - - -# def get_key(server_name): -# print "Getting keys for: %s" % (server_name,) -# targets = [] -# if ":" in server_name: -# target, port = server_name.split(":") -# targets.append((target, int(port))) -# try: -# answers = dns.resolver.query("_matrix._tcp." + server_name, "SRV") -# for srv in answers: -# targets.append((srv.target, srv.port)) -# except dns.resolver.NXDOMAIN: -# targets.append((server_name, 8448)) -# except: -# print "Failed to lookup keys for %s" % (server_name,) -# return {} -# -# for target, port in targets: -# url = "https://%s:%i/_matrix/key/v1" % (target, port) -# try: -# keys = json.load(urllib2.urlopen(url, timeout=2)) -# verify_keys = {} -# for key_id, key_base64 in keys["verify_keys"].items(): -# verify_key = decode_verify_key_bytes( -# key_id, decode_base64(key_base64) -# ) -# verify_signed_json(keys, server_name, verify_key) -# verify_keys[key_id] = verify_key -# print "Got keys for: %s" % (server_name,) -# return verify_keys -# except urllib2.URLError: -# pass -# except urllib2.HTTPError: -# pass -# except httplib.HTTPException: -# pass -# -# print "Failed to get keys for %s" % (server_name,) -# return {} - - -def reinsert_events(cursor, server_name, signing_key): - print "Running delta: v10" - - cursor.executescript(delta_sql) - - cursor.execute( - "SELECT * FROM events ORDER BY rowid ASC" - ) - - print "Getting events..." - - rows = store.cursor_to_dict(cursor) - - events = store._generate_event_json(cursor, rows) - - print "Got events from DB." - - algorithms = { - "sha256": hashlib.sha256, - } - - key_id = "%s:%s" % (signing_key.alg, signing_key.version) - verify_key = signing_key.verify_key - verify_key.alg = signing_key.alg - verify_key.version = signing_key.version - - server_keys = { - server_name: { - key_id: verify_key - } - } - - i = 0 - N = len(events) - - for event in events: - if i % 100 == 0: - print "Processed: %d/%d events" % (i,N,) - i += 1 - - # for alg_name in event.hashes: - # if check_event_content_hash(event, algorithms[alg_name]): - # pass - # else: - # pass - # print "FAIL content hash %s %s" % (alg_name, event.event_id, ) - - have_own_correctly_signed = False - for host, sigs in event.signatures.items(): - pruned = prune_event(event) - - for key_id in sigs: - if host not in server_keys: - server_keys[host] = {} # get_key(host) - if key_id in server_keys[host]: - try: - verify_signed_json( - pruned.get_pdu_json(), - host, - server_keys[host][key_id] - ) - - if host == server_name: - have_own_correctly_signed = True - except SignatureVerifyException: - print "FAIL signature check %s %s" % ( - key_id, event.event_id - ) - - # TODO: Re sign with our own server key - if not have_own_correctly_signed: - sigs = compute_event_signature(event, server_name, signing_key) - event.signatures.update(sigs) - - pruned = prune_event(event) - - for key_id in event.signatures[server_name]: - verify_signed_json( - pruned.get_pdu_json(), - server_name, - server_keys[server_name][key_id] - ) - - event_json = encode_canonical_json( - event.get_dict() - ).decode("UTF-8") - - metadata_json = encode_canonical_json( - event.internal_metadata.get_dict() - ).decode("UTF-8") - - store._simple_insert_txn( - cursor, - table="event_json", - values={ - "event_id": event.event_id, - "room_id": event.room_id, - "internal_metadata": metadata_json, - "json": event_json, - }, - or_replace=True, - ) - - -def main(database, server_name, signing_key): - conn = sqlite3.connect(database) - cursor = conn.cursor() - - # Do other deltas: - cursor.execute("PRAGMA user_version") - row = cursor.fetchone() - - if row and row[0]: - user_version = row[0] - # Run every version since after the current version. - for v in range(user_version + 1, 10): - print "Running delta: %d" % (v,) - sql_script = read_schema("delta/v%d" % (v,)) - cursor.executescript(sql_script) - - reinsert_events(cursor, server_name, signing_key) - - conn.commit() - - print "Success!" - - -if __name__ == "__main__": - parser = argparse.ArgumentParser() - - parser.add_argument("database") - parser.add_argument("server_name") - parser.add_argument( - "signing_key", type=argparse.FileType('r'), - ) - args = parser.parse_args() - - signing_key = signedjson.key.read_signing_keys(args.signing_key) - - main(args.database, args.server_name, signing_key[0]) diff --git a/synapse/app/homeserver.py b/synapse/app/homeserver.py index ff7807c2e6..ffc6299146 100755 --- a/synapse/app/homeserver.py +++ b/synapse/app/homeserver.py @@ -403,7 +403,7 @@ def setup(config_options): database_engine=database_engine, ) - logger.info("Preparing database: %r...", config.database_config) + logger.info("Preparing database: %s...", config.database_config['name']) try: db_conn = database_engine.module.connect( @@ -425,7 +425,7 @@ def setup(config_options): ) sys.exit(1) - logger.info("Database prepared in %r.", config.database_config) + logger.info("Database prepared in %s.", config.database_config['name']) hs.start_listening() diff --git a/synapse/config/_base.py b/synapse/config/_base.py index d01235d31f..1a6784a714 100644 --- a/synapse/config/_base.py +++ b/synapse/config/_base.py @@ -182,6 +182,8 @@ class Config(object): ) % (entry_path, ) continue + files.add(config_path) + config_files.extend(sorted(files)) else: config_files.append(config_path) diff --git a/synapse/handlers/auth.py b/synapse/handlers/auth.py index 1ab19cd1a6..59f687e0f1 100644 --- a/synapse/handlers/auth.py +++ b/synapse/handlers/auth.py @@ -324,7 +324,7 @@ class AuthHandler(BaseHandler): def _check_password(self, user_id, password, stored_hash): """Checks that user_id has passed password, raises LoginError if not.""" - if not bcrypt.checkpw(password, stored_hash): + if not self.validate_hash(password, stored_hash): logger.warn("Failed password login for user %s", user_id) raise LoginError(403, "", errcode=Codes.FORBIDDEN) @@ -369,7 +369,7 @@ class AuthHandler(BaseHandler): @defer.inlineCallbacks def set_password(self, user_id, newpassword): - password_hash = bcrypt.hashpw(newpassword, bcrypt.gensalt()) + password_hash = self.hash(newpassword) yield self.store.user_set_password_hash(user_id, password_hash) yield self.store.user_delete_access_tokens(user_id) @@ -391,3 +391,26 @@ class AuthHandler(BaseHandler): def _remove_session(self, session): logger.debug("Removing session %s", session) del self.sessions[session["id"]] + + def hash(self, password): + """Computes a secure hash of password. + + Args: + password (str): Password to hash. + + Returns: + Hashed password (str). + """ + return bcrypt.hashpw(password, bcrypt.gensalt()) + + def validate_hash(self, password, stored_hash): + """Validates that self.hash(password) == stored_hash. + + Args: + password (str): Password to hash. + stored_hash (str): Expected hash value. + + Returns: + Whether self.hash(password) == stored_hash (bool). + """ + return bcrypt.checkpw(password, stored_hash) diff --git a/synapse/handlers/register.py b/synapse/handlers/register.py index 56d125f753..ef4081e3fe 100644 --- a/synapse/handlers/register.py +++ b/synapse/handlers/register.py @@ -25,7 +25,6 @@ import synapse.util.stringutils as stringutils from synapse.util.async import run_on_reactor from synapse.http.client import CaptchaServerHttpClient -import bcrypt import logging import urllib @@ -82,7 +81,7 @@ class RegistrationHandler(BaseHandler): yield run_on_reactor() password_hash = None if password: - password_hash = bcrypt.hashpw(password, bcrypt.gensalt()) + password_hash = self.auth_handler().hash(password) if localpart: yield self.check_username(localpart) diff --git a/synapse/python_dependencies.py b/synapse/python_dependencies.py index 8ec272fd5f..795ef27182 100644 --- a/synapse/python_dependencies.py +++ b/synapse/python_dependencies.py @@ -56,11 +56,6 @@ def github_link(project, version, egg): return "https://github.com/%s/tarball/%s/#egg=%s" % (project, version, egg) DEPENDENCY_LINKS = { - "matrix-angular-sdk": github_link( - project="matrix-org/matrix-angular-sdk", - version="v0.6.6", - egg="matrix_angular_sdk-0.6.6", - ), } diff --git a/synapse/storage/_base.py b/synapse/storage/_base.py index ce71389f02..495ef087c9 100644 --- a/synapse/storage/_base.py +++ b/synapse/storage/_base.py @@ -167,7 +167,7 @@ class SQLBaseStore(object): self._get_event_cache = Cache("*getEvent*", keylen=3, lru=True, max_entries=hs.config.event_cache_size) - self._state_group_cache = DictionaryCache("*stateGroupCache*", 100000) + self._state_group_cache = DictionaryCache("*stateGroupCache*", 2000) self._event_fetch_lock = threading.Condition() self._event_fetch_list = [] diff --git a/tests/utils.py b/tests/utils.py index 3766a994f2..dd19a16fc7 100644 --- a/tests/utils.py +++ b/tests/utils.py @@ -27,6 +27,7 @@ from twisted.enterprise.adbapi import ConnectionPool from collections import namedtuple from mock import patch, Mock +import hashlib import urllib import urlparse @@ -67,6 +68,18 @@ def setup_test_homeserver(name="test", datastore=None, config=None, **kargs): **kargs ) + # bcrypt is far too slow to be doing in unit tests + def swap_out_hash_for_testing(old_build_handlers): + def build_handlers(): + handlers = old_build_handlers() + auth_handler = handlers.auth_handler + auth_handler.hash = lambda p: hashlib.md5(p).hexdigest() + auth_handler.validate_hash = lambda p, h: hashlib.md5(p).hexdigest() == h + return handlers + return build_handlers + + hs.build_handlers = swap_out_hash_for_testing(hs.build_handlers) + defer.returnValue(hs) diff --git a/tox.ini b/tox.ini index 3f0649b628..a69948484f 100644 --- a/tox.ini +++ b/tox.ini @@ -11,8 +11,6 @@ setenv = commands = coverage run --source=synapse {envbindir}/trial {posargs:tests} coverage report -m -install_command = - pip install --process-dependency-links --pre {opts} {packages} [testenv:packaging] deps = |