From 96cc7c87405e87aacc2318f4634e1c6e5e2f46d3 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 16 Dec 2014 13:57:27 +0000 Subject: Bump version --- synapse/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'synapse') diff --git a/synapse/__init__.py b/synapse/__init__.py index 7e49e1fd08..1cef40925f 100644 --- a/synapse/__init__.py +++ b/synapse/__init__.py @@ -16,4 +16,4 @@ """ This is a reference implementation of a synapse home server. """ -__version__ = "0.5.4a" +__version__ = "0.6.0" -- cgit 1.4.1 From ef5a14105064070a794d83a6542bf01eb62e1512 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 16 Dec 2014 13:57:47 +0000 Subject: Bump database version --- synapse/storage/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'synapse') diff --git a/synapse/storage/__init__.py b/synapse/storage/__init__.py index cc1dcc2e74..62f89b8d8c 100644 --- a/synapse/storage/__init__.py +++ b/synapse/storage/__init__.py @@ -66,7 +66,7 @@ SCHEMAS = [ # Remember to update this number every time an incompatible change is made to # database schema files, so the users will be informed on server restarts. -SCHEMA_VERSION = 9 +SCHEMA_VERSION = 10 class _RollbackButIsFineException(Exception): -- cgit 1.4.1 From 2e44714214af801906c9fbc3e1cb1bc8d6b92427 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 16 Dec 2014 14:20:32 +0000 Subject: Make failure to run appropraite upgrade scripts more helpful. --- synapse/app/homeserver.py | 13 ++++++++++--- synapse/storage/__init__.py | 12 ++++++++++++ 2 files changed, 22 insertions(+), 3 deletions(-) (limited to 'synapse') diff --git a/synapse/app/homeserver.py b/synapse/app/homeserver.py index a6e29c0860..140c99f18a 100755 --- a/synapse/app/homeserver.py +++ b/synapse/app/homeserver.py @@ -14,7 +14,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -from synapse.storage import prepare_database +from synapse.storage import prepare_database, UpgradeDatabaseException from synapse.server import HomeServer @@ -228,8 +228,15 @@ def setup(): logger.info("Preparing database: %s...", db_name) - with sqlite3.connect(db_name) as db_conn: - prepare_database(db_conn) + try: + with sqlite3.connect(db_name) as db_conn: + prepare_database(db_conn) + except UpgradeDatabaseException: + sys.stderr.write( + "\nFailed to upgrade database.\n" + "Have you followed any instructions in UPGRADES.rst?\n" + ) + sys.exit(1) logger.info("Database prepared in %s.", db_name) diff --git a/synapse/storage/__init__.py b/synapse/storage/__init__.py index 62f89b8d8c..2e97bbab3b 100644 --- a/synapse/storage/__init__.py +++ b/synapse/storage/__init__.py @@ -526,6 +526,14 @@ def read_schema(schema): return schema_file.read() +class PrepareDatabaseException(Exception): + pass + + +class UpgradeDatabaseException(PrepareDatabaseException): + pass + + def prepare_database(db_conn): """ Set up all the dbs. Since all the *.sql have IF NOT EXISTS, so we don't have to worry about overwriting existing content. @@ -542,6 +550,10 @@ def prepare_database(db_conn): "Cannot use this database as it is too " + "new for the server to understand" ) + elif user_version < 10: + raise UpgradeDatabaseException( + "No delta for versions less than 10" + ) elif user_version < SCHEMA_VERSION: logger.info( "Upgrading database from version %d", -- cgit 1.4.1 From b3c793e362004f736c84388008a2aad07f61a492 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 16 Dec 2014 14:44:53 +0000 Subject: Do run all deltas up to missing delta 10 --- synapse/storage/__init__.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) (limited to 'synapse') diff --git a/synapse/storage/__init__.py b/synapse/storage/__init__.py index 2e97bbab3b..2a683b25f7 100644 --- a/synapse/storage/__init__.py +++ b/synapse/storage/__init__.py @@ -550,10 +550,6 @@ def prepare_database(db_conn): "Cannot use this database as it is too " + "new for the server to understand" ) - elif user_version < 10: - raise UpgradeDatabaseException( - "No delta for versions less than 10" - ) elif user_version < SCHEMA_VERSION: logger.info( "Upgrading database from version %d", @@ -562,6 +558,10 @@ def prepare_database(db_conn): # Run every version since after the current version. for v in range(user_version + 1, SCHEMA_VERSION + 1): + if v == 10: + raise UpgradeDatabaseException( + "No delta for version 10" + ) sql_script = read_schema("delta/v%d" % (v)) c.executescript(sql_script) -- cgit 1.4.1 From 42b725ce52844b3e858193aa12ddc06933c7584a Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 16 Dec 2014 15:13:34 +0000 Subject: Fix upgrade script to run all the missing deltas. --- scripts/upgrade_db_to_v0.6.0.py | 20 +++++++++++++ synapse/storage/schema/delta/v9.sql | 58 ++++++++++++++++++++++++++++++++++++- 2 files changed, 77 insertions(+), 1 deletion(-) (limited to 'synapse') diff --git a/scripts/upgrade_db_to_v0.6.0.py b/scripts/upgrade_db_to_v0.6.0.py index add088a818..32c415a662 100644 --- a/scripts/upgrade_db_to_v0.6.0.py +++ b/scripts/upgrade_db_to_v0.6.0.py @@ -1,3 +1,5 @@ + +from synapse.storage import SCHEMA_VERSION, read_schema from synapse.storage._base import SQLBaseStore from synapse.storage.signatures import SignatureStore from synapse.storage.event_federation import EventFederationStore @@ -186,12 +188,16 @@ def get_key(server_name): def reinsert_events(cursor, server_name, signing_key): + print "Running delta: v10" + cursor.executescript(delta_sql) cursor.execute( "SELECT * FROM events ORDER BY rowid ASC" ) + print "Getting events..." + rows = store.cursor_to_dict(cursor) events = store._generate_event_json(cursor, rows) @@ -281,7 +287,21 @@ def reinsert_events(cursor, server_name, signing_key): def main(database, server_name, signing_key): conn = sqlite3.connect(database) cursor = conn.cursor() + + # Do other deltas: + cursor.execute("PRAGMA user_version") + row = cursor.fetchone() + + if row and row[0]: + user_version = row[0] + # Run every version since after the current version. + for v in range(user_version + 1, 10): + print "Running delta: %d" % (v,) + sql_script = read_schema("delta/v%d" % (v,)) + cursor.executescript(sql_script) + reinsert_events(cursor, server_name, signing_key) + conn.commit() print "Success!" diff --git a/synapse/storage/schema/delta/v9.sql b/synapse/storage/schema/delta/v9.sql index ad680c64da..0af29733a0 100644 --- a/synapse/storage/schema/delta/v9.sql +++ b/synapse/storage/schema/delta/v9.sql @@ -20,4 +20,60 @@ CREATE TABLE IF NOT EXISTS destinations( retry_interval INTEGER ); -PRAGMA user_version = 9; \ No newline at end of file + +CREATE TABLE IF NOT EXISTS local_media_repository ( + media_id TEXT, -- The id used to refer to the media. + media_type TEXT, -- The MIME-type of the media. + media_length INTEGER, -- Length of the media in bytes. + created_ts INTEGER, -- When the content was uploaded in ms. + upload_name TEXT, -- The name the media was uploaded with. + user_id TEXT, -- The user who uploaded the file. + CONSTRAINT uniqueness UNIQUE (media_id) +); + +CREATE TABLE IF NOT EXISTS local_media_repository_thumbnails ( + media_id TEXT, -- The id used to refer to the media. + thumbnail_width INTEGER, -- The width of the thumbnail in pixels. + thumbnail_height INTEGER, -- The height of the thumbnail in pixels. + thumbnail_type TEXT, -- The MIME-type of the thumbnail. + thumbnail_method TEXT, -- The method used to make the thumbnail. + thumbnail_length INTEGER, -- The length of the thumbnail in bytes. + CONSTRAINT uniqueness UNIQUE ( + media_id, thumbnail_width, thumbnail_height, thumbnail_type + ) +); + +CREATE INDEX IF NOT EXISTS local_media_repository_thumbnails_media_id + ON local_media_repository_thumbnails (media_id); + +CREATE TABLE IF NOT EXISTS remote_media_cache ( + media_origin TEXT, -- The remote HS the media came from. + media_id TEXT, -- The id used to refer to the media on that server. + media_type TEXT, -- The MIME-type of the media. + created_ts INTEGER, -- When the content was uploaded in ms. + upload_name TEXT, -- The name the media was uploaded with. + media_length INTEGER, -- Length of the media in bytes. + filesystem_id TEXT, -- The name used to store the media on disk. + CONSTRAINT uniqueness UNIQUE (media_origin, media_id) +); + +CREATE TABLE IF NOT EXISTS remote_media_cache_thumbnails ( + media_origin TEXT, -- The remote HS the media came from. + media_id TEXT, -- The id used to refer to the media. + thumbnail_width INTEGER, -- The width of the thumbnail in pixels. + thumbnail_height INTEGER, -- The height of the thumbnail in pixels. + thumbnail_method TEXT, -- The method used to make the thumbnail + thumbnail_type TEXT, -- The MIME-type of the thumbnail. + thumbnail_length INTEGER, -- The length of the thumbnail in bytes. + filesystem_id TEXT, -- The name used to store the media on disk. + CONSTRAINT uniqueness UNIQUE ( + media_origin, media_id, thumbnail_width, thumbnail_height, + thumbnail_type, thumbnail_type + ) +); + +CREATE INDEX IF NOT EXISTS remote_media_cache_thumbnails_media_id + ON local_media_repository_thumbnails (media_id); + + +PRAGMA user_version = 9; -- cgit 1.4.1 From 23da4a40510bfda5c617946cf6f2b7ab3f95b480 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 16 Dec 2014 15:59:40 +0000 Subject: Fix typo where we thought a list was a dict --- synapse/state.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) (limited to 'synapse') diff --git a/synapse/state.py b/synapse/state.py index 38adde4dc9..99f873b6e5 100644 --- a/synapse/state.py +++ b/synapse/state.py @@ -263,9 +263,10 @@ class StateHandler(object): } if event_type: - prev_states = conflicted_state.get( - (event_type, state_key), {} - ).keys() + prev_states_events = conflicted_state.get( + (event_type, state_key), [] + ) + prev_states = [s.event_id for s in prev_states_events] else: prev_states = [] -- cgit 1.4.1 From f06161a3077723a47772cbefc08a4bf2f14b176b Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 16 Dec 2014 16:10:17 +0000 Subject: Enable rate limiting for all events --- synapse/handlers/message.py | 5 +++++ 1 file changed, 5 insertions(+) (limited to 'synapse') diff --git a/synapse/handlers/message.py b/synapse/handlers/message.py index baf372fdad..7939d2bffa 100644 --- a/synapse/handlers/message.py +++ b/synapse/handlers/message.py @@ -154,6 +154,11 @@ class MessageHandler(BaseHandler): self.validator.validate_new(builder) + self.ratelimit(builder.user_id) + # TODO(paul): Why does 'event' not have a 'user' object? + user = self.hs.parse_userid(builder.user_id) + assert self.hs.is_mine(user), "User must be our own: %s" % (user,) + if builder.type == EventTypes.Member: membership = builder.content.get("membership", None) if membership == Membership.JOIN: -- cgit 1.4.1