diff --git a/scripts/move_remote_media_to_new_store.py b/scripts/move_remote_media_to_new_store.py
new file mode 100755
index 0000000000..7914ead889
--- /dev/null
+++ b/scripts/move_remote_media_to_new_store.py
@@ -0,0 +1,133 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+# Copyright 2017 New Vector Ltd
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""
+Moves a list of remote media from one media store to another.
+
+The input should be a list of media files to be moved, one per line. Each line
+should be formatted::
+
+ <origin server>|<file id>
+
+This can be extracted from postgres with::
+
+ psql --tuples-only -A -c "select media_origin, filesystem_id from
+ matrix.remote_media_cache where ..."
+
+To use, pipe the above into::
+
+ PYTHON_PATH=. ./scripts/move_remote_media_to_new_store.py <source repo> <dest repo>
+"""
+
+from __future__ import print_function
+
+import argparse
+import logging
+
+import sys
+
+import os
+
+import shutil
+
+from synapse.rest.media.v1.filepath import MediaFilePaths
+
+logger = logging.getLogger()
+
+
+def main(src_repo, dest_repo):
+ src_paths = MediaFilePaths(src_repo)
+ dest_paths = MediaFilePaths(dest_repo)
+ for line in sys.stdin:
+ line = line.strip()
+ parts = line.split('|')
+ if len(parts) != 2:
+ print("Unable to parse input line %s" % line, file=sys.stderr)
+ exit(1)
+
+ move_media(parts[0], parts[1], src_paths, dest_paths)
+
+
+def move_media(origin_server, file_id, src_paths, dest_paths):
+ """Move the given file, and any thumbnails, to the dest repo
+
+ Args:
+ origin_server (str):
+ file_id (str):
+ src_paths (MediaFilePaths):
+ dest_paths (MediaFilePaths):
+ """
+ logger.info("%s/%s", origin_server, file_id)
+
+ # check that the original exists
+ original_file = src_paths.remote_media_filepath(origin_server, file_id)
+ if not os.path.exists(original_file):
+ logger.warn(
+ "Original for %s/%s (%s) does not exist",
+ origin_server, file_id, original_file,
+ )
+ else:
+ mkdir_and_move(
+ original_file,
+ dest_paths.remote_media_filepath(origin_server, file_id),
+ )
+
+ # now look for thumbnails
+ original_thumb_dir = src_paths.remote_media_thumbnail_dir(
+ origin_server, file_id,
+ )
+ if not os.path.exists(original_thumb_dir):
+ return
+
+ mkdir_and_move(
+ original_thumb_dir,
+ dest_paths.remote_media_thumbnail_dir(origin_server, file_id)
+ )
+
+
+def mkdir_and_move(original_file, dest_file):
+ dirname = os.path.dirname(dest_file)
+ if not os.path.exists(dirname):
+ logger.debug("mkdir %s", dirname)
+ os.makedirs(dirname)
+ logger.debug("mv %s %s", original_file, dest_file)
+ shutil.move(original_file, dest_file)
+
+
+if __name__ == "__main__":
+ parser = argparse.ArgumentParser(
+ description=__doc__,
+ formatter_class = argparse.RawDescriptionHelpFormatter,
+ )
+ parser.add_argument(
+ "-v", action='store_true', help='enable debug logging')
+ parser.add_argument(
+ "src_repo",
+ help="Path to source content repo",
+ )
+ parser.add_argument(
+ "dest_repo",
+ help="Path to source content repo",
+ )
+ args = parser.parse_args()
+
+ logging_config = {
+ "level": logging.DEBUG if args.v else logging.INFO,
+ "format": "%(asctime)s - %(name)s - %(lineno)d - %(levelname)s - %(message)s"
+ }
+ logging.basicConfig(**logging_config)
+
+ main(args.src_repo, args.dest_repo)
diff --git a/scripts/register_new_matrix_user b/scripts/register_new_matrix_user
index 12ed20d623..8c3d429351 100755
--- a/scripts/register_new_matrix_user
+++ b/scripts/register_new_matrix_user
@@ -26,11 +26,37 @@ import yaml
def request_registration(user, password, server_location, shared_secret, admin=False):
+ req = urllib2.Request(
+ "%s/_matrix/client/r0/admin/register" % (server_location,),
+ headers={'Content-Type': 'application/json'}
+ )
+
+ try:
+ if sys.version_info[:3] >= (2, 7, 9):
+ # As of version 2.7.9, urllib2 now checks SSL certs
+ import ssl
+ f = urllib2.urlopen(req, context=ssl.SSLContext(ssl.PROTOCOL_SSLv23))
+ else:
+ f = urllib2.urlopen(req)
+ body = f.read()
+ f.close()
+ nonce = json.loads(body)["nonce"]
+ except urllib2.HTTPError as e:
+ print "ERROR! Received %d %s" % (e.code, e.reason,)
+ if 400 <= e.code < 500:
+ if e.info().type == "application/json":
+ resp = json.load(e)
+ if "error" in resp:
+ print resp["error"]
+ sys.exit(1)
+
mac = hmac.new(
key=shared_secret,
digestmod=hashlib.sha1,
)
+ mac.update(nonce)
+ mac.update("\x00")
mac.update(user)
mac.update("\x00")
mac.update(password)
@@ -40,10 +66,10 @@ def request_registration(user, password, server_location, shared_secret, admin=F
mac = mac.hexdigest()
data = {
- "user": user,
+ "nonce": nonce,
+ "username": user,
"password": password,
"mac": mac,
- "type": "org.matrix.login.shared_secret",
"admin": admin,
}
@@ -52,7 +78,7 @@ def request_registration(user, password, server_location, shared_secret, admin=F
print "Sending registration request..."
req = urllib2.Request(
- "%s/_matrix/client/api/v1/register" % (server_location,),
+ "%s/_matrix/client/r0/admin/register" % (server_location,),
data=json.dumps(data),
headers={'Content-Type': 'application/json'}
)
diff --git a/scripts/synapse_port_db b/scripts/synapse_port_db
index 7d158a46a4..b9b828c154 100755
--- a/scripts/synapse_port_db
+++ b/scripts/synapse_port_db
@@ -1,6 +1,7 @@
#!/usr/bin/env python
# -*- coding: utf-8 -*-
# Copyright 2015, 2016 OpenMarket Ltd
+# Copyright 2018 New Vector Ltd
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
@@ -29,6 +30,8 @@ import time
import traceback
import yaml
+from six import string_types
+
logger = logging.getLogger("synapse_port_db")
@@ -42,6 +45,14 @@ BOOLEAN_COLUMNS = {
"public_room_list_stream": ["visibility"],
"device_lists_outbound_pokes": ["sent"],
"users_who_share_rooms": ["share_private"],
+ "groups": ["is_public"],
+ "group_rooms": ["is_public"],
+ "group_users": ["is_public", "is_admin"],
+ "group_summary_rooms": ["is_public"],
+ "group_room_categories": ["is_public"],
+ "group_summary_users": ["is_public"],
+ "group_roles": ["is_public"],
+ "local_group_membership": ["is_publicised", "is_admin"],
}
@@ -112,6 +123,7 @@ class Store(object):
_simple_update_one = SQLBaseStore.__dict__["_simple_update_one"]
_simple_update_one_txn = SQLBaseStore.__dict__["_simple_update_one_txn"]
+ _simple_update_txn = SQLBaseStore.__dict__["_simple_update_txn"]
def runInteraction(self, desc, func, *args, **kwargs):
def r(conn):
@@ -241,6 +253,12 @@ class Porter(object):
@defer.inlineCallbacks
def handle_table(self, table, postgres_size, table_size, forward_chunk,
backward_chunk):
+ logger.info(
+ "Table %s: %i/%i (rows %i-%i) already ported",
+ table, postgres_size, table_size,
+ backward_chunk+1, forward_chunk-1,
+ )
+
if not table_size:
return
@@ -252,6 +270,25 @@ class Porter(object):
)
return
+ if table in (
+ "user_directory", "user_directory_search", "users_who_share_rooms",
+ "users_in_pubic_room",
+ ):
+ # We don't port these tables, as they're a faff and we can regenreate
+ # them anyway.
+ self.progress.update(table, table_size) # Mark table as done
+ return
+
+ if table == "user_directory_stream_pos":
+ # We need to make sure there is a single row, `(X, null), as that is
+ # what synapse expects to be there.
+ yield self.postgres_store._simple_insert(
+ table=table,
+ values={"stream_id": None},
+ )
+ self.progress.update(table, table_size) # Mark table as done
+ return
+
forward_select = (
"SELECT rowid, * FROM %s WHERE rowid >= ? ORDER BY rowid LIMIT ?"
% (table,)
@@ -299,7 +336,7 @@ class Porter(object):
backward_chunk = min(row[0] for row in brows) - 1
rows = frows + brows
- self._convert_rows(table, headers, rows)
+ rows = self._convert_rows(table, headers, rows)
def insert(txn):
self.postgres_store.insert_many_txn(
@@ -357,10 +394,13 @@ class Porter(object):
" VALUES (?,?,?,?,to_tsvector('english', ?),?,?)"
)
- rows_dict = [
- dict(zip(headers, row))
- for row in rows
- ]
+ rows_dict = []
+ for row in rows:
+ d = dict(zip(headers, row))
+ if "\0" in d['value']:
+ logger.warn('dropping search row %s', d)
+ else:
+ rows_dict.append(d)
txn.executemany(sql, [
(
@@ -436,31 +476,10 @@ class Porter(object):
self.progress.set_state("Preparing PostgreSQL")
self.setup_db(postgres_config, postgres_engine)
- # Step 2. Get tables.
- self.progress.set_state("Fetching tables")
- sqlite_tables = yield self.sqlite_store._simple_select_onecol(
- table="sqlite_master",
- keyvalues={
- "type": "table",
- },
- retcol="name",
- )
-
- postgres_tables = yield self.postgres_store._simple_select_onecol(
- table="information_schema.tables",
- keyvalues={},
- retcol="distinct table_name",
- )
-
- tables = set(sqlite_tables) & set(postgres_tables)
-
- self.progress.set_state("Creating tables")
-
- logger.info("Found %d tables", len(tables))
-
+ self.progress.set_state("Creating port tables")
def create_port_table(txn):
txn.execute(
- "CREATE TABLE port_from_sqlite3 ("
+ "CREATE TABLE IF NOT EXISTS port_from_sqlite3 ("
" table_name varchar(100) NOT NULL UNIQUE,"
" forward_rowid bigint NOT NULL,"
" backward_rowid bigint NOT NULL"
@@ -486,18 +505,33 @@ class Porter(object):
"alter_table", alter_table
)
except Exception as e:
- logger.info("Failed to create port table: %s", e)
+ pass
- try:
- yield self.postgres_store.runInteraction(
- "create_port_table", create_port_table
- )
- except Exception as e:
- logger.info("Failed to create port table: %s", e)
+ yield self.postgres_store.runInteraction(
+ "create_port_table", create_port_table
+ )
- self.progress.set_state("Setting up")
+ # Step 2. Get tables.
+ self.progress.set_state("Fetching tables")
+ sqlite_tables = yield self.sqlite_store._simple_select_onecol(
+ table="sqlite_master",
+ keyvalues={
+ "type": "table",
+ },
+ retcol="name",
+ )
- # Set up tables.
+ postgres_tables = yield self.postgres_store._simple_select_onecol(
+ table="information_schema.tables",
+ keyvalues={},
+ retcol="distinct table_name",
+ )
+
+ tables = set(sqlite_tables) & set(postgres_tables)
+ logger.info("Found %d tables", len(tables))
+
+ # Step 3. Figure out what still needs copying
+ self.progress.set_state("Checking on port progress")
setup_res = yield defer.gatherResults(
[
self.setup_table(table)
@@ -508,7 +542,8 @@ class Porter(object):
consumeErrors=True,
)
- # Process tables.
+ # Step 4. Do the copying.
+ self.progress.set_state("Copying to postgres")
yield defer.gatherResults(
[
self.handle_table(*res)
@@ -517,6 +552,9 @@ class Porter(object):
consumeErrors=True,
)
+ # Step 5. Do final post-processing
+ yield self._setup_state_group_id_seq()
+
self.progress.done()
except:
global end_error_exec_info
@@ -532,17 +570,29 @@ class Porter(object):
i for i, h in enumerate(headers) if h in bool_col_names
]
+ class BadValueException(Exception):
+ pass
+
def conv(j, col):
if j in bool_cols:
return bool(col)
+ elif isinstance(col, string_types) and "\0" in col:
+ logger.warn("DROPPING ROW: NUL value in table %s col %s: %r", table, headers[j], col)
+ raise BadValueException();
return col
+ outrows = []
for i, row in enumerate(rows):
- rows[i] = tuple(
- conv(j, col)
- for j, col in enumerate(row)
- if j > 0
- )
+ try:
+ outrows.append(tuple(
+ conv(j, col)
+ for j, col in enumerate(row)
+ if j > 0
+ ))
+ except BadValueException:
+ pass
+
+ return outrows
@defer.inlineCallbacks
def _setup_sent_transactions(self):
@@ -570,7 +620,7 @@ class Porter(object):
"select", r,
)
- self._convert_rows("sent_transactions", headers, rows)
+ rows = self._convert_rows("sent_transactions", headers, rows)
inserted_rows = len(rows)
if inserted_rows:
@@ -664,6 +714,16 @@ class Porter(object):
defer.returnValue((done, remaining + done))
+ def _setup_state_group_id_seq(self):
+ def r(txn):
+ txn.execute("SELECT MAX(id) FROM state_groups")
+ next_id = txn.fetchone()[0]+1
+ txn.execute(
+ "ALTER SEQUENCE state_group_id_seq RESTART WITH %s",
+ (next_id,),
+ )
+ return self.postgres_store.runInteraction("setup_state_group_id_seq", r)
+
##############################################
###### The following is simply UI stuff ######
diff --git a/scripts/sync_room_to_group.pl b/scripts/sync_room_to_group.pl
new file mode 100755
index 0000000000..f0c2dfadfa
--- /dev/null
+++ b/scripts/sync_room_to_group.pl
@@ -0,0 +1,45 @@
+#!/usr/bin/env perl
+
+use strict;
+use warnings;
+
+use JSON::XS;
+use LWP::UserAgent;
+use URI::Escape;
+
+if (@ARGV < 4) {
+ die "usage: $0 <homeserver url> <access_token> <room_id|room_alias> <group_id>\n";
+}
+
+my ($hs, $access_token, $room_id, $group_id) = @ARGV;
+my $ua = LWP::UserAgent->new();
+$ua->timeout(10);
+
+if ($room_id =~ /^#/) {
+ $room_id = uri_escape($room_id);
+ $room_id = decode_json($ua->get("${hs}/_matrix/client/r0/directory/room/${room_id}?access_token=${access_token}")->decoded_content)->{room_id};
+}
+
+my $room_users = [ keys %{decode_json($ua->get("${hs}/_matrix/client/r0/rooms/${room_id}/joined_members?access_token=${access_token}")->decoded_content)->{joined}} ];
+my $group_users = [
+ (map { $_->{user_id} } @{decode_json($ua->get("${hs}/_matrix/client/unstable/groups/${group_id}/users?access_token=${access_token}" )->decoded_content)->{chunk}}),
+ (map { $_->{user_id} } @{decode_json($ua->get("${hs}/_matrix/client/unstable/groups/${group_id}/invited_users?access_token=${access_token}" )->decoded_content)->{chunk}}),
+];
+
+die "refusing to sync from empty room" unless (@$room_users);
+die "refusing to sync to empty group" unless (@$group_users);
+
+my $diff = {};
+foreach my $user (@$room_users) { $diff->{$user}++ }
+foreach my $user (@$group_users) { $diff->{$user}-- }
+
+foreach my $user (keys %$diff) {
+ if ($diff->{$user} == 1) {
+ warn "inviting $user";
+ print STDERR $ua->put("${hs}/_matrix/client/unstable/groups/${group_id}/admin/users/invite/${user}?access_token=${access_token}", Content=>'{}')->status_line."\n";
+ }
+ elsif ($diff->{$user} == -1) {
+ warn "removing $user";
+ print STDERR $ua->put("${hs}/_matrix/client/unstable/groups/${group_id}/admin/users/remove/${user}?access_token=${access_token}", Content=>'{}')->status_line."\n";
+ }
+}
|
