From 9f3c0a8556a82960c795b8dc41a4666e0adebfef Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Mon, 1 Jul 2019 17:55:26 +0100 Subject: Add basic admin cmd app --- synapse/app/admin_cmd.py | 198 +++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 198 insertions(+) create mode 100644 synapse/app/admin_cmd.py (limited to 'synapse/app/admin_cmd.py') diff --git a/synapse/app/admin_cmd.py b/synapse/app/admin_cmd.py new file mode 100644 index 0000000000..bd73c47ae2 --- /dev/null +++ b/synapse/app/admin_cmd.py @@ -0,0 +1,198 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +# Copyright 2016 OpenMarket Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import logging +import sys + +from twisted.internet import defer, task + +import synapse +from synapse.app import _base +from synapse.config._base import ConfigError +from synapse.config.homeserver import HomeServerConfig +from synapse.config.logger import setup_logging +from synapse.handlers.admin import FileExfiltrationWriter +from synapse.replication.slave.storage._base import BaseSlavedStore +from synapse.replication.slave.storage.account_data import SlavedAccountDataStore +from synapse.replication.slave.storage.appservice import SlavedApplicationServiceStore +from synapse.replication.slave.storage.client_ips import SlavedClientIpStore +from synapse.replication.slave.storage.deviceinbox import SlavedDeviceInboxStore +from synapse.replication.slave.storage.devices import SlavedDeviceStore +from synapse.replication.slave.storage.events import SlavedEventStore +from synapse.replication.slave.storage.filtering import SlavedFilteringStore +from synapse.replication.slave.storage.groups import SlavedGroupServerStore +from synapse.replication.slave.storage.presence import SlavedPresenceStore +from synapse.replication.slave.storage.push_rule import SlavedPushRuleStore +from synapse.replication.slave.storage.receipts import SlavedReceiptsStore +from synapse.replication.slave.storage.registration import SlavedRegistrationStore +from synapse.replication.slave.storage.room import RoomStore +from synapse.replication.tcp.client import ReplicationClientHandler +from synapse.server import HomeServer +from synapse.storage.engines import create_engine +from synapse.util.logcontext import LoggingContext +from synapse.util.versionstring import get_version_string + +logger = logging.getLogger("synapse.app.admin_cmd") + + +class AdminCmdSlavedStore( + SlavedReceiptsStore, + SlavedAccountDataStore, + SlavedApplicationServiceStore, + SlavedRegistrationStore, + SlavedFilteringStore, + SlavedPresenceStore, + SlavedGroupServerStore, + SlavedDeviceInboxStore, + SlavedDeviceStore, + SlavedPushRuleStore, + SlavedEventStore, + SlavedClientIpStore, + RoomStore, + BaseSlavedStore, +): + pass + + +class AdminCmdServer(HomeServer): + DATASTORE_CLASS = AdminCmdSlavedStore + + def _listen_http(self, listener_config): + pass + + def start_listening(self, listeners): + pass + + def build_tcp_replication(self): + return AdminCmdReplicationHandler(self) + + +class AdminCmdReplicationHandler(ReplicationClientHandler): + @defer.inlineCallbacks + def on_rdata(self, stream_name, token, rows): + pass + + def get_streams_to_replicate(self): + return {} + + +@defer.inlineCallbacks +def export_data_command(hs, user_id, directory): + """Export data for a user. + + Args: + user_id (str) + directory (str|None): Directory to write output to. Will create a temp + directory if not specified. + """ + + res = yield hs.get_handlers().admin_handler.exfiltrate_user_data( + user_id, FileExfiltrationWriter(user_id, directory=directory) + ) + print(res) + + +def start(config_options): + parser = HomeServerConfig.create_argument_parser("Synapse Admin Command") + + subparser = parser.add_subparsers( + title="Admin Commands", + description="Choose and admin command to perform.", + required=True, + dest="command", + metavar="", + help="The admin command to perform.", + ) + export_data_parser = subparser.add_parser( + "export-data", help="Export all data for a user" + ) + export_data_parser.add_argument("user_id", help="User to extra data from") + export_data_parser.add_argument( + "--output-directory", + action="store", + metavar="DIRECTORY", + required=False, + help="The directory to store the exported data in. Must be emtpy. Defaults" + " to creating a temp directory.", + ) + + try: + config, args = HomeServerConfig.load_config_with_parser(parser, config_options) + except ConfigError as e: + sys.stderr.write("\n" + str(e) + "\n") + sys.exit(1) + + if config.worker_app is not None: + assert config.worker_app == "synapse.app.admin_cmd" + + # Update the config with some basic overrides so that don't have to specify + # a full worker config. + config.worker_app = "synapse.app.admin_cmd" + + if ( + not config.worker_daemonize + and not config.worker_log_file + and not config.worker_log_config + ): + # Since we're meant to be run as a "command" let's not redirect stdio + # unless we've actually set log config. + config.no_redirect_stdio = True + + # Explicitly disable background processes + config.update_user_directory = False + config.start_pushers = False + config.send_federation = False + + setup_logging(config, use_worker_options=True) + + synapse.events.USE_FROZEN_DICTS = config.use_frozen_dicts + + database_engine = create_engine(config.database_config) + + ss = AdminCmdServer( + config.server_name, + db_config=config.database_config, + config=config, + version_string="Synapse/" + get_version_string(synapse), + database_engine=database_engine, + ) + + ss.setup() + + if args.command == "export-data": + command = lambda: export_data_command(ss, args.user_id, args.output_directory) + else: + # This shouldn't happen. + raise ConfigError("Unknown admin command %s" % (args.command,)) + + # We use task.react as the basic run command as it correctly handles tearing + # down the reactor when the deferreds resolve and setting the return value. + # We also make sure that `_base.start` gets run before we actually run the + # command. + + @defer.inlineCallbacks + def run(_reactor): + with LoggingContext("command"): + yield _base.start(ss, []) + yield command() + + _base.start_worker_reactor( + "synapse-admin-cmd", config, run_command=lambda: task.react(run) + ) + + +if __name__ == "__main__": + with LoggingContext("main"): + start(sys.argv[1:]) -- cgit 1.5.1 From 37b524f9718b0faeaaac0dccab62c9a5e270c4a2 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Mon, 15 Jul 2019 13:19:57 +0100 Subject: Fix up comments --- synapse/app/admin_cmd.py | 4 ++-- synapse/config/_base.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) (limited to 'synapse/app/admin_cmd.py') diff --git a/synapse/app/admin_cmd.py b/synapse/app/admin_cmd.py index bd73c47ae2..e618a62432 100644 --- a/synapse/app/admin_cmd.py +++ b/synapse/app/admin_cmd.py @@ -1,6 +1,6 @@ #!/usr/bin/env python # -*- coding: utf-8 -*- -# Copyright 2016 OpenMarket Ltd +# Copyright 2019 Matrix.org Foundation C.I.C. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -109,7 +109,7 @@ def start(config_options): subparser = parser.add_subparsers( title="Admin Commands", - description="Choose and admin command to perform.", + description="Choose an admin command to perform.", required=True, dest="command", metavar="", diff --git a/synapse/config/_base.py b/synapse/config/_base.py index e588f82981..74a7980ebe 100644 --- a/synapse/config/_base.py +++ b/synapse/config/_base.py @@ -284,7 +284,7 @@ class Config(object): Used for workers where we want to add extra flags/subcommands. Args: - conifg_parser (ArgumentParser) + config_parser (ArgumentParser) argv (list[str]) Returns: -- cgit 1.5.1 From fdefb9e29a7c28e5b218fc8d9745a2ec58bc3d27 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Mon, 15 Jul 2019 13:43:25 +0100 Subject: Move creation of ArgumentParser to caller --- synapse/app/admin_cmd.py | 4 +++- synapse/config/_base.py | 15 +++++---------- 2 files changed, 8 insertions(+), 11 deletions(-) (limited to 'synapse/app/admin_cmd.py') diff --git a/synapse/app/admin_cmd.py b/synapse/app/admin_cmd.py index e618a62432..c4d752593a 100644 --- a/synapse/app/admin_cmd.py +++ b/synapse/app/admin_cmd.py @@ -13,6 +13,7 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. +import argparse import logging import sys @@ -105,7 +106,8 @@ def export_data_command(hs, user_id, directory): def start(config_options): - parser = HomeServerConfig.create_argument_parser("Synapse Admin Command") + parser = argparse.ArgumentParser(description="Synapse Admin Command") + HomeServerConfig.add_arguments_to_parser(parser) subparser = parser.add_subparsers( title="Admin Commands", diff --git a/synapse/config/_base.py b/synapse/config/_base.py index 74a7980ebe..8c3acff03e 100644 --- a/synapse/config/_base.py +++ b/synapse/config/_base.py @@ -231,27 +231,24 @@ class Config(object): Returns: Config object. """ - config_parser = cls.create_argument_parser(description) + config_parser = argparse.ArgumentParser(description=description) + cls.add_arguments_to_parser(config_parser) obj, _ = cls.load_config_with_parser(config_parser, argv) return obj @classmethod - def create_argument_parser(cls, description): - """Create an ArgumentParser instance with all the config flags. + def add_arguments_to_parser(cls, config_parser): + """Adds all the config flags to an ArgumentParser. Doesn't support config-file-generation: used by the worker apps. Used for workers where we want to add extra flags/subcommands. Args: - description (str): App description - - Returns: - ArgumentParser + config_parser (ArgumentParser): App description """ - config_parser = argparse.ArgumentParser(description=description) config_parser.add_argument( "-c", "--config-path", @@ -273,8 +270,6 @@ class Config(object): # `add_arguments` should be side effect free so this is probably fine. cls.invoke_all_static("add_arguments", config_parser) - return config_parser - @classmethod def load_config_with_parser(cls, config_parser, argv): """Parse the commandline and config files with the given parser -- cgit 1.5.1 From c8f35d8d38c91b3493bd34363096a6e26756d8d7 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Mon, 15 Jul 2019 13:49:18 +0100 Subject: Use set_defaults(func=) style --- synapse/app/admin_cmd.py | 19 ++++++++----------- 1 file changed, 8 insertions(+), 11 deletions(-) (limited to 'synapse/app/admin_cmd.py') diff --git a/synapse/app/admin_cmd.py b/synapse/app/admin_cmd.py index c4d752593a..611a196e54 100644 --- a/synapse/app/admin_cmd.py +++ b/synapse/app/admin_cmd.py @@ -90,15 +90,17 @@ class AdminCmdReplicationHandler(ReplicationClientHandler): @defer.inlineCallbacks -def export_data_command(hs, user_id, directory): +def export_data_command(hs, args): """Export data for a user. Args: - user_id (str) - directory (str|None): Directory to write output to. Will create a temp - directory if not specified. + hs (HomeServer) + args (argparse.Namespace) """ + user_id = args.user_id + directory = args.output_directory + res = yield hs.get_handlers().admin_handler.exfiltrate_user_data( user_id, FileExfiltrationWriter(user_id, directory=directory) ) @@ -129,6 +131,7 @@ def start(config_options): help="The directory to store the exported data in. Must be emtpy. Defaults" " to creating a temp directory.", ) + export_data_parser.set_defaults(func=export_data_command) try: config, args = HomeServerConfig.load_config_with_parser(parser, config_options) @@ -173,12 +176,6 @@ def start(config_options): ss.setup() - if args.command == "export-data": - command = lambda: export_data_command(ss, args.user_id, args.output_directory) - else: - # This shouldn't happen. - raise ConfigError("Unknown admin command %s" % (args.command,)) - # We use task.react as the basic run command as it correctly handles tearing # down the reactor when the deferreds resolve and setting the return value. # We also make sure that `_base.start` gets run before we actually run the @@ -188,7 +185,7 @@ def start(config_options): def run(_reactor): with LoggingContext("command"): yield _base.start(ss, []) - yield command() + yield args.func(ss, args) _base.start_worker_reactor( "synapse-admin-cmd", config, run_command=lambda: task.react(run) -- cgit 1.5.1 From 1b2067f53d772de1cadca22bdee176b9509d5b6f Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Mon, 15 Jul 2019 14:15:22 +0100 Subject: Add FileExfiltrationWriter --- synapse/app/admin_cmd.py | 70 +++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 69 insertions(+), 1 deletion(-) (limited to 'synapse/app/admin_cmd.py') diff --git a/synapse/app/admin_cmd.py b/synapse/app/admin_cmd.py index 611a196e54..13281d0af9 100644 --- a/synapse/app/admin_cmd.py +++ b/synapse/app/admin_cmd.py @@ -15,7 +15,11 @@ # limitations under the License. import argparse import logging +import os import sys +import tempfile + +from canonicaljson import json from twisted.internet import defer, task @@ -24,7 +28,7 @@ from synapse.app import _base from synapse.config._base import ConfigError from synapse.config.homeserver import HomeServerConfig from synapse.config.logger import setup_logging -from synapse.handlers.admin import FileExfiltrationWriter +from synapse.handlers.admin import ExfiltrationWriter from synapse.replication.slave.storage._base import BaseSlavedStore from synapse.replication.slave.storage.account_data import SlavedAccountDataStore from synapse.replication.slave.storage.appservice import SlavedApplicationServiceStore @@ -107,6 +111,70 @@ def export_data_command(hs, args): print(res) +class FileExfiltrationWriter(ExfiltrationWriter): + """An ExfiltrationWriter that writes the users data to a directory. + Returns the directory location on completion. + + Note: This writes to disk on the main reactor thread. + + Args: + user_id (str): The user whose data is being exfiltrated. + directory (str|None): The directory to write the data to, if None then + will write to a temporary directory. + """ + + def __init__(self, user_id, directory=None): + self.user_id = user_id + + if directory: + self.base_directory = directory + else: + self.base_directory = tempfile.mkdtemp( + prefix="synapse-exfiltrate__%s__" % (user_id,) + ) + + os.makedirs(self.base_directory, exist_ok=True) + if list(os.listdir(self.base_directory)): + raise Exception("Directory must be empty") + + def write_events(self, room_id, events): + room_directory = os.path.join(self.base_directory, "rooms", room_id) + os.makedirs(room_directory, exist_ok=True) + events_file = os.path.join(room_directory, "events") + + with open(events_file, "a") as f: + for event in events: + print(json.dumps(event.get_pdu_json()), file=f) + + def write_state(self, room_id, event_id, state): + room_directory = os.path.join(self.base_directory, "rooms", room_id) + state_directory = os.path.join(room_directory, "state") + os.makedirs(state_directory, exist_ok=True) + + event_file = os.path.join(state_directory, event_id) + + with open(event_file, "a") as f: + for event in state.values(): + print(json.dumps(event.get_pdu_json()), file=f) + + def write_invite(self, room_id, event, state): + self.write_events(room_id, [event]) + + # We write the invite state somewhere else as they aren't full events + # and are only a subset of the state at the event. + room_directory = os.path.join(self.base_directory, "rooms", room_id) + os.makedirs(room_directory, exist_ok=True) + + invite_state = os.path.join(room_directory, "invite_state") + + with open(invite_state, "a") as f: + for event in state.values(): + print(json.dumps(event), file=f) + + def finished(self): + return self.base_directory + + def start(config_options): parser = argparse.ArgumentParser(description="Synapse Admin Command") HomeServerConfig.add_arguments_to_parser(parser) -- cgit 1.5.1 From eca4f5ac730d9ecf29d25888a40a7264b6041a54 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Mon, 15 Jul 2019 14:17:28 +0100 Subject: s/exfiltrate_user_data/export_user_data/ --- synapse/app/admin_cmd.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'synapse/app/admin_cmd.py') diff --git a/synapse/app/admin_cmd.py b/synapse/app/admin_cmd.py index 13281d0af9..3ff19f18e5 100644 --- a/synapse/app/admin_cmd.py +++ b/synapse/app/admin_cmd.py @@ -105,7 +105,7 @@ def export_data_command(hs, args): user_id = args.user_id directory = args.output_directory - res = yield hs.get_handlers().admin_handler.exfiltrate_user_data( + res = yield hs.get_handlers().admin_handler.export_user_data( user_id, FileExfiltrationWriter(user_id, directory=directory) ) print(res) -- cgit 1.5.1 From d0d479c1af78af2204a7ed5aee1e187225a5af90 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 16 Jul 2019 09:52:56 +0100 Subject: Fix typo in synapse/app/admin_cmd.py Co-Authored-By: Aaron Raimist --- synapse/app/admin_cmd.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'synapse/app/admin_cmd.py') diff --git a/synapse/app/admin_cmd.py b/synapse/app/admin_cmd.py index 3ff19f18e5..68802ca751 100644 --- a/synapse/app/admin_cmd.py +++ b/synapse/app/admin_cmd.py @@ -196,7 +196,7 @@ def start(config_options): action="store", metavar="DIRECTORY", required=False, - help="The directory to store the exported data in. Must be emtpy. Defaults" + help="The directory to store the exported data in. Must be empty. Defaults" " to creating a temp directory.", ) export_data_parser.set_defaults(func=export_data_command) -- cgit 1.5.1 From 5ed7853bb018b58e99c6dd3ca91905b877968494 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 16 Jul 2019 11:45:57 +0100 Subject: Remove pointless description --- synapse/app/admin_cmd.py | 1 - 1 file changed, 1 deletion(-) (limited to 'synapse/app/admin_cmd.py') diff --git a/synapse/app/admin_cmd.py b/synapse/app/admin_cmd.py index 68802ca751..1fd52a5526 100644 --- a/synapse/app/admin_cmd.py +++ b/synapse/app/admin_cmd.py @@ -181,7 +181,6 @@ def start(config_options): subparser = parser.add_subparsers( title="Admin Commands", - description="Choose an admin command to perform.", required=True, dest="command", metavar="", -- cgit 1.5.1