Merge branch 'master' of git+ssh://github.com/matrix-org/synapse

1 files changed, 135 insertions, 145 deletions

diff --git a/synapse/app/homeserver.py b/synapse/app/homeserver.py
index 0807def6ca..2b4be7bdd0 100755
--- a/synapse/app/homeserver.py
+++ b/synapse/app/homeserver.py
@@ -1,6 +1,6 @@
 #!/usr/bin/env python
 # -*- coding: utf-8 -*-
-# Copyright 2014, 2015 OpenMarket Ltd
+# Copyright 2014-2016 OpenMarket Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -14,27 +14,23 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+import synapse
+
+import contextlib
+import logging
+import os
+import re
+import resource
+import subprocess
 import sys
-from synapse.rest import ClientRestResource
+import time
+from synapse.config._base import ConfigError
 
-sys.dont_write_bytecode = True
 from synapse.python_dependencies import (
-    check_requirements, DEPENDENCY_LINKS, MissingRequirementError
+    check_requirements, DEPENDENCY_LINKS
 )
 
-if __name__ == '__main__':
-    try:
-        check_requirements()
-    except MissingRequirementError as e:
-        message = "\n".join([
-            "Missing Requirement: %s" % (e.message,),
-            "To install run:",
-            "    pip install --upgrade --force \"%s\"" % (e.dependency,),
-            "",
-        ])
-        sys.stderr.writelines(message)
-        sys.exit(1)
-
+from synapse.rest import ClientRestResource
 from synapse.storage.engines import create_engine, IncorrectDatabaseSetup
 from synapse.storage import are_all_users_on_domain
 from synapse.storage.prepare_database import UpgradeDatabaseException
@@ -42,125 +38,78 @@ from synapse.storage.prepare_database import UpgradeDatabaseException
 from synapse.server import HomeServer
 
 
+from twisted.conch.manhole import ColoredManhole
+from twisted.conch.insults import insults
+from twisted.conch import manhole_ssh
+from twisted.cred import checkers, portal
+
+
 from twisted.internet import reactor, task, defer
 from twisted.application import service
-from twisted.enterprise import adbapi
 from twisted.web.resource import Resource, EncodingResourceWrapper
 from twisted.web.static import File
 from twisted.web.server import Site, GzipEncoderFactory, Request
-from synapse.http.server import JsonResource, RootRedirect
+from synapse.http.server import RootRedirect
 from synapse.rest.media.v0.content_repository import ContentRepoResource
 from synapse.rest.media.v1.media_repository import MediaRepositoryResource
 from synapse.rest.key.v1.server_key_resource import LocalKey
 from synapse.rest.key.v2 import KeyApiV2Resource
-from synapse.http.matrixfederationclient import MatrixFederationHttpClient
 from synapse.api.urls import (
     FEDERATION_PREFIX, WEB_CLIENT_PREFIX, CONTENT_REPO_PREFIX,
-    SERVER_KEY_PREFIX, MEDIA_PREFIX, STATIC_PREFIX,
+    SERVER_KEY_PREFIX, LEGACY_MEDIA_PREFIX, MEDIA_PREFIX, STATIC_PREFIX,
     SERVER_KEY_V2_PREFIX,
 )
 from synapse.config.homeserver import HomeServerConfig
 from synapse.crypto import context_factory
 from synapse.util.logcontext import LoggingContext
 from synapse.metrics.resource import MetricsResource, METRICS_PREFIX
+from synapse.federation.transport.server import TransportLayerServer
 
 from synapse import events
 
 from daemonize import Daemonize
-import twisted.manhole.telnet
-
-import synapse
 
-import contextlib
-import logging
-import os
-import re
-import resource
-import subprocess
-import time
+logger = logging.getLogger("synapse.app.homeserver")
 
 
-logger = logging.getLogger("synapse.app.homeserver")
+ACCESS_TOKEN_RE = re.compile(r'(\?.*access(_|%5[Ff])token=)[^&]*(.*)$')
 
 
 def gz_wrap(r):
     return EncodingResourceWrapper(r, [GzipEncoderFactory()])
 
 
-class SynapseHomeServer(HomeServer):
-
-    def build_http_client(self):
-        return MatrixFederationHttpClient(self)
-
-    def build_client_resource(self):
-        return ClientRestResource(self)
-
-    def build_resource_for_federation(self):
-        return JsonResource(self)
-
-    def build_resource_for_web_client(self):
-        webclient_path = self.get_config().web_client_location
-        if not webclient_path:
-            try:
-                import syweb
-            except ImportError:
-                quit_with_error(
-                    "Could not find a webclient.\n\n"
-                    "Please either install the matrix-angular-sdk or configure\n"
-                    "the location of the source to serve via the configuration\n"
-                    "option `web_client_location`\n\n"
-                    "To install the `matrix-angular-sdk` via pip, run:\n\n"
-                    "    pip install '%(dep)s'\n"
-                    "\n"
-                    "You can also disable hosting of the webclient via the\n"
-                    "configuration option `web_client`\n"
-                    % {"dep": DEPENDENCY_LINKS["matrix-angular-sdk"]}
-                )
-            syweb_path = os.path.dirname(syweb.__file__)
-            webclient_path = os.path.join(syweb_path, "webclient")
-        # GZip is disabled here due to
-        # https://twistedmatrix.com/trac/ticket/7678
-        # (It can stay enabled for the API resources: they call
-        # write() with the whole body and then finish() straight
-        # after and so do not trigger the bug.
-        # GzipFile was removed in commit 184ba09
-        # return GzipFile(webclient_path)  # TODO configurable?
-        return File(webclient_path)  # TODO configurable?
-
-    def build_resource_for_static_content(self):
-        # This is old and should go away: not going to bother adding gzip
-        return File(
-            os.path.join(os.path.dirname(synapse.__file__), "static")
-        )
-
-    def build_resource_for_content_repo(self):
-        return ContentRepoResource(
-            self, self.config.uploads_path, self.auth, self.content_addr
-        )
-
-    def build_resource_for_media_repository(self):
-        return MediaRepositoryResource(self)
-
-    def build_resource_for_server_key(self):
-        return LocalKey(self)
-
-    def build_resource_for_server_key_v2(self):
-        return KeyApiV2Resource(self)
-
-    def build_resource_for_metrics(self):
-        if self.get_config().enable_metrics:
-            return MetricsResource(self)
-        else:
-            return None
-
-    def build_db_pool(self):
-        name = self.db_config["name"]
+def build_resource_for_web_client(hs):
+    webclient_path = hs.get_config().web_client_location
+    if not webclient_path:
+        try:
+            import syweb
+        except ImportError:
+            quit_with_error(
+                "Could not find a webclient.\n\n"
+                "Please either install the matrix-angular-sdk or configure\n"
+                "the location of the source to serve via the configuration\n"
+                "option `web_client_location`\n\n"
+                "To install the `matrix-angular-sdk` via pip, run:\n\n"
+                "    pip install '%(dep)s'\n"
+                "\n"
+                "You can also disable hosting of the webclient via the\n"
+                "configuration option `web_client`\n"
+                % {"dep": DEPENDENCY_LINKS["matrix-angular-sdk"]}
+            )
+        syweb_path = os.path.dirname(syweb.__file__)
+        webclient_path = os.path.join(syweb_path, "webclient")
+    # GZip is disabled here due to
+    # https://twistedmatrix.com/trac/ticket/7678
+    # (It can stay enabled for the API resources: they call
+    # write() with the whole body and then finish() straight
+    # after and so do not trigger the bug.
+    # GzipFile was removed in commit 184ba09
+    # return GzipFile(webclient_path)  # TODO configurable?
+    return File(webclient_path)  # TODO configurable?
 
-        return adbapi.ConnectionPool(
-            name,
-            **self.db_config.get("args", {})
-        )
 
+class SynapseHomeServer(HomeServer):
     def _listener_http(self, config, listener_config):
         port = listener_config["port"]
         bind_address = listener_config.get("bind_address", "")
@@ -170,13 +119,11 @@ class SynapseHomeServer(HomeServer):
         if tls and config.no_tls:
             return
 
-        metrics_resource = self.get_resource_for_metrics()
-
         resources = {}
         for res in listener_config["resources"]:
             for name in res["names"]:
                 if name == "client":
-                    client_resource = self.get_client_resource()
+                    client_resource = ClientRestResource(self)
                     if res["compress"]:
                         client_resource = gz_wrap(client_resource)
 
@@ -185,35 +132,42 @@ class SynapseHomeServer(HomeServer):
                         "/_matrix/client/r0": client_resource,
                         "/_matrix/client/unstable": client_resource,
                         "/_matrix/client/v2_alpha": client_resource,
+                        "/_matrix/client/versions": client_resource,
                     })
 
                 if name == "federation":
                     resources.update({
-                        FEDERATION_PREFIX: self.get_resource_for_federation(),
+                        FEDERATION_PREFIX: TransportLayerServer(self),
                     })
 
                 if name in ["static", "client"]:
                     resources.update({
-                        STATIC_PREFIX: self.get_resource_for_static_content(),
+                        STATIC_PREFIX: File(
+                            os.path.join(os.path.dirname(synapse.__file__), "static")
+                        ),
                     })
 
                 if name in ["media", "federation", "client"]:
+                    media_repo = MediaRepositoryResource(self)
                     resources.update({
-                        MEDIA_PREFIX: self.get_resource_for_media_repository(),
-                        CONTENT_REPO_PREFIX: self.get_resource_for_content_repo(),
+                        MEDIA_PREFIX: media_repo,
+                        LEGACY_MEDIA_PREFIX: media_repo,
+                        CONTENT_REPO_PREFIX: ContentRepoResource(
+                            self, self.config.uploads_path, self.auth, self.content_addr
+                        ),
                     })
 
                 if name in ["keys", "federation"]:
                     resources.update({
-                        SERVER_KEY_PREFIX: self.get_resource_for_server_key(),
-                        SERVER_KEY_V2_PREFIX: self.get_resource_for_server_key_v2(),
+                        SERVER_KEY_PREFIX: LocalKey(self),
+                        SERVER_KEY_V2_PREFIX: KeyApiV2Resource(self),
                     })
 
                 if name == "webclient":
-                    resources[WEB_CLIENT_PREFIX] = self.get_resource_for_web_client()
+                    resources[WEB_CLIENT_PREFIX] = build_resource_for_web_client(self)
 
-                if name == "metrics" and metrics_resource:
-                    resources[METRICS_PREFIX] = metrics_resource
+                if name == "metrics" and self.get_config().enable_metrics:
+                    resources[METRICS_PREFIX] = MetricsResource(self)
 
         root_resource = create_resource_tree(resources)
         if tls:
@@ -248,10 +202,21 @@ class SynapseHomeServer(HomeServer):
             if listener["type"] == "http":
                 self._listener_http(config, listener)
             elif listener["type"] == "manhole":
-                f = twisted.manhole.telnet.ShellFactory()
-                f.username = "matrix"
-                f.password = "rabbithole"
-                f.namespace['hs'] = self
+                checker = checkers.InMemoryUsernamePasswordDatabaseDontUse(
+                    matrix="rabbithole"
+                )
+
+                rlm = manhole_ssh.TerminalRealm()
+                rlm.chainedProtocolFactory = lambda: insults.ServerProtocol(
+                    ColoredManhole,
+                    {
+                        "__name__": "__console__",
+                        "hs": self,
+                    }
+                )
+
+                f = manhole_ssh.ConchFactory(portal.Portal(rlm, [checker]))
+
                 reactor.listenTCP(
                     listener["port"],
                     f,
@@ -276,6 +241,18 @@ class SynapseHomeServer(HomeServer):
         except IncorrectDatabaseSetup as e:
             quit_with_error(e.message)
 
+    def get_db_conn(self):
+        # Any param beginning with cp_ is a parameter for adbapi, and should
+        # not be passed to the database engine.
+        db_params = {
+            k: v for k, v in self.db_config.get("args", {}).items()
+            if not k.startswith("cp_")
+        }
+        db_conn = self.database_engine.module.connect(**db_params)
+
+        self.database_engine.on_new_connection(db_conn)
+        return db_conn
+
 
 def quit_with_error(error_string):
     message_lines = error_string.split("\n")
@@ -358,10 +335,13 @@ def change_resource_limit(soft_file_no):
             soft_file_no = hard
 
         resource.setrlimit(resource.RLIMIT_NOFILE, (soft_file_no, hard))
-
         logger.info("Set file limit to: %d", soft_file_no)
+
+        resource.setrlimit(
+            resource.RLIMIT_CORE, (resource.RLIM_INFINITY, resource.RLIM_INFINITY)
+        )
     except (ValueError, resource.error) as e:
-        logger.warn("Failed to set file limit: %s", e)
+        logger.warn("Failed to set file or core limit: %s", e)
 
 
 def setup(config_options):
@@ -373,11 +353,20 @@ def setup(config_options):
     Returns:
         HomeServer
     """
-    config = HomeServerConfig.load_config(
-        "Synapse Homeserver",
-        config_options,
-        generate_section="Homeserver"
-    )
+    try:
+        config = HomeServerConfig.load_config(
+            "Synapse Homeserver",
+            config_options,
+            generate_section="Homeserver"
+        )
+    except ConfigError as e:
+        sys.stderr.write("\n" + e.message + "\n")
+        sys.exit(1)
+
+    if not config:
+        # If a config isn't returned, and an exception isn't raised, we're just
+        # generating config files and shouldn't try to continue.
+        sys.exit(0)
 
     config.setup_logging()
 
@@ -409,13 +398,7 @@ def setup(config_options):
     logger.info("Preparing database: %s...", config.database_config['name'])
 
     try:
-        db_conn = database_engine.module.connect(
-            **{
-                k: v for k, v in config.database_config.get("args", {}).items()
-                if not k.startswith("cp_")
-            }
-        )
-
+        db_conn = hs.get_db_conn()
         database_engine.prepare_database(db_conn)
         hs.run_startup_checks(db_conn, database_engine)
 
@@ -430,13 +413,17 @@ def setup(config_options):
 
     logger.info("Database prepared in %s.", config.database_config['name'])
 
+    hs.setup()
     hs.start_listening()
 
-    hs.get_pusherpool().start()
-    hs.get_state_handler().start_caching()
-    hs.get_datastore().start_profiling()
-    hs.get_datastore().start_doing_background_updates()
-    hs.get_replication_layer().start_get_pdu_cache()
+    def start():
+        hs.get_pusherpool().start()
+        hs.get_state_handler().start_caching()
+        hs.get_datastore().start_profiling()
+        hs.get_datastore().start_doing_background_updates()
+        hs.get_replication_layer().start_get_pdu_cache()
+
+    reactor.callWhenRunning(start)
 
     return hs
 
@@ -475,9 +462,8 @@ class SynapseRequest(Request):
         )
 
     def get_redacted_uri(self):
-        return re.sub(
-            r'(\?.*access_token=)[^&]*(.*)$',
-            r'\1<redacted>\2',
+        return ACCESS_TOKEN_RE.sub(
+            r'\1<redacted>\3',
             self.uri
         )
 
@@ -653,7 +639,7 @@ def _resource_id(resource, path_seg):
     the mapping should looks like _resource_id(A,C) = B.
 
     Args:
-        resource (Resource): The *parent* Resource
+        resource (Resource): The *parent* Resourceb
         path_seg (str): The name of the child Resource to be attached.
     Returns:
         str: A unique string which can be a key to the child Resource.
@@ -688,6 +674,7 @@ def run(hs):
 
     @defer.inlineCallbacks
     def phone_stats_home():
+        logger.info("Gathering stats for reporting")
         now = int(hs.get_clock().time())
         uptime = int(now - start_time)
         if uptime < 0:
@@ -699,8 +686,8 @@ def run(hs):
         stats["uptime_seconds"] = uptime
         stats["total_users"] = yield hs.get_datastore().count_all_users()
 
-        all_rooms = yield hs.get_datastore().get_rooms(False)
-        stats["total_room_count"] = len(all_rooms)
+        room_count = yield hs.get_datastore().get_room_count()
+        stats["total_room_count"] = room_count
 
         stats["daily_active_users"] = yield hs.get_datastore().count_daily_users()
         daily_messages = yield hs.get_datastore().count_daily_messages()
@@ -718,9 +705,12 @@ def run(hs):
 
     if hs.config.report_stats:
         phone_home_task = task.LoopingCall(phone_stats_home)
+        logger.info("Scheduling stats reporting for 24 hour intervals")
         phone_home_task.start(60 * 60 * 24, now=False)
 
     def in_thread():
+        # Uncomment to enable tracing of log context changes.
+        # sys.settrace(logcontext_tracer)
         with LoggingContext("run"):
             change_resource_limit(hs.config.soft_file_limit)
             reactor.run()