Merge branch 'develop' into rav/enforce_report_api

14 files changed, 110 insertions, 57 deletions

diff --git a/synapse/config/_base.py b/synapse/config/_base.py
index b748ed2b0a..3d2e90dd5b 100644
--- a/synapse/config/_base.py
+++ b/synapse/config/_base.py
@@ -16,11 +16,12 @@
 import argparse
 import errno
 import os
-import yaml
 from textwrap import dedent
 
 from six import integer_types
 
+import yaml
+
 
 class ConfigError(Exception):
     pass
diff --git a/synapse/config/api.py b/synapse/config/api.py
index 20ba33226a..403d96ba76 100644
--- a/synapse/config/api.py
+++ b/synapse/config/api.py
@@ -12,10 +12,10 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-from ._base import Config
-
 from synapse.api.constants import EventTypes
 
+from ._base import Config
+
 
 class ApiConfig(Config):
 
diff --git a/synapse/config/appservice.py b/synapse/config/appservice.py
index 277305e184..3b161d708a 100644
--- a/synapse/config/appservice.py
+++ b/synapse/config/appservice.py
@@ -12,17 +12,19 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-from ._base import Config, ConfigError
-
-from synapse.appservice import ApplicationService
-from synapse.types import UserID
-
-import yaml
 import logging
 
 from six import string_types
 from six.moves.urllib import parse as urlparse
 
+import yaml
+from netaddr import IPSet
+
+from synapse.appservice import ApplicationService
+from synapse.types import UserID
+
+from ._base import Config, ConfigError
+
 logger = logging.getLogger(__name__)
 
 
@@ -154,6 +156,13 @@ def _load_appservice(hostname, as_info, config_filename):
             " will not receive events or queries.",
             config_filename,
         )
+
+    ip_range_whitelist = None
+    if as_info.get('ip_range_whitelist'):
+        ip_range_whitelist = IPSet(
+            as_info.get('ip_range_whitelist')
+        )
+
     return ApplicationService(
         token=as_info["as_token"],
         hostname=hostname,
@@ -163,5 +172,6 @@ def _load_appservice(hostname, as_info, config_filename):
         sender=user_id,
         id=as_info["id"],
         protocols=protocols,
-        rate_limited=rate_limited
+        rate_limited=rate_limited,
+        ip_range_whitelist=ip_range_whitelist,
     )
diff --git a/synapse/config/consent_config.py b/synapse/config/consent_config.py
index 8f6ed73328..e22c731aad 100644
--- a/synapse/config/consent_config.py
+++ b/synapse/config/consent_config.py
@@ -18,6 +18,9 @@ from ._base import Config
 DEFAULT_CONFIG = """\
 # User Consent configuration
 #
+# for detailed instructions, see
+# https://github.com/matrix-org/synapse/blob/master/docs/consent_tracking.md
+#
 # Parts of this section are required if enabling the 'consent' resource under
 # 'listeners', in particular 'template_dir' and 'version'.
 #
diff --git a/synapse/config/homeserver.py b/synapse/config/homeserver.py
index 1dea2ad024..2fd9c48abf 100644
--- a/synapse/config/homeserver.py
+++ b/synapse/config/homeserver.py
@@ -13,32 +13,32 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-from .tls import TlsConfig
-from .server import ServerConfig
-from .logger import LoggingConfig
-from .database import DatabaseConfig
-from .ratelimiting import RatelimitConfig
-from .repository import ContentRepositoryConfig
-from .captcha import CaptchaConfig
-from .voip import VoipConfig
-from .registration import RegistrationConfig
-from .metrics import MetricsConfig
 from .api import ApiConfig
 from .appservice import AppServiceConfig
-from .key import KeyConfig
-from .saml2 import SAML2Config
+from .captcha import CaptchaConfig
 from .cas import CasConfig
-from .password import PasswordConfig
+from .consent_config import ConsentConfig
+from .database import DatabaseConfig
+from .emailconfig import EmailConfig
+from .groups import GroupsConfig
 from .jwt import JWTConfig
+from .key import KeyConfig
+from .logger import LoggingConfig
+from .metrics import MetricsConfig
+from .password import PasswordConfig
 from .password_auth_providers import PasswordAuthProviderConfig
-from .emailconfig import EmailConfig
-from .workers import WorkerConfig
 from .push import PushConfig
+from .ratelimiting import RatelimitConfig
+from .registration import RegistrationConfig
+from .repository import ContentRepositoryConfig
+from .saml2 import SAML2Config
+from .server import ServerConfig
+from .server_notices_config import ServerNoticesConfig
 from .spam_checker import SpamCheckerConfig
-from .groups import GroupsConfig
+from .tls import TlsConfig
 from .user_directory import UserDirectoryConfig
-from .consent_config import ConsentConfig
-from .server_notices_config import ServerNoticesConfig
+from .voip import VoipConfig
+from .workers import WorkerConfig
 
 
 class HomeServerConfig(TlsConfig, ServerConfig, DatabaseConfig, LoggingConfig,
diff --git a/synapse/config/jwt.py b/synapse/config/jwt.py
index 47f145c589..51e7f7e003 100644
--- a/synapse/config/jwt.py
+++ b/synapse/config/jwt.py
@@ -15,7 +15,6 @@
 
 from ._base import Config, ConfigError
 
-
 MISSING_JWT = (
     """Missing jwt library. This is required for jwt login.
 
diff --git a/synapse/config/key.py b/synapse/config/key.py
index d1382ad9ac..279c47bb48 100644
--- a/synapse/config/key.py
+++ b/synapse/config/key.py
@@ -13,21 +13,24 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-from ._base import Config, ConfigError
+import hashlib
+import logging
+import os
 
-from synapse.util.stringutils import random_string
 from signedjson.key import (
-    generate_signing_key, is_signing_algorithm_supported,
-    decode_signing_key_base64, decode_verify_key_bytes,
-    read_signing_keys, write_signing_keys, NACL_ED25519
+    NACL_ED25519,
+    decode_signing_key_base64,
+    decode_verify_key_bytes,
+    generate_signing_key,
+    is_signing_algorithm_supported,
+    read_signing_keys,
+    write_signing_keys,
 )
 from unpaddedbase64 import decode_base64
-from synapse.util.stringutils import random_string_with_symbols
 
-import os
-import hashlib
-import logging
+from synapse.util.stringutils import random_string, random_string_with_symbols
 
+from ._base import Config, ConfigError
 
 logger = logging.getLogger(__name__)
 
diff --git a/synapse/config/logger.py b/synapse/config/logger.py
index 6a7228dc2f..a87b11a1df 100644
--- a/synapse/config/logger.py
+++ b/synapse/config/logger.py
@@ -12,17 +12,22 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-
-from ._base import Config
-from synapse.util.logcontext import LoggingContextFilter
-from twisted.logger import globalLogBeginner, STDLibLogObserver
 import logging
 import logging.config
-import yaml
-from string import Template
 import os
 import signal
+import sys
+from string import Template
+
+import yaml
+
+from twisted.logger import STDLibLogObserver, globalLogBeginner
 
+import synapse
+from synapse.util.logcontext import LoggingContextFilter
+from synapse.util.versionstring import get_version_string
+
+from ._base import Config
 
 DEFAULT_LOG_CONFIG = Template("""
 version: 1
@@ -202,6 +207,15 @@ def setup_logging(config, use_worker_options=False):
     if getattr(signal, "SIGHUP"):
         signal.signal(signal.SIGHUP, sighup)
 
+    # make sure that the first thing we log is a thing we can grep backwards
+    # for
+    logging.warn("***** STARTING SERVER *****")
+    logging.warn(
+        "Server %s version %s",
+        sys.argv[0], get_version_string(synapse),
+    )
+    logging.info("Server hostname: %s", config.server_name)
+
     # It's critical to point twisted's internal logging somewhere, otherwise it
     # stacks up and leaks kup to 64K object;
     # see: https://twistedmatrix.com/trac/ticket/8164
diff --git a/synapse/config/password_auth_providers.py b/synapse/config/password_auth_providers.py
index 6602c5b4c7..f4066abc28 100644
--- a/synapse/config/password_auth_providers.py
+++ b/synapse/config/password_auth_providers.py
@@ -13,10 +13,10 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-from ._base import Config
-
 from synapse.util.module_loader import load_module
 
+from ._base import Config
+
 LDAP_PROVIDER = 'ldap_auth_provider.LdapAuthProvider'
 
 
diff --git a/synapse/config/registration.py b/synapse/config/registration.py
index c5384b3ad4..0fb964eb67 100644
--- a/synapse/config/registration.py
+++ b/synapse/config/registration.py
@@ -13,11 +13,11 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-from ._base import Config
+from distutils.util import strtobool
 
 from synapse.util.stringutils import random_string_with_symbols
 
-from distutils.util import strtobool
+from ._base import Config
 
 
 class RegistrationConfig(Config):
diff --git a/synapse/config/repository.py b/synapse/config/repository.py
index 25ea77738a..fc909c1fac 100644
--- a/synapse/config/repository.py
+++ b/synapse/config/repository.py
@@ -13,11 +13,11 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-from ._base import Config, ConfigError
 from collections import namedtuple
 
 from synapse.util.module_loader import load_module
 
+from ._base import Config, ConfigError
 
 MISSING_NETADDR = (
     "Missing netaddr library. This is required for URL preview API."
@@ -250,6 +250,9 @@ class ContentRepositoryConfig(Config):
         # - '192.168.0.0/16'
         # - '100.64.0.0/10'
         # - '169.254.0.0/16'
+        # - '::1/128'
+        # - 'fe80::/64'
+        # - 'fc00::/7'
         #
         # List of IP address CIDR ranges that the URL preview spider is allowed
         # to access even if they are specified in url_preview_ip_range_blacklist.
diff --git a/synapse/config/server.py b/synapse/config/server.py
index 8f0b6d1f28..18102656b0 100644
--- a/synapse/config/server.py
+++ b/synapse/config/server.py
@@ -14,13 +14,25 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+import logging
+
+from synapse.http.endpoint import parse_and_validate_server_name
+
 from ._base import Config, ConfigError
 
+logger = logging.Logger(__name__)
+
 
 class ServerConfig(Config):
 
     def read_config(self, config):
         self.server_name = config["server_name"]
+
+        try:
+            parse_and_validate_server_name(self.server_name)
+        except ValueError as e:
+            raise ConfigError(str(e))
+
         self.pid_file = self.abspath(config.get("pid_file"))
         self.web_client = config["web_client"]
         self.web_client_location = config.get("web_client_location", None)
@@ -138,6 +150,12 @@ class ServerConfig(Config):
 
         metrics_port = config.get("metrics_port")
         if metrics_port:
+            logger.warn(
+                ("The metrics_port configuration option is deprecated in Synapse 0.31 "
+                 "in favour of a listener. Please see "
+                 "http://github.com/matrix-org/synapse/blob/master/docs/metrics-howto.rst"
+                 " on how to configure the new listener."))
+
             self.listeners.append({
                 "port": metrics_port,
                 "bind_addresses": [config.get("metrics_bind_host", "127.0.0.1")],
@@ -152,8 +170,8 @@ class ServerConfig(Config):
             })
 
     def default_config(self, server_name, **kwargs):
-        if ":" in server_name:
-            bind_port = int(server_name.split(":")[1])
+        _, bind_port = parse_and_validate_server_name(server_name)
+        if bind_port is not None:
             unsecure_port = bind_port - 400
         else:
             bind_port = 8448
diff --git a/synapse/config/server_notices_config.py b/synapse/config/server_notices_config.py
index be1d1f762c..3c39850ac6 100644
--- a/synapse/config/server_notices_config.py
+++ b/synapse/config/server_notices_config.py
@@ -12,9 +12,10 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-from ._base import Config
 from synapse.types import UserID
 
+from ._base import Config
+
 DEFAULT_CONFIG = """\
 # Server Notices room configuration
 #
diff --git a/synapse/config/tls.py b/synapse/config/tls.py
index b66154bc7c..fef1ea99cb 100644
--- a/synapse/config/tls.py
+++ b/synapse/config/tls.py
@@ -13,15 +13,16 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-from ._base import Config
-
-from OpenSSL import crypto
-import subprocess
 import os
-
+import subprocess
 from hashlib import sha256
+
 from unpaddedbase64 import encode_base64
 
+from OpenSSL import crypto
+
+from ._base import Config
+
 GENERATE_DH_PARAMS = False