diff --git a/synapse/config/oidc_config.py b/synapse/config/oidc_config.py
index 4c24c50629..9d8196d8c3 100644
--- a/synapse/config/oidc_config.py
+++ b/synapse/config/oidc_config.py
@@ -198,9 +198,9 @@ class OIDCConfig(Config):
# user_mapping_provider:
# config:
# subject_claim: "id"
- # localpart_template: "{{ user.login }}"
- # display_name_template: "{{ user.name }}"
- # email_template: "{{ user.email }}"
+ # localpart_template: "{{{{ user.login }}}}"
+ # display_name_template: "{{{{ user.name }}}}"
+ # email_template: "{{{{ user.email }}}}"
# For use with Keycloak
#
@@ -227,8 +227,8 @@ class OIDCConfig(Config):
# user_mapping_provider:
# config:
# subject_claim: "id"
- # localpart_template: "{{ user.login }}"
- # display_name_template: "{{ user.name }}"
+ # localpart_template: "{{{{ user.login }}}}"
+ # display_name_template: "{{{{ user.name }}}}"
""".format(
mapping_provider=DEFAULT_USER_MAPPING_PROVIDER
)
diff --git a/synapse/config/registration.py b/synapse/config/registration.py
index c96530d4e3..3472afadd9 100644
--- a/synapse/config/registration.py
+++ b/synapse/config/registration.py
@@ -355,6 +355,8 @@ class RegistrationConfig(Config):
# By default, any room aliases included in this list will be created
# as a publicly joinable room when the first user registers for the
# homeserver. This behaviour can be customised with the settings below.
+ # If the room already exists, make certain it is a publicly joinable
+ # room. The join rule of the room must be set to 'public'.
#
#auto_join_rooms:
# - "#example:example.com"
diff --git a/synapse/config/repository.py b/synapse/config/repository.py
index 31e3f7148b..ef86450ed2 100644
--- a/synapse/config/repository.py
+++ b/synapse/config/repository.py
@@ -17,9 +17,7 @@ import os
from collections import namedtuple
from typing import Dict, List
-from netaddr import IPSet
-
-from synapse.config.server import DEFAULT_IP_RANGE_BLACKLIST
+from synapse.config.server import DEFAULT_IP_RANGE_BLACKLIST, generate_ip_set
from synapse.python_dependencies import DependencyException, check_requirements
from synapse.util.module_loader import load_module
@@ -193,16 +191,17 @@ class ContentRepositoryConfig(Config):
"to work"
)
- self.url_preview_ip_range_blacklist = IPSet(
- config["url_preview_ip_range_blacklist"]
- )
-
# we always blacklist '0.0.0.0' and '::', which are supposed to be
# unroutable addresses.
- self.url_preview_ip_range_blacklist.update(["0.0.0.0", "::"])
+ self.url_preview_ip_range_blacklist = generate_ip_set(
+ config["url_preview_ip_range_blacklist"],
+ ["0.0.0.0", "::"],
+ config_path=("url_preview_ip_range_blacklist",),
+ )
- self.url_preview_ip_range_whitelist = IPSet(
- config.get("url_preview_ip_range_whitelist", ())
+ self.url_preview_ip_range_whitelist = generate_ip_set(
+ config.get("url_preview_ip_range_whitelist", ()),
+ config_path=("url_preview_ip_range_whitelist",),
)
self.url_preview_url_blacklist = config.get("url_preview_url_blacklist", ())
diff --git a/synapse/config/server.py b/synapse/config/server.py
index b76afce5e5..cb0d3c9901 100644
--- a/synapse/config/server.py
+++ b/synapse/config/server.py
@@ -15,6 +15,7 @@
# See the License for the specific language governing permissions and
# limitations under the License.
+import itertools
import logging
import os.path
import re
@@ -23,7 +24,7 @@ from typing import Any, Dict, Iterable, List, Optional, Set
import attr
import yaml
-from netaddr import IPSet
+from netaddr import AddrFormatError, IPNetwork, IPSet
from synapse.api.room_versions import KNOWN_ROOM_VERSIONS
from synapse.util.stringutils import parse_and_validate_server_name
@@ -40,6 +41,66 @@ logger = logging.Logger(__name__)
# in the list.
DEFAULT_BIND_ADDRESSES = ["::", "0.0.0.0"]
+
+def _6to4(network: IPNetwork) -> IPNetwork:
+ """Convert an IPv4 network into a 6to4 IPv6 network per RFC 3056."""
+
+ # 6to4 networks consist of:
+ # * 2002 as the first 16 bits
+ # * The first IPv4 address in the network hex-encoded as the next 32 bits
+ # * The new prefix length needs to include the bits from the 2002 prefix.
+ hex_network = hex(network.first)[2:]
+ hex_network = ("0" * (8 - len(hex_network))) + hex_network
+ return IPNetwork(
+ "2002:%s:%s::/%d" % (hex_network[:4], hex_network[4:], 16 + network.prefixlen,)
+ )
+
+
+def generate_ip_set(
+ ip_addresses: Optional[Iterable[str]],
+ extra_addresses: Optional[Iterable[str]] = None,
+ config_path: Optional[Iterable[str]] = None,
+) -> IPSet:
+ """
+ Generate an IPSet from a list of IP addresses or CIDRs.
+
+ Additionally, for each IPv4 network in the list of IP addresses, also
+ includes the corresponding IPv6 networks.
+
+ This includes:
+
+ * IPv4-Compatible IPv6 Address (see RFC 4291, section 2.5.5.1)
+ * IPv4-Mapped IPv6 Address (see RFC 4291, section 2.5.5.2)
+ * 6to4 Address (see RFC 3056, section 2)
+
+ Args:
+ ip_addresses: An iterable of IP addresses or CIDRs.
+ extra_addresses: An iterable of IP addresses or CIDRs.
+ config_path: The path in the configuration for error messages.
+
+ Returns:
+ A new IP set.
+ """
+ result = IPSet()
+ for ip in itertools.chain(ip_addresses or (), extra_addresses or ()):
+ try:
+ network = IPNetwork(ip)
+ except AddrFormatError as e:
+ raise ConfigError(
+ "Invalid IP range provided: %s." % (ip,), config_path
+ ) from e
+ result.add(network)
+
+ # It is possible that these already exist in the set, but that's OK.
+ if ":" not in str(network):
+ result.add(IPNetwork(network).ipv6(ipv4_compatible=True))
+ result.add(IPNetwork(network).ipv6(ipv4_compatible=False))
+ result.add(_6to4(network))
+
+ return result
+
+
+# IP ranges that are considered private / unroutable / don't make sense.
DEFAULT_IP_RANGE_BLACKLIST = [
# Localhost
"127.0.0.0/8",
@@ -53,6 +114,8 @@ DEFAULT_IP_RANGE_BLACKLIST = [
"192.0.0.0/24",
# Link-local networks.
"169.254.0.0/16",
+ # Formerly used for 6to4 relay.
+ "192.88.99.0/24",
# Testing networks.
"198.18.0.0/15",
"192.0.2.0/24",
@@ -66,6 +129,12 @@ DEFAULT_IP_RANGE_BLACKLIST = [
"fe80::/10",
# Unique local addresses.
"fc00::/7",
+ # Testing networks.
+ "2001:db8::/32",
+ # Multicast.
+ "ff00::/8",
+ # Site-local addresses
+ "fec0::/10",
]
DEFAULT_ROOM_VERSION = "6"
@@ -294,17 +363,15 @@ class ServerConfig(Config):
)
# Attempt to create an IPSet from the given ranges
- try:
- self.ip_range_blacklist = IPSet(ip_range_blacklist)
- except Exception as e:
- raise ConfigError("Invalid range(s) provided in ip_range_blacklist.") from e
+
# Always blacklist 0.0.0.0, ::
- self.ip_range_blacklist.update(["0.0.0.0", "::"])
+ self.ip_range_blacklist = generate_ip_set(
+ ip_range_blacklist, ["0.0.0.0", "::"], config_path=("ip_range_blacklist",)
+ )
- try:
- self.ip_range_whitelist = IPSet(config.get("ip_range_whitelist", ()))
- except Exception as e:
- raise ConfigError("Invalid range(s) provided in ip_range_whitelist.") from e
+ self.ip_range_whitelist = generate_ip_set(
+ config.get("ip_range_whitelist", ()), config_path=("ip_range_whitelist",)
+ )
# The federation_ip_range_blacklist is used for backwards-compatibility
# and only applies to federation and identity servers. If it is not given,
@@ -312,14 +379,12 @@ class ServerConfig(Config):
federation_ip_range_blacklist = config.get(
"federation_ip_range_blacklist", ip_range_blacklist
)
- try:
- self.federation_ip_range_blacklist = IPSet(federation_ip_range_blacklist)
- except Exception as e:
- raise ConfigError(
- "Invalid range(s) provided in federation_ip_range_blacklist."
- ) from e
# Always blacklist 0.0.0.0, ::
- self.federation_ip_range_blacklist.update(["0.0.0.0", "::"])
+ self.federation_ip_range_blacklist = generate_ip_set(
+ federation_ip_range_blacklist,
+ ["0.0.0.0", "::"],
+ config_path=("federation_ip_range_blacklist",),
+ )
self.start_pushers = config.get("start_pushers", True)
diff --git a/synapse/config/sso.py b/synapse/config/sso.py
index 939eeac6de..6c60c6fea4 100644
--- a/synapse/config/sso.py
+++ b/synapse/config/sso.py
@@ -106,8 +106,7 @@ class SSOConfig(Config):
#
# When rendering, this template is given the following variables:
# * redirect_url: the URL that the user will be redirected to after
- # login. Needs manual escaping (see
- # https://jinja.palletsprojects.com/en/2.11.x/templates/#html-escaping).
+ # login.
#
# * server_name: the homeserver's name.
#
@@ -185,15 +184,12 @@ class SSOConfig(Config):
#
# When rendering, this template is given the following variables:
#
- # * redirect_url: the URL the user is about to be redirected to. Needs
- # manual escaping (see
- # https://jinja.palletsprojects.com/en/2.11.x/templates/#html-escaping).
+ # * redirect_url: the URL the user is about to be redirected to.
#
# * display_url: the same as `redirect_url`, but with the query
# parameters stripped. The intention is to have a
# human-readable URL to show to users, not to use it as
- # the final address to redirect to. Needs manual escaping
- # (see https://jinja.palletsprojects.com/en/2.11.x/templates/#html-escaping).
+ # the final address to redirect to.
#
# * server_name: the homeserver's name.
#
@@ -213,9 +209,7 @@ class SSOConfig(Config):
# process: 'sso_auth_confirm.html'.
#
# When rendering, this template is given the following variables:
- # * redirect_url: the URL the user is about to be redirected to. Needs
- # manual escaping (see
- # https://jinja.palletsprojects.com/en/2.11.x/templates/#html-escaping).
+ # * redirect_url: the URL the user is about to be redirected to.
#
# * description: the operation which the user is being asked to confirm
#
diff --git a/synapse/handlers/federation.py b/synapse/handlers/federation.py
index fa56b31438..61bc0c8bc6 100644
--- a/synapse/handlers/federation.py
+++ b/synapse/handlers/federation.py
@@ -1697,7 +1697,9 @@ class FederationHandler(BaseHandler):
# We retrieve the room member handler here as to not cause a cyclic dependency
member_handler = self.hs.get_room_member_handler()
- member_handler.ratelimit_invite(event.room_id, event.state_key)
+ # We don't rate limit based on room ID, as that should be done by
+ # sending server.
+ member_handler.ratelimit_invite(None, event.state_key)
# keep a record of the room version, if we don't yet know it.
# (this may get overwritten if we later get a different room version in a
diff --git a/synapse/handlers/room_member.py b/synapse/handlers/room_member.py
index a92f7ba012..eb3193e554 100644
--- a/synapse/handlers/room_member.py
+++ b/synapse/handlers/room_member.py
@@ -198,10 +198,14 @@ class RoomMemberHandler(metaclass=abc.ABCMeta):
"""
raise NotImplementedError()
- def ratelimit_invite(self, room_id: str, invitee_user_id: str):
+ def ratelimit_invite(self, room_id: Optional[str], invitee_user_id: str):
"""Ratelimit invites by room and by target user.
+
+ If room ID is missing then we just rate limit by target user.
"""
- self._invites_per_room_limiter.ratelimit(room_id)
+ if room_id:
+ self._invites_per_room_limiter.ratelimit(room_id)
+
self._invites_per_user_limiter.ratelimit(invitee_user_id)
async def _local_membership_update(
@@ -452,7 +456,9 @@ class RoomMemberHandler(metaclass=abc.ABCMeta):
if effective_membership_state == Membership.INVITE:
target_id = target.to_string()
if ratelimit:
- self.ratelimit_invite(room_id, target_id)
+ # Don't ratelimit application services.
+ if not requester.app_service or requester.app_service.is_rate_limited():
+ self.ratelimit_invite(room_id, target_id)
# block any attempts to invite the server notices mxid
if target_id == self._server_notices_mxid:
diff --git a/synapse/http/federation/matrix_federation_agent.py b/synapse/http/federation/matrix_federation_agent.py
index 4c06a117d3..113fd47134 100644
--- a/synapse/http/federation/matrix_federation_agent.py
+++ b/synapse/http/federation/matrix_federation_agent.py
@@ -323,12 +323,19 @@ class MatrixHostnameEndpoint:
if port or _is_ip_literal(host):
return [Server(host, port or 8448)]
+ logger.debug("Looking up SRV record for %s", host.decode(errors="replace"))
server_list = await self._srv_resolver.resolve_service(b"_matrix._tcp." + host)
if server_list:
+ logger.debug(
+ "Got %s from SRV lookup for %s",
+ ", ".join(map(str, server_list)),
+ host.decode(errors="replace"),
+ )
return server_list
# No SRV records, so we fallback to host and 8448
+ logger.debug("No SRV records for %s", host.decode(errors="replace"))
return [Server(host, 8448)]
diff --git a/synapse/python_dependencies.py b/synapse/python_dependencies.py
index 60e6793c8d..0857efbe71 100644
--- a/synapse/python_dependencies.py
+++ b/synapse/python_dependencies.py
@@ -86,8 +86,12 @@ REQUIREMENTS = [
CONDITIONAL_REQUIREMENTS = {
"matrix-synapse-ldap3": ["matrix-synapse-ldap3>=0.1"],
- # we use execute_values with the fetch param, which arrived in psycopg 2.8.
- "postgres": ["psycopg2>=2.8"],
+ "postgres": [
+ # we use execute_values with the fetch param, which arrived in psycopg 2.8.
+ "psycopg2>=2.8 ; platform_python_implementation != 'PyPy'",
+ "psycopg2cffi>=2.8 ; platform_python_implementation == 'PyPy'",
+ "psycopg2cffi-compat==1.1 ; platform_python_implementation == 'PyPy'",
+ ],
# ACME support is required to provision TLS certificates from authorities
# that use the protocol, such as Let's Encrypt.
"acme": [
diff --git a/synapse/res/templates/sso_auth_account_details.html b/synapse/res/templates/sso_auth_account_details.html
index 7ad58ad214..f4fdc40b22 100644
--- a/synapse/res/templates/sso_auth_account_details.html
+++ b/synapse/res/templates/sso_auth_account_details.html
@@ -35,6 +35,19 @@
font-size: 12px;
}
+ .username_input.invalid {
+ border-color: #FE2928;
+ }
+
+ .username_input.invalid input, .username_input.invalid label {
+ color: #FE2928;
+ }
+
+ .username_input div, .username_input input {
+ line-height: 18px;
+ font-size: 14px;
+ }
+
.username_input label {
position: absolute;
top: -5px;
@@ -104,6 +117,15 @@
display: block;
margin-top: 8px;
}
+
+ output {
+ padding: 0 14px;
+ display: block;
+ }
+
+ output.error {
+ color: #FE2928;
+ }
</style>
</head>
<body>
@@ -113,12 +135,13 @@
</header>
<main>
<form method="post" class="form__input" id="form">
- <div class="username_input">
+ <div class="username_input" id="username_input">
<label for="field-username">Username</label>
<div class="prefix">@</div>
- <input type="text" name="username" id="field-username" autofocus required pattern="[a-z0-9\-=_\/\.]+">
+ <input type="text" name="username" id="field-username" autofocus>
<div class="postfix">:{{ server_name }}</div>
</div>
+ <output for="username_input" id="field-username-output"></output>
<input type="submit" value="Continue" class="primary-button">
{% if user_attributes.avatar_url or user_attributes.display_name or user_attributes.emails %}
<section class="idp-pick-details">
diff --git a/synapse/res/templates/sso_auth_account_details.js b/synapse/res/templates/sso_auth_account_details.js
index deef419bb6..3c45df9078 100644
--- a/synapse/res/templates/sso_auth_account_details.js
+++ b/synapse/res/templates/sso_auth_account_details.js
@@ -1,14 +1,24 @@
const usernameField = document.getElementById("field-username");
+const usernameOutput = document.getElementById("field-username-output");
+const form = document.getElementById("form");
+
+// needed to validate on change event when no input was changed
+let needsValidation = true;
+let isValid = false;
function throttle(fn, wait) {
let timeout;
- return function() {
+ const throttleFn = function() {
const args = Array.from(arguments);
if (timeout) {
clearTimeout(timeout);
}
timeout = setTimeout(fn.bind.apply(fn, [null].concat(args)), wait);
- }
+ };
+ throttleFn.cancelQueued = function() {
+ clearTimeout(timeout);
+ };
+ return throttleFn;
}
function checkUsernameAvailable(username) {
@@ -16,14 +26,14 @@ function checkUsernameAvailable(username) {
return fetch(check_uri, {
// include the cookie
"credentials": "same-origin",
- }).then((response) => {
+ }).then(function(response) {
if(!response.ok) {
// for non-200 responses, raise the body of the response as an exception
return response.text().then((text) => { throw new Error(text); });
} else {
return response.json();
}
- }).then((json) => {
+ }).then(function(json) {
if(json.error) {
return {message: json.error};
} else if(json.available) {
@@ -34,33 +44,49 @@ function checkUsernameAvailable(username) {
});
}
+const allowedUsernameCharacters = new RegExp("^[a-z0-9\\.\\_\\-\\/\\=]+$");
+const allowedCharactersString = "lowercase letters, digits, ., _, -, /, =";
+
+function reportError(error) {
+ throttledCheckUsernameAvailable.cancelQueued();
+ usernameOutput.innerText = error;
+ usernameOutput.classList.add("error");
+ usernameField.parentElement.classList.add("invalid");
+ usernameField.focus();
+}
+
function validateUsername(username) {
- usernameField.setCustomValidity("");
- if (usernameField.validity.valueMissing) {
- usernameField.setCustomValidity("Please provide a username");
- return;
+ isValid = false;
+ needsValidation = false;
+ usernameOutput.innerText = "";
+ usernameField.parentElement.classList.remove("invalid");
+ usernameOutput.classList.remove("error");
+ if (!username) {
+ return reportError("Please provide a username");
}
- if (usernameField.validity.patternMismatch) {
- usernameField.setCustomValidity("Invalid username, please only use " + allowedCharactersString);
- return;
+ if (username.length > 255) {
+ return reportError("Too long, please choose something shorter");
}
- usernameField.setCustomValidity("Checking if username is available …");
+ if (!allowedUsernameCharacters.test(username)) {
+ return reportError("Invalid username, please only use " + allowedCharactersString);
+ }
+ usernameOutput.innerText = "Checking if username is available …";
throttledCheckUsernameAvailable(username);
}
const throttledCheckUsernameAvailable = throttle(function(username) {
- const handleError = function(err) {
+ const handleError = function(err) {
// don't prevent form submission on error
- usernameField.setCustomValidity("");
- console.log(err.message);
+ usernameOutput.innerText = "";
+ isValid = true;
};
try {
checkUsernameAvailable(username).then(function(result) {
if (!result.available) {
- usernameField.setCustomValidity(result.message);
- usernameField.reportValidity();
+ reportError(result.message);
} else {
- usernameField.setCustomValidity("");
+ isValid = true;
+ usernameOutput.innerText = "";
}
}, handleError);
} catch (err) {
@@ -68,9 +94,23 @@ const throttledCheckUsernameAvailable = throttle(function(username) {
}
}, 500);
+form.addEventListener("submit", function(evt) {
+ if (needsValidation) {
+ validateUsername(usernameField.value);
+ evt.preventDefault();
+ return;
+ }
+ if (!isValid) {
+ evt.preventDefault();
+ usernameField.focus();
+ return;
+ }
+});
usernameField.addEventListener("input", function(evt) {
validateUsername(usernameField.value);
});
usernameField.addEventListener("change", function(evt) {
- validateUsername(usernameField.value);
+ if (needsValidation) {
+ validateUsername(usernameField.value);
+ }
});
diff --git a/synapse/storage/engines/__init__.py b/synapse/storage/engines/__init__.py
index 035f9ea6e9..d15ccfacde 100644
--- a/synapse/storage/engines/__init__.py
+++ b/synapse/storage/engines/__init__.py
@@ -12,7 +12,6 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
-import platform
from ._base import BaseDatabaseEngine, IncorrectDatabaseSetup
from .postgres import PostgresEngine
@@ -28,11 +27,8 @@ def create_engine(database_config) -> BaseDatabaseEngine:
return Sqlite3Engine(sqlite3, database_config)
if name == "psycopg2":
- # pypy requires psycopg2cffi rather than psycopg2
- if platform.python_implementation() == "PyPy":
- import psycopg2cffi as psycopg2 # type: ignore
- else:
- import psycopg2 # type: ignore
+ # Note that psycopg2cffi-compat provides the psycopg2 module on pypy.
+ import psycopg2 # type: ignore
return PostgresEngine(psycopg2, database_config)
diff --git a/synapse/storage/engines/sqlite.py b/synapse/storage/engines/sqlite.py
index 5db0f0b520..b3d1834efb 100644
--- a/synapse/storage/engines/sqlite.py
+++ b/synapse/storage/engines/sqlite.py
@@ -12,6 +12,7 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
+import platform
import struct
import threading
import typing
@@ -30,6 +31,11 @@ class Sqlite3Engine(BaseDatabaseEngine["sqlite3.Connection"]):
database = database_config.get("args", {}).get("database")
self._is_in_memory = database in (None, ":memory:",)
+ if platform.python_implementation() == "PyPy":
+ # pypy's sqlite3 module doesn't handle bytearrays, convert them
+ # back to bytes.
+ database_module.register_adapter(bytearray, lambda array: bytes(array))
+
# The current max state_group, or None if we haven't looked
# in the DB yet.
self._current_state_group_id = None
diff --git a/synapse/visibility.py b/synapse/visibility.py
index ec50e7e977..4a5df293a4 100644
--- a/synapse/visibility.py
+++ b/synapse/visibility.py
@@ -233,7 +233,7 @@ async def filter_events_for_client(
elif visibility == HistoryVisibility.SHARED and is_peeking:
# if the visibility is shared, users cannot see the event unless
- # they have *subequently* joined the room (or were members at the
+ # they have *subsequently* joined the room (or were members at the
# time, of course)
#
# XXX: if the user has subsequently joined and then left again,
|
