Merge branch 'develop' into madlittlemods/re-use-work-to-grab-state-from-previous-group github/madlittlemods/re-use-work-to-grab-state-from-previous-group madlittlemods/re-use-work-to-grab-state-from-previous-group

21 files changed, 169 insertions, 59 deletions

diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml
index 51cbeb3298..f84a4ef644 100644
--- a/.github/workflows/tests.yml
+++ b/.github/workflows/tests.yml
@@ -314,8 +314,9 @@ jobs:
       # There aren't wheels for some of the older deps, so we need to install
       # their build dependencies
       - run: |
+          sudo apt update
           sudo apt-get -qq install build-essential libffi-dev python-dev \
-          libxml2-dev libxslt-dev xmlsec1 zlib1g-dev libjpeg-dev libwebp-dev
+            libxml2-dev libxslt-dev xmlsec1 zlib1g-dev libjpeg-dev libwebp-dev
 
       - uses: actions/setup-python@v4
         with:
diff --git a/changelog.d/15611.feature b/changelog.d/15611.feature
new file mode 100644
index 0000000000..7cfb46fd0a
--- /dev/null
+++ b/changelog.d/15611.feature
@@ -0,0 +1 @@
+Add a new admin API to create a new device for a user.
diff --git a/changelog.d/15613.doc b/changelog.d/15613.doc
new file mode 100644
index 0000000000..94733facf0
--- /dev/null
+++ b/changelog.d/15613.doc
@@ -0,0 +1 @@
+Warn users that at least 3.75GB of space is needed for the nix Synapse development environment.
diff --git a/changelog.d/15614.bugfix b/changelog.d/15614.bugfix
new file mode 100644
index 0000000000..b523ae6eb1
--- /dev/null
+++ b/changelog.d/15614.bugfix
@@ -0,0 +1 @@
+Fix a bug introduced in Synapse 1.82.0 where the error message displayed when validation of the `app_service_config_files` config option fails would be incorrectly formatted.
diff --git a/changelog.d/15615.misc b/changelog.d/15615.misc
new file mode 100644
index 0000000000..a39fd0a098
--- /dev/null
+++ b/changelog.d/15615.misc
@@ -0,0 +1 @@
+Re-type config paths in `ConfigError`s to be `StrSequence`s instead of `Iterable[str]`s.
diff --git a/changelog.d/15621.misc b/changelog.d/15621.misc
new file mode 100644
index 0000000000..5d060f4dbc
--- /dev/null
+++ b/changelog.d/15621.misc
@@ -0,0 +1 @@
+Update Mutual Rooms (MSC2666) implementation to match new proposal text.
\ No newline at end of file
diff --git a/changelog.d/15626.misc b/changelog.d/15626.misc
new file mode 100644
index 0000000000..0016cdbf10
--- /dev/null
+++ b/changelog.d/15626.misc
@@ -0,0 +1 @@
+Fix the olddeps CI.
diff --git a/docs/admin_api/user_admin_api.md b/docs/admin_api/user_admin_api.md
index 6b952ba396..229942b311 100644
--- a/docs/admin_api/user_admin_api.md
+++ b/docs/admin_api/user_admin_api.md
@@ -813,6 +813,33 @@ The following fields are returned in the JSON response body:
 
 - `total` - Total number of user's devices.
 
+### Create a device
+
+Creates a new device for a specific `user_id` and `device_id`. Does nothing if the `device_id` 
+exists already.
+
+The API is:
+
+```
+POST /_synapse/admin/v2/users/<user_id>/devices
+
+{
+  "device_id": "QBUAZIFURK"
+}
+```
+
+An empty JSON dict is returned.
+
+**Parameters**
+
+The following parameters should be set in the URL:
+
+- `user_id` - fully qualified: for example, `@user:server.com`.
+
+The following fields are required in the JSON request body:
+
+- `device_id` - The device ID to create.
+
 ### Delete multiple devices
 Deletes the given devices for a specific `user_id`, and invalidates
 any access token associated with them.
diff --git a/flake.nix b/flake.nix
index 7351571e61..8d2bf779bd 100644
--- a/flake.nix
+++ b/flake.nix
@@ -1,35 +1,30 @@
-# A nix flake that sets up a complete Synapse development environment. Dependencies
+# A Nix flake that sets up a complete Synapse development environment. Dependencies
 # for the SyTest (https://github.com/matrix-org/sytest) and Complement
 # (https://github.com/matrix-org/complement) Matrix homeserver test suites are also
 # installed automatically.
 #
-# You must have already installed nix (https://nixos.org) on your system to use this.
-# nix can be installed on Linux or MacOS; NixOS is not required. Windows is not
-# directly supported, but nix can be installed inside of WSL2 or even Docker
+# You must have already installed Nix (https://nixos.org) on your system to use this.
+# Nix can be installed on Linux or MacOS; NixOS is not required. Windows is not
+# directly supported, but Nix can be installed inside of WSL2 or even Docker
 # containers. Please refer to https://nixos.org/download for details.
 #
 # You must also enable support for flakes in Nix. See the following for how to
 # do so permanently: https://nixos.wiki/wiki/Flakes#Enable_flakes
 #
+# Be warned: you'll need over 3.75 GB of free space to download all the dependencies.
+#
 # Usage:
 #
-# With nix installed, navigate to the directory containing this flake and run
+# With Nix installed, navigate to the directory containing this flake and run
 # `nix develop --impure`. The `--impure` is necessary in order to store state
 # locally from "services", such as PostgreSQL and Redis.
 #
 # You should now be dropped into a new shell with all programs and dependencies
 # availabile to you!
 #
-# You can start up pre-configured, local PostgreSQL and Redis instances by
+# You can start up pre-configured local Synapse, PostgreSQL and Redis instances by
 # running: `devenv up`. To stop them, use Ctrl-C.
 #
-# A PostgreSQL database called 'synapse' will be set up for you, along with
-# a PostgreSQL user named 'synapse_user'.
-# The 'host' can be found by running `echo $PGHOST` with the development
-# shell activated. Use these values to configure your Synapse to connect
-# to the local PostgreSQL database. You do not need to specify a password.
-# https://matrix-org.github.io/synapse/latest/postgres
-#
 # All state (the venv, postgres and redis data and config) are stored in
 # .devenv/state. Deleting a file from here and then re-entering the shell
 # will recreate these files from scratch.
@@ -66,7 +61,7 @@
         let
           pkgs = nixpkgs.legacyPackages.${system};
         in {
-          # Everything is configured via devenv - a nix module for creating declarative
+          # Everything is configured via devenv - a Nix module for creating declarative
           # developer environments. See https://devenv.sh/reference/options/ for a list
           # of all possible options.
           default = devenv.lib.mkShell {
@@ -153,11 +148,39 @@
                 # Redis is needed in order to run Synapse in worker mode.
                 services.redis.enable = true;
 
+                # Configure and start Synapse. Before starting Synapse, this shell code:
+                #  * generates a default homeserver.yaml config file if one does not exist, and
+                #  * ensures a directory containing two additional homeserver config files exists;
+                #    one to configure using the development environment's PostgreSQL as the
+                #    database backend and another for enabling Redis support.
+                process.before = ''
+                  python -m synapse.app.homeserver -c homeserver.yaml --generate-config --server-name=synapse.dev --report-stats=no
+                  mkdir -p homeserver-config-overrides.d
+                  cat > homeserver-config-overrides.d/database.yaml << EOF
+                  ## Do not edit this file. This file is generated by flake.nix
+                  database:
+                    name: psycopg2
+                    args:
+                      user: synapse_user
+                      database: synapse
+                      host: $PGHOST
+                      cp_min: 5
+                      cp_max: 10
+                  EOF
+                  cat > homeserver-config-overrides.d/redis.yaml << EOF
+                  ## Do not edit this file. This file is generated by flake.nix
+                  redis:
+                    enabled: true
+                  EOF
+                '';
+                # Start synapse when `devenv up` is run.
+                processes.synapse.exec = "poetry run python -m synapse.app.homeserver -c homeserver.yaml --config-directory homeserver-config-overrides.d";
+
                 # Define the perl modules we require to run SyTest.
                 #
                 # This list was compiled by cross-referencing https://metacpan.org/
                 # with the modules defined in './cpanfile' and then finding the
-                # corresponding nix packages on https://search.nixos.org/packages.
+                # corresponding Nix packages on https://search.nixos.org/packages.
                 #
                 # This was done until `./install-deps.pl --dryrun` produced no output.
                 env.PERL5LIB = "${with pkgs.perl536Packages; makePerlPath [
diff --git a/synapse/config/_base.py b/synapse/config/_base.py
index 2ce60610ca..1d268a1817 100644
--- a/synapse/config/_base.py
+++ b/synapse/config/_base.py
@@ -44,6 +44,7 @@ import jinja2
 import pkg_resources
 import yaml
 
+from synapse.types import StrSequence
 from synapse.util.templates import _create_mxc_to_http_filter, _format_ts_filter
 
 logger = logging.getLogger(__name__)
@@ -58,7 +59,7 @@ class ConfigError(Exception):
            the problem lies.
     """
 
-    def __init__(self, msg: str, path: Optional[Iterable[str]] = None):
+    def __init__(self, msg: str, path: Optional[StrSequence] = None):
         self.msg = msg
         self.path = path
 
diff --git a/synapse/config/_base.pyi b/synapse/config/_base.pyi
index b5cec132b4..fc51aed234 100644
--- a/synapse/config/_base.pyi
+++ b/synapse/config/_base.pyi
@@ -61,9 +61,10 @@ from synapse.config import (  # noqa: F401
     voip,
     workers,
 )
+from synapse.types import StrSequence
 
 class ConfigError(Exception):
-    def __init__(self, msg: str, path: Optional[Iterable[str]] = None):
+    def __init__(self, msg: str, path: Optional[StrSequence] = None):
         self.msg = msg
         self.path = path
 
diff --git a/synapse/config/_util.py b/synapse/config/_util.py
index dfc5d12210..acccca413b 100644
--- a/synapse/config/_util.py
+++ b/synapse/config/_util.py
@@ -11,17 +11,17 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-from typing import Any, Dict, Iterable, Type, TypeVar
+from typing import Any, Dict, Type, TypeVar
 
 import jsonschema
 from pydantic import BaseModel, ValidationError, parse_obj_as
 
 from synapse.config._base import ConfigError
-from synapse.types import JsonDict
+from synapse.types import JsonDict, StrSequence
 
 
 def validate_config(
-    json_schema: JsonDict, config: Any, config_path: Iterable[str]
+    json_schema: JsonDict, config: Any, config_path: StrSequence
 ) -> None:
     """Validates a config setting against a JsonSchema definition
 
@@ -45,7 +45,7 @@ def validate_config(
 
 
 def json_error_to_config_error(
-    e: jsonschema.ValidationError, config_path: Iterable[str]
+    e: jsonschema.ValidationError, config_path: StrSequence
 ) -> ConfigError:
     """Converts a json validation error to a user-readable ConfigError
 
diff --git a/synapse/config/appservice.py b/synapse/config/appservice.py
index fd89960e72..c2710fdf04 100644
--- a/synapse/config/appservice.py
+++ b/synapse/config/appservice.py
@@ -36,11 +36,10 @@ class AppServiceConfig(Config):
         if not isinstance(self.app_service_config_files, list) or not all(
             type(x) is str for x in self.app_service_config_files
         ):
-            # type-ignore: this function gets arbitrary json value; we do use this path.
             raise ConfigError(
                 "Expected '%s' to be a list of AS config files:"
                 % (self.app_service_config_files),
-                "app_service_config_files",
+                ("app_service_config_files",),
             )
 
         self.track_appservice_user_ips = config.get("track_appservice_user_ips", False)
diff --git a/synapse/config/oembed.py b/synapse/config/oembed.py
index 0d32aba70a..d7959639ee 100644
--- a/synapse/config/oembed.py
+++ b/synapse/config/oembed.py
@@ -19,7 +19,7 @@ from urllib import parse as urlparse
 import attr
 import pkg_resources
 
-from synapse.types import JsonDict
+from synapse.types import JsonDict, StrSequence
 
 from ._base import Config, ConfigError
 from ._util import validate_config
@@ -80,7 +80,7 @@ class OembedConfig(Config):
             )
 
     def _parse_and_validate_provider(
-        self, providers: List[JsonDict], config_path: Iterable[str]
+        self, providers: List[JsonDict], config_path: StrSequence
     ) -> Iterable[OEmbedEndpointConfig]:
         # Ensure it is the proper form.
         validate_config(
@@ -112,7 +112,7 @@ class OembedConfig(Config):
                     api_endpoint, patterns, endpoint.get("formats")
                 )
 
-    def _glob_to_pattern(self, glob: str, config_path: Iterable[str]) -> Pattern:
+    def _glob_to_pattern(self, glob: str, config_path: StrSequence) -> Pattern:
         """
         Convert the glob into a sane regular expression to match against. The
         rules followed will be slightly different for the domain portion vs.
diff --git a/synapse/config/server.py b/synapse/config/server.py
index 386c3194b8..64201238d6 100644
--- a/synapse/config/server.py
+++ b/synapse/config/server.py
@@ -27,7 +27,7 @@ from netaddr import AddrFormatError, IPNetwork, IPSet
 from twisted.conch.ssh.keys import Key
 
 from synapse.api.room_versions import KNOWN_ROOM_VERSIONS
-from synapse.types import JsonDict
+from synapse.types import JsonDict, StrSequence
 from synapse.util.module_loader import load_module
 from synapse.util.stringutils import parse_and_validate_server_name
 
@@ -73,7 +73,7 @@ def _6to4(network: IPNetwork) -> IPNetwork:
 def generate_ip_set(
     ip_addresses: Optional[Iterable[str]],
     extra_addresses: Optional[Iterable[str]] = None,
-    config_path: Optional[Iterable[str]] = None,
+    config_path: Optional[StrSequence] = None,
 ) -> IPSet:
     """
     Generate an IPSet from a list of IP addresses or CIDRs.
diff --git a/synapse/rest/admin/devices.py b/synapse/rest/admin/devices.py
index 3b2f2d9abb..11ebed9bfd 100644
--- a/synapse/rest/admin/devices.py
+++ b/synapse/rest/admin/devices.py
@@ -137,6 +137,35 @@ class DevicesRestServlet(RestServlet):
         devices = await self.device_handler.get_devices_by_user(target_user.to_string())
         return HTTPStatus.OK, {"devices": devices, "total": len(devices)}
 
+    async def on_POST(
+        self, request: SynapseRequest, user_id: str
+    ) -> Tuple[int, JsonDict]:
+        """Creates a new device for the user."""
+        await assert_requester_is_admin(self.auth, request)
+
+        target_user = UserID.from_string(user_id)
+        if not self.is_mine(target_user):
+            raise SynapseError(
+                HTTPStatus.BAD_REQUEST, "Can only create devices for local users"
+            )
+
+        u = await self.store.get_user_by_id(target_user.to_string())
+        if u is None:
+            raise NotFoundError("Unknown user")
+
+        body = parse_json_object_from_request(request)
+        device_id = body.get("device_id")
+        if not device_id:
+            raise SynapseError(HTTPStatus.BAD_REQUEST, "Missing device_id")
+        if not isinstance(device_id, str):
+            raise SynapseError(HTTPStatus.BAD_REQUEST, "device_id must be a string")
+
+        await self.device_handler.check_device_registered(
+            user_id=user_id, device_id=device_id
+        )
+
+        return HTTPStatus.CREATED, {}
+
 
 class DeleteDevicesRestServlet(RestServlet):
     """
diff --git a/synapse/rest/client/mutual_rooms.py b/synapse/rest/client/mutual_rooms.py
index 38ef4e459f..c99445da30 100644
--- a/synapse/rest/client/mutual_rooms.py
+++ b/synapse/rest/client/mutual_rooms.py
@@ -12,13 +12,14 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 import logging
-from typing import TYPE_CHECKING, Tuple
+from http import HTTPStatus
+from typing import TYPE_CHECKING, Dict, List, Tuple
 
 from synapse.api.errors import Codes, SynapseError
 from synapse.http.server import HttpServer
-from synapse.http.servlet import RestServlet
+from synapse.http.servlet import RestServlet, parse_strings_from_args
 from synapse.http.site import SynapseRequest
-from synapse.types import JsonDict, UserID
+from synapse.types import JsonDict
 
 from ._base import client_patterns
 
@@ -30,11 +31,11 @@ logger = logging.getLogger(__name__)
 
 class UserMutualRoomsServlet(RestServlet):
     """
-    GET /uk.half-shot.msc2666/user/mutual_rooms/{user_id} HTTP/1.1
+    GET /uk.half-shot.msc2666/user/mutual_rooms?user_id={user_id} HTTP/1.1
     """
 
     PATTERNS = client_patterns(
-        "/uk.half-shot.msc2666/user/mutual_rooms/(?P<user_id>[^/]*)",
+        "/uk.half-shot.msc2666/user/mutual_rooms$",
         releases=(),  # This is an unstable feature
     )
 
@@ -43,17 +44,35 @@ class UserMutualRoomsServlet(RestServlet):
         self.auth = hs.get_auth()
         self.store = hs.get_datastores().main
 
-    async def on_GET(
-        self, request: SynapseRequest, user_id: str
-    ) -> Tuple[int, JsonDict]:
-        UserID.from_string(user_id)
+    async def on_GET(self, request: SynapseRequest) -> Tuple[int, JsonDict]:
+        # twisted.web.server.Request.args is incorrectly defined as Optional[Any]
+        args: Dict[bytes, List[bytes]] = request.args  # type: ignore
+
+        user_ids = parse_strings_from_args(args, "user_id", required=True)
+
+        if len(user_ids) > 1:
+            raise SynapseError(
+                HTTPStatus.BAD_REQUEST,
+                "Duplicate user_id query parameter",
+                errcode=Codes.INVALID_PARAM,
+            )
+
+        # We don't do batching, so a batch token is illegal by default
+        if b"batch_token" in args:
+            raise SynapseError(
+                HTTPStatus.BAD_REQUEST,
+                "Unknown batch_token",
+                errcode=Codes.INVALID_PARAM,
+            )
+
+        user_id = user_ids[0]
 
         requester = await self.auth.get_user_by_req(request)
         if user_id == requester.user.to_string():
             raise SynapseError(
-                code=400,
-                msg="You cannot request a list of shared rooms with yourself",
-                errcode=Codes.FORBIDDEN,
+                HTTPStatus.UNPROCESSABLE_ENTITY,
+                "You cannot request a list of shared rooms with yourself",
+                errcode=Codes.INVALID_PARAM,
             )
 
         rooms = await self.store.get_mutual_rooms_between_users(
diff --git a/synapse/rest/client/versions.py b/synapse/rest/client/versions.py
index 58c5b07390..32df054f56 100644
--- a/synapse/rest/client/versions.py
+++ b/synapse/rest/client/versions.py
@@ -91,7 +91,7 @@ class VersionsRestServlet(RestServlet):
                     # Implements additional endpoints as described in MSC2432
                     "org.matrix.msc2432": True,
                     # Implements additional endpoints as described in MSC2666
-                    "uk.half-shot.msc2666.mutual_rooms": True,
+                    "uk.half-shot.msc2666.query_mutual_rooms": True,
                     # Whether new rooms will be set to encrypted or not (based on presets).
                     "io.element.e2ee_forced.public": self.e2ee_forced_public,
                     "io.element.e2ee_forced.private": self.e2ee_forced_private,
diff --git a/synapse/types/__init__.py b/synapse/types/__init__.py
index 325219656a..42baf8ac6b 100644
--- a/synapse/types/__init__.py
+++ b/synapse/types/__init__.py
@@ -84,7 +84,15 @@ JsonSerializable = object
 
 # Collection[str] that does not include str itself; str being a Sequence[str]
 # is very misleading and results in bugs.
+#
+# StrCollection is an unordered collection of strings. If ordering is important,
+# StrSequence can be used instead.
 StrCollection = Union[Tuple[str, ...], List[str], AbstractSet[str]]
+# Sequence[str] that does not include str itself; str being a Sequence[str]
+# is very misleading and results in bugs.
+#
+# Unlike StrCollection, StrSequence is an ordered collection of strings.
+StrSequence = Union[Tuple[str, ...], List[str]]
 
 
 # Note that this seems to require inheriting *directly* from Interface in order
diff --git a/synapse/util/module_loader.py b/synapse/util/module_loader.py
index 5a638c6e9a..e3a54df48b 100644
--- a/synapse/util/module_loader.py
+++ b/synapse/util/module_loader.py
@@ -14,17 +14,17 @@
 
 import importlib
 import importlib.util
-import itertools
 from types import ModuleType
-from typing import Any, Iterable, Tuple, Type
+from typing import Any, Tuple, Type
 
 import jsonschema
 
 from synapse.config._base import ConfigError
 from synapse.config._util import json_error_to_config_error
+from synapse.types import StrSequence
 
 
-def load_module(provider: dict, config_path: Iterable[str]) -> Tuple[Type, Any]:
+def load_module(provider: dict, config_path: StrSequence) -> Tuple[Type, Any]:
     """Loads a synapse module with its config
 
     Args:
@@ -39,9 +39,7 @@ def load_module(provider: dict, config_path: Iterable[str]) -> Tuple[Type, Any]:
 
     modulename = provider.get("module")
     if not isinstance(modulename, str):
-        raise ConfigError(
-            "expected a string", path=itertools.chain(config_path, ("module",))
-        )
+        raise ConfigError("expected a string", path=tuple(config_path) + ("module",))
 
     # We need to import the module, and then pick the class out of
     # that, so we split based on the last dot.
@@ -55,19 +53,17 @@ def load_module(provider: dict, config_path: Iterable[str]) -> Tuple[Type, Any]:
         try:
             provider_config = provider_class.parse_config(module_config)
         except jsonschema.ValidationError as e:
-            raise json_error_to_config_error(
-                e, itertools.chain(config_path, ("config",))
-            )
+            raise json_error_to_config_error(e, tuple(config_path) + ("config",))
         except ConfigError as e:
             raise _wrap_config_error(
                 "Failed to parse config for module %r" % (modulename,),
-                prefix=itertools.chain(config_path, ("config",)),
+                prefix=tuple(config_path) + ("config",),
                 e=e,
             )
         except Exception as e:
             raise ConfigError(
                 "Failed to parse config for module %r" % (modulename,),
-                path=itertools.chain(config_path, ("config",)),
+                path=tuple(config_path) + ("config",),
             ) from e
     else:
         provider_config = module_config
@@ -92,9 +88,7 @@ def load_python_module(location: str) -> ModuleType:
     return mod
 
 
-def _wrap_config_error(
-    msg: str, prefix: Iterable[str], e: ConfigError
-) -> "ConfigError":
+def _wrap_config_error(msg: str, prefix: StrSequence, e: ConfigError) -> "ConfigError":
     """Wrap a relative ConfigError with a new path
 
     This is useful when we have a ConfigError with a relative path due to a problem
@@ -102,7 +96,7 @@ def _wrap_config_error(
     """
     path = prefix
     if e.path:
-        path = itertools.chain(prefix, e.path)
+        path = tuple(prefix) + tuple(e.path)
 
     e1 = ConfigError(msg, path)
 
diff --git a/tests/rest/client/test_mutual_rooms.py b/tests/rest/client/test_mutual_rooms.py
index a4327f7ace..22fddbd6d6 100644
--- a/tests/rest/client/test_mutual_rooms.py
+++ b/tests/rest/client/test_mutual_rooms.py
@@ -11,6 +11,8 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
+from urllib.parse import quote
+
 from twisted.test.proto_helpers import MemoryReactor
 
 import synapse.rest.admin
@@ -44,8 +46,8 @@ class UserMutualRoomsTest(unittest.HomeserverTestCase):
     def _get_mutual_rooms(self, token: str, other_user: str) -> FakeChannel:
         return self.make_request(
             "GET",
-            "/_matrix/client/unstable/uk.half-shot.msc2666/user/mutual_rooms/%s"
-            % other_user,
+            "/_matrix/client/unstable/uk.half-shot.msc2666/user/mutual_rooms"
+            f"?user_id={quote(other_user)}",
             access_token=token,
         )