From c604d2c218a80f169876cf3063817e038063f7b9 Mon Sep 17 00:00:00 2001 From: Patrick Cloke Date: Wed, 12 Oct 2022 06:46:13 -0400 Subject: Mark /relations endpoint as usable on workers. (#14028) Co-authored-by: Eric Eastwood --- scripts-dev/complement.sh | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) (limited to 'scripts-dev') diff --git a/scripts-dev/complement.sh b/scripts-dev/complement.sh index eab23f18f1..a7b1e1e3a8 100755 --- a/scripts-dev/complement.sh +++ b/scripts-dev/complement.sh @@ -126,7 +126,7 @@ export COMPLEMENT_BASE_IMAGE=complement-synapse extra_test_args=() -test_tags="synapse_blacklist,msc2716,msc3030,msc3787" +test_tags="synapse_blacklist,msc3787" # All environment variables starting with PASS_ will be shared. # (The prefix is stripped off before reaching the container.) @@ -158,7 +158,10 @@ else # We only test faster room joins on monoliths, because they are purposefully # being developed without worker support to start with. - test_tags="$test_tags,faster_joins" + # + # The tests for importing historical messages (MSC2716) and jump to date (MSC3030) + # also only pass with monoliths, currently. + test_tags="$test_tags,faster_joins,msc2716,msc3030" fi -- cgit 1.5.1 From 59ca73006c84a74f1bf4934f1c109202ff82408e Mon Sep 17 00:00:00 2001 From: Patrick Cloke Date: Tue, 1 Nov 2022 13:26:28 -0400 Subject: Enable testing MSC3874 in complement. (#14339) --- changelog.d/14339.misc | 1 + docker/complement/conf/workers-shared-extra.yaml.j2 | 4 ++-- scripts-dev/complement.sh | 2 +- 3 files changed, 4 insertions(+), 3 deletions(-) create mode 100644 changelog.d/14339.misc (limited to 'scripts-dev') diff --git a/changelog.d/14339.misc b/changelog.d/14339.misc new file mode 100644 index 0000000000..3761d453a8 --- /dev/null +++ b/changelog.d/14339.misc @@ -0,0 +1 @@ +Enabling testing of [MSC3874](https://github.com/matrix-org/matrix-spec-proposals/pull/3874) (filtering of `/messages` by relation type) in complement. diff --git a/docker/complement/conf/workers-shared-extra.yaml.j2 b/docker/complement/conf/workers-shared-extra.yaml.j2 index c651645115..883a87159c 100644 --- a/docker/complement/conf/workers-shared-extra.yaml.j2 +++ b/docker/complement/conf/workers-shared-extra.yaml.j2 @@ -92,8 +92,6 @@ allow_device_name_lookup_over_federation: true ## Experimental Features ## experimental_features: - # Enable spaces support - spaces_enabled: true # Enable history backfilling support msc2716_enabled: true # server-side support for partial state in /send_join responses @@ -104,6 +102,8 @@ experimental_features: {% endif %} # Enable jump to date endpoint msc3030_enabled: true + # Filtering /messages by relation type. + msc3874_enabled: true server_notices: system_mxid_localpart: _server diff --git a/scripts-dev/complement.sh b/scripts-dev/complement.sh index a7b1e1e3a8..76c859694f 100755 --- a/scripts-dev/complement.sh +++ b/scripts-dev/complement.sh @@ -126,7 +126,7 @@ export COMPLEMENT_BASE_IMAGE=complement-synapse extra_test_args=() -test_tags="synapse_blacklist,msc3787" +test_tags="synapse_blacklist,msc3787,msc3874" # All environment variables starting with PASS_ will be shared. # (The prefix is stripped off before reaching the container.) -- cgit 1.5.1 From 69814eb2824daf846f869cb9579eb1008e61f8ad Mon Sep 17 00:00:00 2001 From: realtyem Date: Tue, 8 Nov 2022 06:34:09 -0600 Subject: Allow override for requesting specific worker types for Complement on command line. (#14324) * Expose getting SYNAPSE_WORKER_TYPES from external, allowing override of workers requested. * Add WORKER_TYPES variable option to complement.sh script that passes requested workers into start_for_complement.sh entrypoint. * Update docs to reflect this new ability. * Changelog * Don't rely on soft wrapping to format long strings Good idea dklimpel. Thanks for catching that. Co-authored-by: Dirk Klimpel <5740567+dklimpel@users.noreply.github.com> * Small nits just noticed in docs. * Fixup new line in docs. Co-authored-by: Dirk Klimpel <5740567+dklimpel@users.noreply.github.com> --- changelog.d/14324.misc | 1 + docker/complement/conf/start_for_complement.sh | 9 ++++++++- docs/development/contributing_guide.md | 6 ++++++ scripts-dev/complement.sh | 3 +++ 4 files changed, 18 insertions(+), 1 deletion(-) create mode 100644 changelog.d/14324.misc (limited to 'scripts-dev') diff --git a/changelog.d/14324.misc b/changelog.d/14324.misc new file mode 100644 index 0000000000..54d8198a8f --- /dev/null +++ b/changelog.d/14324.misc @@ -0,0 +1 @@ +Add override ability to `complement.sh` command line script to request certain types of workers. diff --git a/docker/complement/conf/start_for_complement.sh b/docker/complement/conf/start_for_complement.sh index bb85d9fed7..49d79745b0 100755 --- a/docker/complement/conf/start_for_complement.sh +++ b/docker/complement/conf/start_for_complement.sh @@ -45,7 +45,12 @@ esac if [[ -n "$SYNAPSE_COMPLEMENT_USE_WORKERS" ]]; then # Specify the workers to test with - export SYNAPSE_WORKER_TYPES="\ + # Allow overriding by explicitly setting SYNAPSE_WORKER_TYPES outside, while still + # utilizing WORKERS=1 for backwards compatibility. + # -n True if the length of string is non-zero. + # -z True if the length of string is zero. + if [[ -z "$SYNAPSE_WORKER_TYPES" ]]; then + export SYNAPSE_WORKER_TYPES="\ event_persister, \ event_persister, \ background_worker, \ @@ -61,6 +66,8 @@ if [[ -n "$SYNAPSE_COMPLEMENT_USE_WORKERS" ]]; then appservice, \ pusher" + fi + log "Workers requested: $SYNAPSE_WORKER_TYPES" # Improve startup times by using a launcher based on fork() export SYNAPSE_USE_EXPERIMENTAL_FORKING_LAUNCHER=1 else diff --git a/docs/development/contributing_guide.md b/docs/development/contributing_guide.md index 1e52f9808c..342bc1d340 100644 --- a/docs/development/contributing_guide.md +++ b/docs/development/contributing_guide.md @@ -324,6 +324,12 @@ The above will run a monolithic (single-process) Synapse with SQLite as the data - Passing `POSTGRES=1` as an environment variable to use the Postgres database instead. - Passing `WORKERS=1` as an environment variable to use a workerised setup instead. This option implies the use of Postgres. + - If setting `WORKERS=1`, optionally set `WORKER_TYPES=` to declare which worker + types you wish to test. A simple comma-delimited string containing the worker types + defined from the `WORKERS_CONFIG` template in + [here](https://github.com/matrix-org/synapse/blob/develop/docker/configure_workers_and_start.py#L54). + A safe example would be `WORKER_TYPES="federation_inbound, federation_sender, synchrotron"`. + See the [worker documentation](../workers.md) for additional information on workers. To increase the log level for the tests, set `SYNAPSE_TEST_LOG_LEVEL`, e.g: ```sh diff --git a/scripts-dev/complement.sh b/scripts-dev/complement.sh index 76c859694f..803c6ce92d 100755 --- a/scripts-dev/complement.sh +++ b/scripts-dev/complement.sh @@ -139,6 +139,9 @@ if [[ -n "$WORKERS" ]]; then # Use workers. export PASS_SYNAPSE_COMPLEMENT_USE_WORKERS=true + # Pass through the workers defined. If none, it will be an empty string + export PASS_SYNAPSE_WORKER_TYPES="$WORKER_TYPES" + # Workers can only use Postgres as a database. export PASS_SYNAPSE_COMPLEMENT_DATABASE=postgres -- cgit 1.5.1 From 0cf48f2d5fb0b7c9d76413045c2affbc5d88c0ee Mon Sep 17 00:00:00 2001 From: Sean Quah <8349537+squahtx@users.noreply.github.com> Date: Wed, 9 Nov 2022 10:33:13 +0000 Subject: Build Debian packages for Ubuntu 22.10 Kinetic Kudu (#14396) Signed-off-by: Sean Quah --- changelog.d/14396.feature | 1 + scripts-dev/build_debian_packages.py | 1 + 2 files changed, 2 insertions(+) create mode 100644 changelog.d/14396.feature (limited to 'scripts-dev') diff --git a/changelog.d/14396.feature b/changelog.d/14396.feature new file mode 100644 index 0000000000..e875916885 --- /dev/null +++ b/changelog.d/14396.feature @@ -0,0 +1 @@ +Build Debian packages for Ubuntu 22.10 (Kinetic Kudu). diff --git a/scripts-dev/build_debian_packages.py b/scripts-dev/build_debian_packages.py index cd2e64b75f..7442300196 100755 --- a/scripts-dev/build_debian_packages.py +++ b/scripts-dev/build_debian_packages.py @@ -27,6 +27,7 @@ DISTS = ( "debian:sid", "ubuntu:focal", # 20.04 LTS (our EOL forced by Py38 on 2024-10-14) "ubuntu:jammy", # 22.04 LTS (EOL 2027-04) + "ubuntu:kinetic", # 22.10 (EOL 2023-07-20) ) DESC = """\ -- cgit 1.5.1 From 2cc592584ae9f225216b7663e9144ac6f565b757 Mon Sep 17 00:00:00 2001 From: David Robertson Date: Mon, 14 Nov 2022 13:46:29 +0000 Subject: Remove unused type-ignores (#14433) * Remove unused type-ignores Oversights in #14427 and #14429. * Changelog --- changelog.d/14433.misc | 1 + scripts-dev/release.py | 4 +--- synapse/streams/events.py | 9 ++++++--- 3 files changed, 8 insertions(+), 6 deletions(-) create mode 100644 changelog.d/14433.misc (limited to 'scripts-dev') diff --git a/changelog.d/14433.misc b/changelog.d/14433.misc new file mode 100644 index 0000000000..08a350b13b --- /dev/null +++ b/changelog.d/14433.misc @@ -0,0 +1 @@ +Fix mypy errors introduced by bumping the locked version of `attrs` and `gitpython`. diff --git a/scripts-dev/release.py b/scripts-dev/release.py index c82c58c54b..bf47b6c713 100755 --- a/scripts-dev/release.py +++ b/scripts-dev/release.py @@ -219,9 +219,7 @@ def _prepare() -> None: update_branch(repo) # Create the new release branch - # Type ignore will no longer be needed after GitPython 3.1.28. - # See https://github.com/gitpython-developers/GitPython/pull/1419 - repo.create_head(release_branch_name, commit=base_branch) # type: ignore[arg-type] + repo.create_head(release_branch_name, commit=base_branch) # Special-case SyTest: we don't actually prepare any files so we may # as well push it now (and only when we create a release branch; diff --git a/synapse/streams/events.py b/synapse/streams/events.py index bcd840bd88..f331e1af16 100644 --- a/synapse/streams/events.py +++ b/synapse/streams/events.py @@ -45,9 +45,12 @@ class _EventSourcesInner: class EventSources: def __init__(self, hs: "HomeServer"): self.sources = _EventSourcesInner( - # mypy thinks attribute.type is `Optional`, but we know it's never `None` here since - # all the attributes of `_EventSourcesInner` are annotated. - *(attribute.type(hs) for attribute in attr.fields(_EventSourcesInner)) # type: ignore[misc] + # mypy previously warned that attribute.type is `Optional`, but we know it's + # never `None` here since all the attributes of `_EventSourcesInner` are + # annotated. + # As of the stubs in attrs 22.1.0, `attr.fields()` now returns Any, + # so the call to `attribute.type` is not checked. + *(attribute.type(hs) for attribute in attr.fields(_EventSourcesInner)) ) self.store = hs.get_datastores().main -- cgit 1.5.1 From 8d133a8464e9b50640a60f23e60b735e9a818062 Mon Sep 17 00:00:00 2001 From: Richard van der Hoff <1389908+richvdh@users.noreply.github.com> Date: Sun, 20 Nov 2022 17:41:17 +0000 Subject: Fixes to `federation_client` dev script (#14479) * Attempt to fix federation-client devscript handling of .well-known The script was setting the wrong value in the Host header * Fix TLS verification Turns out that actually doing TLS verification isn't that hard. Let's enable it. --- changelog.d/14479.misc | 1 + scripts-dev/federation_client.py | 122 ++++++++++++++++++++++++++++----------- 2 files changed, 88 insertions(+), 35 deletions(-) create mode 100644 changelog.d/14479.misc (limited to 'scripts-dev') diff --git a/changelog.d/14479.misc b/changelog.d/14479.misc new file mode 100644 index 0000000000..08edd2f929 --- /dev/null +++ b/changelog.d/14479.misc @@ -0,0 +1 @@ +`scripts-dev/federation_client`: Fix routing on servers with `.well-known` files. \ No newline at end of file diff --git a/scripts-dev/federation_client.py b/scripts-dev/federation_client.py index 763dd02c47..b1d5e2e616 100755 --- a/scripts-dev/federation_client.py +++ b/scripts-dev/federation_client.py @@ -46,11 +46,12 @@ import signedjson.key import signedjson.types import srvlookup import yaml +from requests import PreparedRequest, Response from requests.adapters import HTTPAdapter from urllib3 import HTTPConnectionPool # uncomment the following to enable debug logging of http requests -# from httplib import HTTPConnection +# from http.client import HTTPConnection # HTTPConnection.debuglevel = 1 @@ -103,6 +104,7 @@ def request( destination: str, path: str, content: Optional[str], + verify_tls: bool, ) -> requests.Response: if method is None: if content is None: @@ -141,7 +143,6 @@ def request( s.mount("matrix://", MatrixConnectionAdapter()) headers: Dict[str, str] = { - "Host": destination, "Authorization": authorization_headers[0], } @@ -152,7 +153,7 @@ def request( method=method, url=dest, headers=headers, - verify=False, + verify=verify_tls, data=content, stream=True, ) @@ -202,6 +203,12 @@ def main() -> None: parser.add_argument("--body", help="Data to send as the body of the HTTP request") + parser.add_argument( + "--insecure", + action="store_true", + help="Disable TLS certificate verification", + ) + parser.add_argument( "path", help="request path, including the '/_matrix/federation/...' prefix." ) @@ -227,6 +234,7 @@ def main() -> None: args.destination, args.path, content=args.body, + verify_tls=not args.insecure, ) sys.stderr.write("Status Code: %d\n" % (result.status_code,)) @@ -254,36 +262,93 @@ def read_args_from_config(args: argparse.Namespace) -> None: class MatrixConnectionAdapter(HTTPAdapter): + def send( + self, + request: PreparedRequest, + *args: Any, + **kwargs: Any, + ) -> Response: + # overrides the send() method in the base class. + + # We need to look for .well-known redirects before passing the request up to + # HTTPAdapter.send(). + assert isinstance(request.url, str) + parsed = urlparse.urlsplit(request.url) + server_name = parsed.netloc + well_known = self._get_well_known(parsed.netloc) + + if well_known: + server_name = well_known + + # replace the scheme in the uri with https, so that cert verification is done + # also replace the hostname if we got a .well-known result + request.url = urlparse.urlunsplit( + ("https", server_name, parsed.path, parsed.query, parsed.fragment) + ) + + # at this point we also add the host header (otherwise urllib will add one + # based on the `host` from the connection returned by `get_connection`, + # which will be wrong if there is an SRV record). + request.headers["Host"] = server_name + + return super().send(request, *args, **kwargs) + + def get_connection( + self, url: str, proxies: Optional[Dict[str, str]] = None + ) -> HTTPConnectionPool: + # overrides the get_connection() method in the base class + parsed = urlparse.urlsplit(url) + (host, port, ssl_server_name) = self._lookup(parsed.netloc) + print( + f"Connecting to {host}:{port} with SNI {ssl_server_name}", file=sys.stderr + ) + return self.poolmanager.connection_from_host( + host, + port=port, + scheme="https", + pool_kwargs={"server_hostname": ssl_server_name}, + ) + @staticmethod - def lookup(s: str, skip_well_known: bool = False) -> Tuple[str, int]: - if s[-1] == "]": + def _lookup(server_name: str) -> Tuple[str, int, str]: + """ + Do an SRV lookup on a server name and return the host:port to connect to + Given the server_name (after any .well-known lookup), return the host, port and + the ssl server name + """ + if server_name[-1] == "]": # ipv6 literal (with no port) - return s, 8448 + return server_name, 8448, server_name - if ":" in s: - out = s.rsplit(":", 1) + if ":" in server_name: + # explicit port + out = server_name.rsplit(":", 1) try: port = int(out[1]) except ValueError: - raise ValueError("Invalid host:port '%s'" % s) - return out[0], port - - # try a .well-known lookup - if not skip_well_known: - well_known = MatrixConnectionAdapter.get_well_known(s) - if well_known: - return MatrixConnectionAdapter.lookup(well_known, skip_well_known=True) + raise ValueError("Invalid host:port '%s'" % (server_name,)) + return out[0], port, out[0] try: - srv = srvlookup.lookup("matrix", "tcp", s)[0] - return srv.host, srv.port + srv = srvlookup.lookup("matrix", "tcp", server_name)[0] + print( + f"SRV lookup on _matrix._tcp.{server_name} gave {srv}", + file=sys.stderr, + ) + return srv.host, srv.port, server_name except Exception: - return s, 8448 + return server_name, 8448, server_name @staticmethod - def get_well_known(server_name: str) -> Optional[str]: - uri = "https://%s/.well-known/matrix/server" % (server_name,) - print("fetching %s" % (uri,), file=sys.stderr) + def _get_well_known(server_name: str) -> Optional[str]: + if ":" in server_name: + # explicit port, or ipv6 literal. Either way, no .well-known + return None + + # TODO: check for ipv4 literals + + uri = f"https://{server_name}/.well-known/matrix/server" + print(f"fetching {uri}", file=sys.stderr) try: resp = requests.get(uri) @@ -304,19 +369,6 @@ class MatrixConnectionAdapter(HTTPAdapter): print("Invalid response from %s: %s" % (uri, e), file=sys.stderr) return None - def get_connection( - self, url: str, proxies: Optional[Dict[str, str]] = None - ) -> HTTPConnectionPool: - parsed = urlparse.urlparse(url) - - (host, port) = self.lookup(parsed.netloc) - netloc = "%s:%d" % (host, port) - print("Connecting to %s" % (netloc,), file=sys.stderr) - url = urlparse.urlunparse( - ("https", netloc, parsed.path, parsed.params, parsed.query, parsed.fragment) - ) - return super().get_connection(url, proxies) - if __name__ == "__main__": main() -- cgit 1.5.1 From 8f10c8b054fc970838be9ae6f1f5aea95f166c98 Mon Sep 17 00:00:00 2001 From: Eric Eastwood Date: Mon, 28 Nov 2022 15:54:18 -0600 Subject: Move MSC3030 `/timestamp_to_event` endpoint to stable v1 location (#14471) Fix https://github.com/matrix-org/synapse/issues/14390 - Client API: `/_matrix/client/unstable/org.matrix.msc3030/rooms//timestamp_to_event?ts=&dir=` -> `/_matrix/client/v1/rooms//timestamp_to_event?ts=&dir=` - Federation API: `/_matrix/federation/unstable/org.matrix.msc3030/timestamp_to_event/?ts=&dir=` -> `/_matrix/federation/v1/timestamp_to_event/?ts=&dir=` Complement test changes: https://github.com/matrix-org/complement/pull/559 --- changelog.d/14471.feature | 1 + docker/complement/conf/workers-shared-extra.yaml.j2 | 2 -- docker/configure_workers_and_start.py | 2 ++ docs/workers.md | 2 ++ scripts-dev/complement.sh | 6 +++--- synapse/config/experimental.py | 3 --- synapse/federation/federation_client.py | 12 +++++++++++- synapse/federation/transport/client.py | 5 ++--- synapse/federation/transport/server/__init__.py | 8 -------- synapse/federation/transport/server/federation.py | 3 +-- synapse/rest/client/room.py | 10 +++------- synapse/rest/client/versions.py | 2 -- tests/rest/client/test_rooms.py | 7 +------ 13 files changed, 26 insertions(+), 37 deletions(-) create mode 100644 changelog.d/14471.feature (limited to 'scripts-dev') diff --git a/changelog.d/14471.feature b/changelog.d/14471.feature new file mode 100644 index 0000000000..a0e0c74f1a --- /dev/null +++ b/changelog.d/14471.feature @@ -0,0 +1 @@ +Move MSC3030 `/timestamp_to_event` endpoints to stable `v1` location (`/_matrix/client/v1/rooms//timestamp_to_event?ts=&dir=`, `/_matrix/federation/v1/timestamp_to_event/?ts=&dir=`). diff --git a/docker/complement/conf/workers-shared-extra.yaml.j2 b/docker/complement/conf/workers-shared-extra.yaml.j2 index 883a87159c..ca640c343b 100644 --- a/docker/complement/conf/workers-shared-extra.yaml.j2 +++ b/docker/complement/conf/workers-shared-extra.yaml.j2 @@ -100,8 +100,6 @@ experimental_features: # client-side support for partial state in /send_join responses faster_joins: true {% endif %} - # Enable jump to date endpoint - msc3030_enabled: true # Filtering /messages by relation type. msc3874_enabled: true diff --git a/docker/configure_workers_and_start.py b/docker/configure_workers_and_start.py index c1e1544536..58c62f2231 100755 --- a/docker/configure_workers_and_start.py +++ b/docker/configure_workers_and_start.py @@ -140,6 +140,7 @@ WORKERS_CONFIG: Dict[str, Dict[str, Any]] = { "^/_matrix/client/(api/v1|r0|v3|unstable)/rooms/.*/event", "^/_matrix/client/(api/v1|r0|v3|unstable)/joined_rooms", "^/_matrix/client/(api/v1|r0|v3|unstable/.*)/rooms/.*/aliases", + "^/_matrix/client/v1/rooms/.*/timestamp_to_event$", "^/_matrix/client/(api/v1|r0|v3|unstable)/search", ], "shared_extra_conf": {}, @@ -163,6 +164,7 @@ WORKERS_CONFIG: Dict[str, Dict[str, Any]] = { "^/_matrix/federation/(v1|v2)/invite/", "^/_matrix/federation/(v1|v2)/query_auth/", "^/_matrix/federation/(v1|v2)/event_auth/", + "^/_matrix/federation/v1/timestamp_to_event/", "^/_matrix/federation/(v1|v2)/exchange_third_party_invite/", "^/_matrix/federation/(v1|v2)/user/devices/", "^/_matrix/federation/(v1|v2)/get_groups_publicised$", diff --git a/docs/workers.md b/docs/workers.md index 27e54c5846..2b65acb5ed 100644 --- a/docs/workers.md +++ b/docs/workers.md @@ -191,6 +191,7 @@ information. ^/_matrix/federation/(v1|v2)/send_leave/ ^/_matrix/federation/(v1|v2)/invite/ ^/_matrix/federation/v1/event_auth/ + ^/_matrix/federation/v1/timestamp_to_event/ ^/_matrix/federation/v1/exchange_third_party_invite/ ^/_matrix/federation/v1/user/devices/ ^/_matrix/key/v2/query @@ -218,6 +219,7 @@ information. ^/_matrix/client/(api/v1|r0|v3|unstable)/voip/turnServer$ ^/_matrix/client/(api/v1|r0|v3|unstable)/rooms/.*/event/ ^/_matrix/client/(api/v1|r0|v3|unstable)/joined_rooms$ + ^/_matrix/client/v1/rooms/.*/timestamp_to_event$ ^/_matrix/client/(api/v1|r0|v3|unstable)/search$ # Encryption requests diff --git a/scripts-dev/complement.sh b/scripts-dev/complement.sh index 803c6ce92d..7744b47097 100755 --- a/scripts-dev/complement.sh +++ b/scripts-dev/complement.sh @@ -162,9 +162,9 @@ else # We only test faster room joins on monoliths, because they are purposefully # being developed without worker support to start with. # - # The tests for importing historical messages (MSC2716) and jump to date (MSC3030) - # also only pass with monoliths, currently. - test_tags="$test_tags,faster_joins,msc2716,msc3030" + # The tests for importing historical messages (MSC2716) also only pass with monoliths, + # currently. + test_tags="$test_tags,faster_joins,msc2716" fi diff --git a/synapse/config/experimental.py b/synapse/config/experimental.py index d4b71d1673..a503abf364 100644 --- a/synapse/config/experimental.py +++ b/synapse/config/experimental.py @@ -53,9 +53,6 @@ class ExperimentalConfig(Config): # MSC3266 (room summary api) self.msc3266_enabled: bool = experimental.get("msc3266_enabled", False) - # MSC3030 (Jump to date API endpoint) - self.msc3030_enabled: bool = experimental.get("msc3030_enabled", False) - # MSC2409 (this setting only relates to optionally sending to-device messages). # Presence, typing and read receipt EDUs are already sent to application services that # have opted in to receive them. If enabled, this adds to-device messages to that list. diff --git a/synapse/federation/federation_client.py b/synapse/federation/federation_client.py index c4c0bc7315..8bccc9c60d 100644 --- a/synapse/federation/federation_client.py +++ b/synapse/federation/federation_client.py @@ -1691,9 +1691,19 @@ class FederationClient(FederationBase): # to return events on *both* sides of the timestamp to # help reconcile the gap faster. _timestamp_to_event_from_destination, + # Since this endpoint is new, we should try other servers before giving up. + # We can safely remove this in a year (remove after 2023-11-16). + failover_on_unknown_endpoint=True, ) return timestamp_to_event_response - except SynapseError: + except SynapseError as e: + logger.warn( + "timestamp_to_event(room_id=%s, timestamp=%s, direction=%s): encountered error when trying to fetch from destinations: %s", + room_id, + timestamp, + direction, + e, + ) return None async def _timestamp_to_event_from_destination( diff --git a/synapse/federation/transport/client.py b/synapse/federation/transport/client.py index a3cfc701cd..77f1f39cac 100644 --- a/synapse/federation/transport/client.py +++ b/synapse/federation/transport/client.py @@ -185,9 +185,8 @@ class TransportLayerClient: Raises: Various exceptions when the request fails """ - path = _create_path( - FEDERATION_UNSTABLE_PREFIX, - "/org.matrix.msc3030/timestamp_to_event/%s", + path = _create_v1_path( + "/timestamp_to_event/%s", room_id, ) diff --git a/synapse/federation/transport/server/__init__.py b/synapse/federation/transport/server/__init__.py index 50623cd385..2725f53cf6 100644 --- a/synapse/federation/transport/server/__init__.py +++ b/synapse/federation/transport/server/__init__.py @@ -25,7 +25,6 @@ from synapse.federation.transport.server._base import ( from synapse.federation.transport.server.federation import ( FEDERATION_SERVLET_CLASSES, FederationAccountStatusServlet, - FederationTimestampLookupServlet, ) from synapse.http.server import HttpServer, JsonResource from synapse.http.servlet import ( @@ -291,13 +290,6 @@ def register_servlets( ) for servletclass in SERVLET_GROUPS[servlet_group]: - # Only allow the `/timestamp_to_event` servlet if msc3030 is enabled - if ( - servletclass == FederationTimestampLookupServlet - and not hs.config.experimental.msc3030_enabled - ): - continue - # Only allow the `/account_status` servlet if msc3720 is enabled if ( servletclass == FederationAccountStatusServlet diff --git a/synapse/federation/transport/server/federation.py b/synapse/federation/transport/server/federation.py index 205fd16daa..53e77b4bb6 100644 --- a/synapse/federation/transport/server/federation.py +++ b/synapse/federation/transport/server/federation.py @@ -218,14 +218,13 @@ class FederationTimestampLookupServlet(BaseFederationServerServlet): `dir` can be `f` or `b` to indicate forwards and backwards in time from the given timestamp. - GET /_matrix/federation/unstable/org.matrix.msc3030/timestamp_to_event/?ts=&dir= + GET /_matrix/federation/v1/timestamp_to_event/?ts=&dir= { "event_id": ... } """ PATH = "/timestamp_to_event/(?P[^/]*)/?" - PREFIX = FEDERATION_UNSTABLE_PREFIX + "/org.matrix.msc3030" async def on_GET( self, diff --git a/synapse/rest/client/room.py b/synapse/rest/client/room.py index 91cb791139..636cc62877 100644 --- a/synapse/rest/client/room.py +++ b/synapse/rest/client/room.py @@ -1284,17 +1284,14 @@ class TimestampLookupRestServlet(RestServlet): `dir` can be `f` or `b` to indicate forwards and backwards in time from the given timestamp. - GET /_matrix/client/unstable/org.matrix.msc3030/rooms//timestamp_to_event?ts=&dir= + GET /_matrix/client/v1/rooms//timestamp_to_event?ts=&dir= { "event_id": ... } """ PATTERNS = ( - re.compile( - "^/_matrix/client/unstable/org.matrix.msc3030" - "/rooms/(?P[^/]*)/timestamp_to_event$" - ), + re.compile("^/_matrix/client/v1/rooms/(?P[^/]*)/timestamp_to_event$"), ) def __init__(self, hs: "HomeServer"): @@ -1421,8 +1418,7 @@ def register_servlets( RoomAliasListServlet(hs).register(http_server) SearchRestServlet(hs).register(http_server) RoomCreateRestServlet(hs).register(http_server) - if hs.config.experimental.msc3030_enabled: - TimestampLookupRestServlet(hs).register(http_server) + TimestampLookupRestServlet(hs).register(http_server) # Some servlets only get registered for the main process. if not is_worker: diff --git a/synapse/rest/client/versions.py b/synapse/rest/client/versions.py index 180a11ef88..3c0a90010b 100644 --- a/synapse/rest/client/versions.py +++ b/synapse/rest/client/versions.py @@ -101,8 +101,6 @@ class VersionsRestServlet(RestServlet): "org.matrix.msc3827.stable": True, # Adds support for importing historical messages as per MSC2716 "org.matrix.msc2716": self.config.experimental.msc2716_enabled, - # Adds support for jump to date endpoints (/timestamp_to_event) as per MSC3030 - "org.matrix.msc3030": self.config.experimental.msc3030_enabled, # Adds support for thread relations, per MSC3440. "org.matrix.msc3440.stable": True, # TODO: remove when "v1.3" is added above # Support for thread read receipts & notification counts. diff --git a/tests/rest/client/test_rooms.py b/tests/rest/client/test_rooms.py index e919e089cb..b4daace556 100644 --- a/tests/rest/client/test_rooms.py +++ b/tests/rest/client/test_rooms.py @@ -3546,11 +3546,6 @@ class TimestampLookupTestCase(unittest.HomeserverTestCase): login.register_servlets, ] - def default_config(self) -> JsonDict: - config = super().default_config() - config["experimental_features"] = {"msc3030_enabled": True} - return config - def prepare(self, reactor: MemoryReactor, clock: Clock, hs: HomeServer) -> None: self._storage_controllers = self.hs.get_storage_controllers() @@ -3592,7 +3587,7 @@ class TimestampLookupTestCase(unittest.HomeserverTestCase): channel = self.make_request( "GET", - f"/_matrix/client/unstable/org.matrix.msc3030/rooms/{room_id}/timestamp_to_event?dir=b&ts={outlier_event.origin_server_ts}", + f"/_matrix/client/v1/rooms/{room_id}/timestamp_to_event?dir=b&ts={outlier_event.origin_server_ts}", access_token=self.room_owner_tok, ) self.assertEqual(HTTPStatus.OK, channel.code, msg=channel.json_body) -- cgit 1.5.1 From be3a8a85e3820f1f0e713be58c366ac10c65cd28 Mon Sep 17 00:00:00 2001 From: reivilibre Date: Wed, 7 Dec 2022 15:45:31 +0000 Subject: Add `--editable` flag to `complement.sh` which uses an editable install of Synapse for faster turn-around times whilst developing iteratively. (#14548) Co-authored-by: Mathieu Velten --- changelog.d/14548.misc | 1 + docker/Dockerfile-workers | 3 +- docker/complement/Dockerfile | 3 +- docker/editable.Dockerfile | 75 ++++++++++++++++++++++++++++++++ scripts-dev/complement.sh | 100 +++++++++++++++++++++++++++++++++++-------- 5 files changed, 162 insertions(+), 20 deletions(-) create mode 100644 changelog.d/14548.misc create mode 100644 docker/editable.Dockerfile (limited to 'scripts-dev') diff --git a/changelog.d/14548.misc b/changelog.d/14548.misc new file mode 100644 index 0000000000..416332015c --- /dev/null +++ b/changelog.d/14548.misc @@ -0,0 +1 @@ +Add `--editable` flag to `complement.sh` which uses an editable install of Synapse for faster turn-around times whilst developing iteratively. \ No newline at end of file diff --git a/docker/Dockerfile-workers b/docker/Dockerfile-workers index 0c2d4f3047..faf7f2cef8 100644 --- a/docker/Dockerfile-workers +++ b/docker/Dockerfile-workers @@ -1,6 +1,7 @@ # syntax=docker/dockerfile:1 ARG SYNAPSE_VERSION=latest +ARG FROM=matrixdotorg/synapse:$SYNAPSE_VERSION # first of all, we create a base image with an nginx which we can copy into the # target image. For repeated rebuilds, this is much faster than apt installing @@ -23,7 +24,7 @@ FROM debian:bullseye-slim AS deps_base FROM redis:6-bullseye AS redis_base # now build the final image, based on the the regular Synapse docker image -FROM matrixdotorg/synapse:$SYNAPSE_VERSION +FROM $FROM # Install supervisord with pip instead of apt, to avoid installing a second # copy of python. diff --git a/docker/complement/Dockerfile b/docker/complement/Dockerfile index c0935c99a8..be1aa1c55e 100644 --- a/docker/complement/Dockerfile +++ b/docker/complement/Dockerfile @@ -7,8 +7,9 @@ # https://github.com/matrix-org/synapse/blob/develop/docker/README-testing.md#testing-with-postgresql-and-single-or-multi-process-synapse ARG SYNAPSE_VERSION=latest +ARG FROM=matrixdotorg/synapse-workers:$SYNAPSE_VERSION -FROM matrixdotorg/synapse-workers:$SYNAPSE_VERSION +FROM $FROM # First of all, we copy postgres server from the official postgres image, # since for repeated rebuilds, this is much faster than apt installing # postgres each time. diff --git a/docker/editable.Dockerfile b/docker/editable.Dockerfile new file mode 100644 index 0000000000..0e8cf2e712 --- /dev/null +++ b/docker/editable.Dockerfile @@ -0,0 +1,75 @@ +# syntax=docker/dockerfile:1 +# This dockerfile builds an editable install of Synapse. +# +# Used by `complement.sh`. Not suitable for production use. + +ARG PYTHON_VERSION=3.9 + +### +### Stage 0: generate requirements.txt +### +# We hardcode the use of Debian bullseye here because this could change upstream +# and other Dockerfiles used for testing are expecting bullseye. +FROM docker.io/python:${PYTHON_VERSION}-slim-bullseye + +# Install Rust and other dependencies (stolen from normal Dockerfile) +# install the OS build deps +RUN \ + --mount=type=cache,target=/var/cache/apt,sharing=locked \ + --mount=type=cache,target=/var/lib/apt,sharing=locked \ + apt-get update -qq && apt-get install -yqq \ + build-essential \ + libffi-dev \ + libjpeg-dev \ + libpq-dev \ + libssl-dev \ + libwebp-dev \ + libxml++2.6-dev \ + libxslt1-dev \ + openssl \ + zlib1g-dev \ + git \ + curl \ + gosu \ + libjpeg62-turbo \ + libpq5 \ + libwebp6 \ + xmlsec1 \ + libjemalloc2 \ + && rm -rf /var/lib/apt/lists/* +ENV RUSTUP_HOME=/rust +ENV CARGO_HOME=/cargo +ENV PATH=/cargo/bin:/rust/bin:$PATH +RUN mkdir /rust /cargo +RUN curl -sSf https://sh.rustup.rs | sh -s -- -y --no-modify-path --default-toolchain stable --profile minimal + + +# Make a base copy of the editable source tree, so that we have something to +# install and build now — even though it's going to be covered up by a mount +# at runtime. +COPY synapse /editable-src/synapse/ +COPY rust /editable-src/rust/ +# ... and what we need to `pip install`. +COPY pyproject.toml poetry.lock README.rst build_rust.py Cargo.toml Cargo.lock /editable-src/ + +RUN pip install poetry +RUN poetry config virtualenvs.create false +RUN cd /editable-src && poetry install --extras all + +# Make copies of useful things for inspection: +# - the Rust module (must be copied to the editable source tree before startup) +# - poetry.lock is useful for checking if dependencies have changed. +RUN cp /editable-src/synapse/synapse_rust.abi3.so /synapse_rust.abi3.so.bak +RUN cp /editable-src/poetry.lock /poetry.lock.bak + + +### Extra setup from original Dockerfile +COPY ./docker/start.py /start.py +COPY ./docker/conf /conf + +EXPOSE 8008/tcp 8009/tcp 8448/tcp + +ENTRYPOINT ["/start.py"] + +HEALTHCHECK --start-period=5s --interval=15s --timeout=5s \ + CMD curl -fSs http://localhost:8008/health || exit 1 diff --git a/scripts-dev/complement.sh b/scripts-dev/complement.sh index 7744b47097..8741ba3e34 100755 --- a/scripts-dev/complement.sh +++ b/scripts-dev/complement.sh @@ -53,6 +53,12 @@ Run the complement test suite on Synapse. Only build the Docker images. Don't actually run Complement. Conflicts with -f/--fast. + -e, --editable + Use an editable build of Synapse, rebuilding the image if necessary. + This is suitable for use in development where a fast turn-around time + is important. + Not suitable for use in CI in case the editable environment is impure. + For help on arguments to 'go test', run 'go help testflag'. EOF } @@ -73,6 +79,9 @@ while [ $# -ge 1 ]; do "--build-only") skip_complement_run=1 ;; + "-e"|"--editable") + use_editable_synapse=1 + ;; *) # unknown arg: presumably an argument to gotest. break the loop. break @@ -96,25 +105,76 @@ if [[ -z "$COMPLEMENT_DIR" ]]; then echo "Checkout available at 'complement-${COMPLEMENT_REF}'" fi +if [ -n "$use_editable_synapse" ]; then + if [[ -e synapse/synapse_rust.abi3.so ]]; then + # In an editable install, back up the host's compiled Rust module to prevent + # inconvenience; the container will overwrite the module with its own copy. + mv -n synapse/synapse_rust.abi3.so synapse/synapse_rust.abi3.so~host + # And restore it on exit: + synapse_pkg=`realpath synapse` + trap "mv -f '$synapse_pkg/synapse_rust.abi3.so~host' '$synapse_pkg/synapse_rust.abi3.so'" EXIT + fi + + editable_mount="$(realpath .):/editable-src:z" + if docker inspect complement-synapse-editable &>/dev/null; then + # complement-synapse-editable already exists: see if we can still use it: + # - The Rust module must still be importable; it will fail to import if the Rust source has changed. + # - The Poetry lock file must be the same (otherwise we assume dependencies have changed) + + # First set up the module in the right place for an editable installation. + docker run --rm -v $editable_mount --entrypoint 'cp' complement-synapse-editable -- /synapse_rust.abi3.so.bak /editable-src/synapse/synapse_rust.abi3.so + + if (docker run --rm -v $editable_mount --entrypoint 'python' complement-synapse-editable -c 'import synapse.synapse_rust' \ + && docker run --rm -v $editable_mount --entrypoint 'diff' complement-synapse-editable --brief /editable-src/poetry.lock /poetry.lock.bak); then + skip_docker_build=1 + else + echo "Editable Synapse image is stale. Will rebuild." + unset skip_docker_build + fi + fi +fi + if [ -z "$skip_docker_build" ]; then - # Build the base Synapse image from the local checkout - echo_if_github "::group::Build Docker image: matrixdotorg/synapse" - docker build -t matrixdotorg/synapse \ - --build-arg TEST_ONLY_SKIP_DEP_HASH_VERIFICATION \ - --build-arg TEST_ONLY_IGNORE_POETRY_LOCKFILE \ - -f "docker/Dockerfile" . - echo_if_github "::endgroup::" - - # Build the workers docker image (from the base Synapse image we just built). - echo_if_github "::group::Build Docker image: matrixdotorg/synapse-workers" - docker build -t matrixdotorg/synapse-workers -f "docker/Dockerfile-workers" . - echo_if_github "::endgroup::" - - # Build the unified Complement image (from the worker Synapse image we just built). - echo_if_github "::group::Build Docker image: complement/Dockerfile" - docker build -t complement-synapse \ - -f "docker/complement/Dockerfile" "docker/complement" - echo_if_github "::endgroup::" + if [ -n "$use_editable_synapse" ]; then + + # Build a special image designed for use in development with editable + # installs. + docker build -t synapse-editable \ + -f "docker/editable.Dockerfile" . + + docker build -t synapse-workers-editable \ + --build-arg FROM=synapse-editable \ + -f "docker/Dockerfile-workers" . + + docker build -t complement-synapse-editable \ + --build-arg FROM=synapse-workers-editable \ + -f "docker/complement/Dockerfile" "docker/complement" + + # Prepare the Rust module + docker run --rm -v $editable_mount --entrypoint 'cp' complement-synapse-editable -- /synapse_rust.abi3.so.bak /editable-src/synapse/synapse_rust.abi3.so + + else + + # Build the base Synapse image from the local checkout + echo_if_github "::group::Build Docker image: matrixdotorg/synapse" + docker build -t matrixdotorg/synapse \ + --build-arg TEST_ONLY_SKIP_DEP_HASH_VERIFICATION \ + --build-arg TEST_ONLY_IGNORE_POETRY_LOCKFILE \ + -f "docker/Dockerfile" . + echo_if_github "::endgroup::" + + # Build the workers docker image (from the base Synapse image we just built). + echo_if_github "::group::Build Docker image: matrixdotorg/synapse-workers" + docker build -t matrixdotorg/synapse-workers -f "docker/Dockerfile-workers" . + echo_if_github "::endgroup::" + + # Build the unified Complement image (from the worker Synapse image we just built). + echo_if_github "::group::Build Docker image: complement/Dockerfile" + docker build -t complement-synapse \ + -f "docker/complement/Dockerfile" "docker/complement" + echo_if_github "::endgroup::" + + fi fi if [ -n "$skip_complement_run" ]; then @@ -123,6 +183,10 @@ if [ -n "$skip_complement_run" ]; then fi export COMPLEMENT_BASE_IMAGE=complement-synapse +if [ -n "$use_editable_synapse" ]; then + export COMPLEMENT_BASE_IMAGE=complement-synapse-editable + export COMPLEMENT_HOST_MOUNTS="$editable_mount" +fi extra_test_args=() -- cgit 1.5.1 From 3d87847ecc943c689c4587c5327d744e4a8f92c2 Mon Sep 17 00:00:00 2001 From: David Robertson Date: Mon, 12 Dec 2022 21:25:07 +0000 Subject: Enable `--warn-redundant-casts` option in mypy (#14671) * Enable `--warn-redundant-casts` option in mypy Doesn't do much but helps me sleep better at night. * Changelog * Fix name of the ignore * Fix one more missed cast Not sure why I didn't see this one locally, maybe I needed a poetry update * Remove old comment Co-authored-by: Patrick Cloke Co-authored-by: Patrick Cloke --- changelog.d/14671.misc | 1 + mypy.ini | 1 + scripts-dev/release.py | 6 ++---- synapse/storage/database.py | 3 ++- synapse/storage/engines/postgres.py | 2 +- 5 files changed, 7 insertions(+), 6 deletions(-) create mode 100644 changelog.d/14671.misc (limited to 'scripts-dev') diff --git a/changelog.d/14671.misc b/changelog.d/14671.misc new file mode 100644 index 0000000000..93ceaeafc9 --- /dev/null +++ b/changelog.d/14671.misc @@ -0,0 +1 @@ +Improve type hints. diff --git a/mypy.ini b/mypy.ini index a4a1e4511a..727536df50 100644 --- a/mypy.ini +++ b/mypy.ini @@ -12,6 +12,7 @@ local_partial_types = True no_implicit_optional = True disallow_untyped_defs = True strict_equality = True +warn_redundant_casts = True files = docker/, diff --git a/scripts-dev/release.py b/scripts-dev/release.py index bf47b6c713..6974fd7895 100755 --- a/scripts-dev/release.py +++ b/scripts-dev/release.py @@ -27,7 +27,7 @@ import time import urllib.request from os import path from tempfile import TemporaryDirectory -from typing import Any, List, Optional, cast +from typing import Any, List, Optional import attr import click @@ -174,9 +174,7 @@ def _prepare() -> None: click.get_current_context().abort() # Switch to the release branch. - # Cast safety: parse() won't return a version.LegacyVersion from our - # version string format. - parsed_new_version = cast(version.Version, version.parse(new_version)) + parsed_new_version = version.parse(new_version) # We assume for debian changelogs that we only do RCs or full releases. assert not parsed_new_version.is_devrelease diff --git a/synapse/storage/database.py b/synapse/storage/database.py index 55bcb90001..0b29e67b94 100644 --- a/synapse/storage/database.py +++ b/synapse/storage/database.py @@ -667,7 +667,8 @@ class DatabasePool: ) # also check variables referenced in func's closure if inspect.isfunction(func): - f = cast(types.FunctionType, func) + # Keep the cast for now---it helps PyCharm to understand what `func` is. + f = cast(types.FunctionType, func) # type: ignore[redundant-cast] if f.__closure__: for i, cell in enumerate(f.__closure__): if inspect.isgenerator(cell.cell_contents): diff --git a/synapse/storage/engines/postgres.py b/synapse/storage/engines/postgres.py index 719a517336..f9f562ea45 100644 --- a/synapse/storage/engines/postgres.py +++ b/synapse/storage/engines/postgres.py @@ -77,7 +77,7 @@ class PostgresEngine( # docs: The number is formed by converting the major, minor, and # revision numbers into two-decimal-digit numbers and appending them # together. For example, version 8.1.5 will be returned as 80105 - self._version = cast(int, db_conn.server_version) + self._version = db_conn.server_version allow_unsafe_locale = self.config.get("allow_unsafe_locale", False) # Are we on a supported PostgreSQL version? -- cgit 1.5.1