From 405aeb0b2c40443d22ce8c265df18e81bd995b44 Mon Sep 17 00:00:00 2001 From: Brendan Abolivier Date: Thu, 18 Mar 2021 16:34:47 +0100 Subject: Implement MSC3026: busy presence state --- changelog.d/9644.feature | 1 + synapse/api/constants.py | 1 + synapse/app/generic_worker.py | 1 + synapse/handlers/presence.py | 3 ++- synapse/rest/client/versions.py | 2 ++ tests/handlers/test_presence.py | 20 ++++++++++++++++++++ 6 files changed, 27 insertions(+), 1 deletion(-) create mode 100644 changelog.d/9644.feature diff --git a/changelog.d/9644.feature b/changelog.d/9644.feature new file mode 100644 index 0000000000..556bcf0f9f --- /dev/null +++ b/changelog.d/9644.feature @@ -0,0 +1 @@ +Implement the busy presence state as described in [MSC3026](https://github.com/matrix-org/matrix-doc/pull/3026). diff --git a/synapse/api/constants.py b/synapse/api/constants.py index 691f8f9adf..cc8541bc16 100644 --- a/synapse/api/constants.py +++ b/synapse/api/constants.py @@ -51,6 +51,7 @@ class PresenceState: OFFLINE = "offline" UNAVAILABLE = "unavailable" ONLINE = "online" + BUSY = "org.matrix.msc3026.busy" class JoinRules: diff --git a/synapse/app/generic_worker.py b/synapse/app/generic_worker.py index 274d582d07..236d98a29d 100644 --- a/synapse/app/generic_worker.py +++ b/synapse/app/generic_worker.py @@ -439,6 +439,7 @@ class GenericWorkerPresence(BasePresenceHandler): PresenceState.ONLINE, PresenceState.UNAVAILABLE, PresenceState.OFFLINE, + PresenceState.BUSY, ) if presence not in valid_presence: raise SynapseError(400, "Invalid presence state") diff --git a/synapse/handlers/presence.py b/synapse/handlers/presence.py index 54631b4ee2..bcb99f627b 100644 --- a/synapse/handlers/presence.py +++ b/synapse/handlers/presence.py @@ -730,6 +730,7 @@ class PresenceHandler(BasePresenceHandler): PresenceState.ONLINE, PresenceState.UNAVAILABLE, PresenceState.OFFLINE, + PresenceState.BUSY, ) if presence not in valid_presence: raise SynapseError(400, "Invalid presence state") @@ -744,7 +745,7 @@ class PresenceHandler(BasePresenceHandler): msg = status_msg if presence != PresenceState.OFFLINE else None new_fields["status_msg"] = msg - if presence == PresenceState.ONLINE: + if presence == PresenceState.ONLINE or presence == PresenceState.BUSY: new_fields["last_active_ts"] = self.clock.time_msec() await self._update_states([prev_state.copy_and_replace(**new_fields)]) diff --git a/synapse/rest/client/versions.py b/synapse/rest/client/versions.py index d24a199318..f387d29b57 100644 --- a/synapse/rest/client/versions.py +++ b/synapse/rest/client/versions.py @@ -81,6 +81,8 @@ class VersionsRestServlet(RestServlet): "io.element.e2ee_forced.public": self.e2ee_forced_public, "io.element.e2ee_forced.private": self.e2ee_forced_private, "io.element.e2ee_forced.trusted_private": self.e2ee_forced_trusted_private, + # Supports the busy presence state described in MSC3026. + "org.matrix.msc3026.busy_presence": True, }, }, ) diff --git a/tests/handlers/test_presence.py b/tests/handlers/test_presence.py index 996c614198..77330f59a9 100644 --- a/tests/handlers/test_presence.py +++ b/tests/handlers/test_presence.py @@ -310,6 +310,26 @@ class PresenceTimeoutTestCase(unittest.TestCase): self.assertIsNotNone(new_state) self.assertEquals(new_state.state, PresenceState.UNAVAILABLE) + def test_busy_no_idle(self): + """ + Tests that a user setting their presence to busy but idling doesn't turn their + presence state into unavailable. + """ + user_id = "@foo:bar" + now = 5000000 + + state = UserPresenceState.default(user_id) + state = state.copy_and_replace( + state=PresenceState.BUSY, + last_active_ts=now - IDLE_TIMER - 1, + last_user_sync_ts=now, + ) + + new_state = handle_timeout(state, is_mine=True, syncing_user_ids=set(), now=now) + + self.assertIsNotNone(new_state) + self.assertEquals(new_state.state, PresenceState.BUSY) + def test_sync_timeout(self): user_id = "@foo:bar" now = 5000000 -- cgit 1.5.1 From 066c703729d72b5da8bb6574d7f7f5f13e12f773 Mon Sep 17 00:00:00 2001 From: Brendan Abolivier Date: Thu, 18 Mar 2021 18:37:19 +0100 Subject: Move support for MSC3026 behind an experimental flag --- synapse/app/generic_worker.py | 7 ++++++- synapse/config/experimental.py | 2 ++ synapse/handlers/presence.py | 12 ++++++++++-- 3 files changed, 18 insertions(+), 3 deletions(-) diff --git a/synapse/app/generic_worker.py b/synapse/app/generic_worker.py index 236d98a29d..207d5ccd02 100644 --- a/synapse/app/generic_worker.py +++ b/synapse/app/generic_worker.py @@ -302,6 +302,8 @@ class GenericWorkerPresence(BasePresenceHandler): self.send_stop_syncing, UPDATE_SYNCING_USERS_MS ) + self._busy_presence_enabled = hs.config.experimental.msc3026_enabled + hs.get_reactor().addSystemEventTrigger( "before", "shutdown", @@ -439,8 +441,11 @@ class GenericWorkerPresence(BasePresenceHandler): PresenceState.ONLINE, PresenceState.UNAVAILABLE, PresenceState.OFFLINE, - PresenceState.BUSY, ) + + if self._busy_presence_enabled: + valid_presence += (PresenceState.BUSY,) + if presence not in valid_presence: raise SynapseError(400, "Invalid presence state") diff --git a/synapse/config/experimental.py b/synapse/config/experimental.py index b1c1c51e4d..2f0cd0cfdf 100644 --- a/synapse/config/experimental.py +++ b/synapse/config/experimental.py @@ -27,3 +27,5 @@ class ExperimentalConfig(Config): # MSC2858 (multiple SSO identity providers) self.msc2858_enabled = experimental.get("msc2858_enabled", False) # type: bool + # MSC3026 (busy presence state) + self.msc3026_enabled = experimental.get("msc3026_enabled", False) # type: bool diff --git a/synapse/handlers/presence.py b/synapse/handlers/presence.py index bcb99f627b..372017590d 100644 --- a/synapse/handlers/presence.py +++ b/synapse/handlers/presence.py @@ -104,6 +104,8 @@ class BasePresenceHandler(abc.ABC): self.clock = hs.get_clock() self.store = hs.get_datastore() + self._busy_presence_enabled = hs.config.experimental.msc3026_enabled + active_presence = self.store.take_presence_startup_info() self.user_to_current_state = {state.user_id: state for state in active_presence} @@ -730,8 +732,11 @@ class PresenceHandler(BasePresenceHandler): PresenceState.ONLINE, PresenceState.UNAVAILABLE, PresenceState.OFFLINE, - PresenceState.BUSY, ) + + if self._busy_presence_enabled: + valid_presence += (PresenceState.BUSY,) + if presence not in valid_presence: raise SynapseError(400, "Invalid presence state") @@ -745,7 +750,10 @@ class PresenceHandler(BasePresenceHandler): msg = status_msg if presence != PresenceState.OFFLINE else None new_fields["status_msg"] = msg - if presence == PresenceState.ONLINE or presence == PresenceState.BUSY: + if ( + presence == PresenceState.ONLINE or + (self._busy_presence_enabled and presence == PresenceState.BUSY) + ): new_fields["last_active_ts"] = self.clock.time_msec() await self._update_states([prev_state.copy_and_replace(**new_fields)]) -- cgit 1.5.1 From 066068f03478753b7d838ae49e87d7a6cde80fd6 Mon Sep 17 00:00:00 2001 From: Richard van der Hoff Date: Fri, 19 Mar 2021 12:20:11 +0000 Subject: fix mypy --- synapse/rest/client/v1/room.py | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/synapse/rest/client/v1/room.py b/synapse/rest/client/v1/room.py index b7aa82a654..6c722d634d 100644 --- a/synapse/rest/client/v1/room.py +++ b/synapse/rest/client/v1/room.py @@ -21,8 +21,6 @@ import re from typing import TYPE_CHECKING, List, Optional, Tuple from urllib import parse as urlparse -from twisted.web.server import Request - from synapse.api.constants import EventTypes, Membership from synapse.api.errors import ( AuthError, @@ -42,6 +40,7 @@ from synapse.http.servlet import ( parse_json_object_from_request, parse_string, ) +from synapse.http.site import SynapseRequest from synapse.logging.opentracing import set_tag from synapse.rest.client.transactions import HttpTransactionCache from synapse.rest.client.v2_alpha._base import client_patterns @@ -1010,7 +1009,9 @@ class RoomSpaceSummaryRestServlet(RestServlet): self._auth = hs.get_auth() self._space_summary_handler = hs.get_space_summary_handler() - async def on_GET(self, request: Request, room_id: str) -> Tuple[int, JsonDict]: + async def on_GET( + self, request: SynapseRequest, room_id: str + ) -> Tuple[int, JsonDict]: requester = await self._auth.get_user_by_req(request, allow_guest=True) return 200, await self._space_summary_handler.get_space_summary( @@ -1020,7 +1021,9 @@ class RoomSpaceSummaryRestServlet(RestServlet): max_rooms_per_space=parse_integer(request, "max_rooms_per_space"), ) - async def on_POST(self, request: Request, room_id: str) -> Tuple[int, JsonDict]: + async def on_POST( + self, request: SynapseRequest, room_id: str + ) -> Tuple[int, JsonDict]: requester = await self._auth.get_user_by_req(request, allow_guest=True) content = parse_json_object_from_request(request) -- cgit 1.5.1 From 0b56481caafc56c2e624d4b6506c91fc3913615e Mon Sep 17 00:00:00 2001 From: Brendan Abolivier Date: Fri, 19 Mar 2021 16:11:08 +0100 Subject: Fix lint --- synapse/app/generic_worker.py | 8 ++++---- synapse/handlers/presence.py | 13 ++++++------- 2 files changed, 10 insertions(+), 11 deletions(-) diff --git a/synapse/app/generic_worker.py b/synapse/app/generic_worker.py index 207d5ccd02..caef394e1d 100644 --- a/synapse/app/generic_worker.py +++ b/synapse/app/generic_worker.py @@ -441,12 +441,12 @@ class GenericWorkerPresence(BasePresenceHandler): PresenceState.ONLINE, PresenceState.UNAVAILABLE, PresenceState.OFFLINE, + PresenceState.BUSY, ) - if self._busy_presence_enabled: - valid_presence += (PresenceState.BUSY,) - - if presence not in valid_presence: + if presence not in valid_presence or ( + presence == PresenceState.BUSY and not self._busy_presence_enabled + ): raise SynapseError(400, "Invalid presence state") user_id = target_user.to_string() diff --git a/synapse/handlers/presence.py b/synapse/handlers/presence.py index 372017590d..492c4478fa 100644 --- a/synapse/handlers/presence.py +++ b/synapse/handlers/presence.py @@ -732,12 +732,12 @@ class PresenceHandler(BasePresenceHandler): PresenceState.ONLINE, PresenceState.UNAVAILABLE, PresenceState.OFFLINE, + PresenceState.BUSY, ) - if self._busy_presence_enabled: - valid_presence += (PresenceState.BUSY,) - - if presence not in valid_presence: + if presence not in valid_presence or ( + presence == PresenceState.BUSY and not self._busy_presence_enabled + ): raise SynapseError(400, "Invalid presence state") user_id = target_user.to_string() @@ -750,9 +750,8 @@ class PresenceHandler(BasePresenceHandler): msg = status_msg if presence != PresenceState.OFFLINE else None new_fields["status_msg"] = msg - if ( - presence == PresenceState.ONLINE or - (self._busy_presence_enabled and presence == PresenceState.BUSY) + if presence == PresenceState.ONLINE or ( + self._busy_presence_enabled and presence == PresenceState.BUSY ): new_fields["last_active_ts"] = self.clock.time_msec() -- cgit 1.5.1 From b6ed4f55acf7af44af1d33097407d2dd7f08b5a5 Mon Sep 17 00:00:00 2001 From: Brendan Abolivier Date: Fri, 19 Mar 2021 18:19:50 +0100 Subject: Incorporate review --- synapse/handlers/presence.py | 2 +- synapse/rest/client/versions.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/synapse/handlers/presence.py b/synapse/handlers/presence.py index 492c4478fa..da92feacc9 100644 --- a/synapse/handlers/presence.py +++ b/synapse/handlers/presence.py @@ -751,7 +751,7 @@ class PresenceHandler(BasePresenceHandler): new_fields["status_msg"] = msg if presence == PresenceState.ONLINE or ( - self._busy_presence_enabled and presence == PresenceState.BUSY + presence == PresenceState.BUSY and self._busy_presence_enabled ): new_fields["last_active_ts"] = self.clock.time_msec() diff --git a/synapse/rest/client/versions.py b/synapse/rest/client/versions.py index f387d29b57..3e3d8839f4 100644 --- a/synapse/rest/client/versions.py +++ b/synapse/rest/client/versions.py @@ -82,7 +82,7 @@ class VersionsRestServlet(RestServlet): "io.element.e2ee_forced.private": self.e2ee_forced_private, "io.element.e2ee_forced.trusted_private": self.e2ee_forced_trusted_private, # Supports the busy presence state described in MSC3026. - "org.matrix.msc3026.busy_presence": True, + "org.matrix.msc3026.busy_presence": self.config.experimental.msc3026_enabled, }, }, ) -- cgit 1.5.1 From d66f9070cd0f826e5b6630f8e1f6ed5837a3c3cb Mon Sep 17 00:00:00 2001 From: Ankit Dobhal Date: Mon, 22 Mar 2021 20:48:13 +0530 Subject: Fixed code misc. quality issues (#9649) - Merge 'isinstance' calls. - Remove unnecessary dict call outside of comprehension. - Use 'sys.exit()' calls. --- changelog.d/9649.misc | 1 + scripts/move_remote_media_to_new_store.py | 2 +- synapse/push/httppusher.py | 2 +- synapse/util/frozenutils.py | 2 +- 4 files changed, 4 insertions(+), 3 deletions(-) create mode 100644 changelog.d/9649.misc diff --git a/changelog.d/9649.misc b/changelog.d/9649.misc new file mode 100644 index 0000000000..58c5fd0537 --- /dev/null +++ b/changelog.d/9649.misc @@ -0,0 +1 @@ +Fixed some antipattern issues to improve code quality. diff --git a/scripts/move_remote_media_to_new_store.py b/scripts/move_remote_media_to_new_store.py index ab2e763386..8477955a90 100755 --- a/scripts/move_remote_media_to_new_store.py +++ b/scripts/move_remote_media_to_new_store.py @@ -51,7 +51,7 @@ def main(src_repo, dest_repo): parts = line.split("|") if len(parts) != 2: print("Unable to parse input line %s" % line, file=sys.stderr) - exit(1) + sys.exit(1) move_media(parts[0], parts[1], src_paths, dest_paths) diff --git a/synapse/push/httppusher.py b/synapse/push/httppusher.py index eb6de8ba72..026134ae26 100644 --- a/synapse/push/httppusher.py +++ b/synapse/push/httppusher.py @@ -290,7 +290,7 @@ class HttpPusher(Pusher): if rejected is False: return False - if isinstance(rejected, list) or isinstance(rejected, tuple): + if isinstance(rejected, (list, tuple)): for pk in rejected: if pk != self.pushkey: # for sanity, we only remove the pushkey if it diff --git a/synapse/util/frozenutils.py b/synapse/util/frozenutils.py index 5f7a6dd1d3..5ca2e71e60 100644 --- a/synapse/util/frozenutils.py +++ b/synapse/util/frozenutils.py @@ -36,7 +36,7 @@ def freeze(o): def unfreeze(o): if isinstance(o, (dict, frozendict)): - return dict({k: unfreeze(v) for k, v in o.items()}) + return {k: unfreeze(v) for k, v in o.items()} if isinstance(o, (bytes, str)): return o -- cgit 1.5.1 From 4612302399dc95ba781af6ebc9eedc768f7f0a7d Mon Sep 17 00:00:00 2001 From: Johannes Wienke Date: Mon, 22 Mar 2021 16:31:00 +0100 Subject: Include opencontainers labels in Docker image (#9612) Cf. https://github.com/opencontainers/image-spec/blob/master/annotations.md#pre-defined-annotation-keys Signed-off-by: Johannes Wienke --- changelog.d/9612.docker | 1 + docker/Dockerfile | 5 +++++ 2 files changed, 6 insertions(+) create mode 100644 changelog.d/9612.docker diff --git a/changelog.d/9612.docker b/changelog.d/9612.docker new file mode 100644 index 0000000000..d95c503c8b --- /dev/null +++ b/changelog.d/9612.docker @@ -0,0 +1 @@ +Include [opencontainers labels](https://github.com/opencontainers/image-spec/blob/master/annotations.md#pre-defined-annotation-keys) in the Docker image. diff --git a/docker/Dockerfile b/docker/Dockerfile index def4501541..7cd4dd7d1e 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -18,6 +18,11 @@ ARG PYTHON_VERSION=3.8 ### FROM docker.io/python:${PYTHON_VERSION}-slim as builder +LABEL org.opencontainers.image.url='https://matrix.org/docs/projects/server/synapse' +LABEL org.opencontainers.image.documentation='https://github.com/matrix-org/synapse/blob/master/docker/README.md' +LABEL org.opencontainers.image.source='https://github.com/matrix-org/synapse.git' +LABEL org.opencontainers.image.licenses='Apache-2.0' + # install the OS build deps RUN apt-get update && apt-get install -y \ build-essential \ -- cgit 1.5.1 From 5b268997bdde04c905eed0a7c04e9e2352e35fba Mon Sep 17 00:00:00 2001 From: Andrew Morgan <1342360+anoadragon453@users.noreply.github.com> Date: Mon, 22 Mar 2021 17:20:47 +0000 Subject: Allow providing credentials to HTTPS_PROXY (#9657) Addresses https://github.com/matrix-org/synapse-dinsic/issues/70 This PR causes `ProxyAgent` to attempt to extract credentials from an `HTTPS_PROXY` env var. If credentials are found, a `Proxy-Authorization` header ([details](https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Proxy-Authorization)) is sent to the proxy server to authenticate against it. The headers are *not* passed to the remote server. Also added some type hints. --- changelog.d/9657.feature | 1 + synapse/http/connectproxyclient.py | 96 +++++++++++++++++++++++++++----------- synapse/http/proxyagent.py | 81 ++++++++++++++++++++++++++++---- tests/http/test_proxyagent.py | 40 ++++++++++++++++ 4 files changed, 184 insertions(+), 34 deletions(-) create mode 100644 changelog.d/9657.feature diff --git a/changelog.d/9657.feature b/changelog.d/9657.feature new file mode 100644 index 0000000000..c56a615a8b --- /dev/null +++ b/changelog.d/9657.feature @@ -0,0 +1 @@ +Add support for credentials for proxy authentication in the `HTTPS_PROXY` environment variable. diff --git a/synapse/http/connectproxyclient.py b/synapse/http/connectproxyclient.py index 856e28454f..b797e3ce80 100644 --- a/synapse/http/connectproxyclient.py +++ b/synapse/http/connectproxyclient.py @@ -19,9 +19,10 @@ from zope.interface import implementer from twisted.internet import defer, protocol from twisted.internet.error import ConnectError -from twisted.internet.interfaces import IStreamClientEndpoint -from twisted.internet.protocol import connectionDone +from twisted.internet.interfaces import IReactorCore, IStreamClientEndpoint +from twisted.internet.protocol import ClientFactory, Protocol, connectionDone from twisted.web import http +from twisted.web.http_headers import Headers logger = logging.getLogger(__name__) @@ -43,23 +44,33 @@ class HTTPConnectProxyEndpoint: Args: reactor: the Twisted reactor to use for the connection - proxy_endpoint (IStreamClientEndpoint): the endpoint to use to connect to the - proxy - host (bytes): hostname that we want to CONNECT to - port (int): port that we want to connect to + proxy_endpoint: the endpoint to use to connect to the proxy + host: hostname that we want to CONNECT to + port: port that we want to connect to + headers: Extra HTTP headers to include in the CONNECT request """ - def __init__(self, reactor, proxy_endpoint, host, port): + def __init__( + self, + reactor: IReactorCore, + proxy_endpoint: IStreamClientEndpoint, + host: bytes, + port: int, + headers: Headers, + ): self._reactor = reactor self._proxy_endpoint = proxy_endpoint self._host = host self._port = port + self._headers = headers def __repr__(self): return "" % (self._proxy_endpoint,) - def connect(self, protocolFactory): - f = HTTPProxiedClientFactory(self._host, self._port, protocolFactory) + def connect(self, protocolFactory: ClientFactory): + f = HTTPProxiedClientFactory( + self._host, self._port, protocolFactory, self._headers + ) d = self._proxy_endpoint.connect(f) # once the tcp socket connects successfully, we need to wait for the # CONNECT to complete. @@ -74,15 +85,23 @@ class HTTPProxiedClientFactory(protocol.ClientFactory): HTTP Protocol object and run the rest of the connection. Args: - dst_host (bytes): hostname that we want to CONNECT to - dst_port (int): port that we want to connect to - wrapped_factory (protocol.ClientFactory): The original Factory + dst_host: hostname that we want to CONNECT to + dst_port: port that we want to connect to + wrapped_factory: The original Factory + headers: Extra HTTP headers to include in the CONNECT request """ - def __init__(self, dst_host, dst_port, wrapped_factory): + def __init__( + self, + dst_host: bytes, + dst_port: int, + wrapped_factory: ClientFactory, + headers: Headers, + ): self.dst_host = dst_host self.dst_port = dst_port self.wrapped_factory = wrapped_factory + self.headers = headers self.on_connection = defer.Deferred() def startedConnecting(self, connector): @@ -92,7 +111,11 @@ class HTTPProxiedClientFactory(protocol.ClientFactory): wrapped_protocol = self.wrapped_factory.buildProtocol(addr) return HTTPConnectProtocol( - self.dst_host, self.dst_port, wrapped_protocol, self.on_connection + self.dst_host, + self.dst_port, + wrapped_protocol, + self.on_connection, + self.headers, ) def clientConnectionFailed(self, connector, reason): @@ -112,24 +135,37 @@ class HTTPConnectProtocol(protocol.Protocol): """Protocol that wraps an existing Protocol to do a CONNECT handshake at connect Args: - host (bytes): The original HTTP(s) hostname or IPv4 or IPv6 address literal + host: The original HTTP(s) hostname or IPv4 or IPv6 address literal to put in the CONNECT request - port (int): The original HTTP(s) port to put in the CONNECT request + port: The original HTTP(s) port to put in the CONNECT request - wrapped_protocol (interfaces.IProtocol): the original protocol (probably - HTTPChannel or TLSMemoryBIOProtocol, but could be anything really) + wrapped_protocol: the original protocol (probably HTTPChannel or + TLSMemoryBIOProtocol, but could be anything really) - connected_deferred (Deferred): a Deferred which will be callbacked with + connected_deferred: a Deferred which will be callbacked with wrapped_protocol when the CONNECT completes + + headers: Extra HTTP headers to include in the CONNECT request """ - def __init__(self, host, port, wrapped_protocol, connected_deferred): + def __init__( + self, + host: bytes, + port: int, + wrapped_protocol: Protocol, + connected_deferred: defer.Deferred, + headers: Headers, + ): self.host = host self.port = port self.wrapped_protocol = wrapped_protocol self.connected_deferred = connected_deferred - self.http_setup_client = HTTPConnectSetupClient(self.host, self.port) + self.headers = headers + + self.http_setup_client = HTTPConnectSetupClient( + self.host, self.port, self.headers + ) self.http_setup_client.on_connected.addCallback(self.proxyConnected) def connectionMade(self): @@ -154,7 +190,7 @@ class HTTPConnectProtocol(protocol.Protocol): if buf: self.wrapped_protocol.dataReceived(buf) - def dataReceived(self, data): + def dataReceived(self, data: bytes): # if we've set up the HTTP protocol, we can send the data there if self.wrapped_protocol.connected: return self.wrapped_protocol.dataReceived(data) @@ -168,21 +204,29 @@ class HTTPConnectSetupClient(http.HTTPClient): """HTTPClient protocol to send a CONNECT message for proxies and read the response. Args: - host (bytes): The hostname to send in the CONNECT message - port (int): The port to send in the CONNECT message + host: The hostname to send in the CONNECT message + port: The port to send in the CONNECT message + headers: Extra headers to send with the CONNECT message """ - def __init__(self, host, port): + def __init__(self, host: bytes, port: int, headers: Headers): self.host = host self.port = port + self.headers = headers self.on_connected = defer.Deferred() def connectionMade(self): logger.debug("Connected to proxy, sending CONNECT") self.sendCommand(b"CONNECT", b"%s:%d" % (self.host, self.port)) + + # Send any additional specified headers + for name, values in self.headers.getAllRawHeaders(): + for value in values: + self.sendHeader(name, value) + self.endHeaders() - def handleStatus(self, version, status, message): + def handleStatus(self, version: bytes, status: bytes, message: bytes): logger.debug("Got Status: %s %s %s", status, message, version) if status != b"200": raise ProxyConnectError("Unexpected status on CONNECT: %s" % status) diff --git a/synapse/http/proxyagent.py b/synapse/http/proxyagent.py index 3d553ae236..16ec850064 100644 --- a/synapse/http/proxyagent.py +++ b/synapse/http/proxyagent.py @@ -12,10 +12,13 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. +import base64 import logging import re +from typing import Optional, Tuple from urllib.request import getproxies_environment, proxy_bypass_environment +import attr from zope.interface import implementer from twisted.internet import defer @@ -23,6 +26,7 @@ from twisted.internet.endpoints import HostnameEndpoint, wrapClientTLS from twisted.python.failure import Failure from twisted.web.client import URI, BrowserLikePolicyForHTTPS, _AgentBase from twisted.web.error import SchemeNotSupported +from twisted.web.http_headers import Headers from twisted.web.iweb import IAgent from synapse.http.connectproxyclient import HTTPConnectProxyEndpoint @@ -32,6 +36,22 @@ logger = logging.getLogger(__name__) _VALID_URI = re.compile(br"\A[\x21-\x7e]+\Z") +@attr.s +class ProxyCredentials: + username_password = attr.ib(type=bytes) + + def as_proxy_authorization_value(self) -> bytes: + """ + Return the value for a Proxy-Authorization header (i.e. 'Basic abdef=='). + + Returns: + A transformation of the authentication string the encoded value for + a Proxy-Authorization header. + """ + # Encode as base64 and prepend the authorization type + return b"Basic " + base64.encodebytes(self.username_password) + + @implementer(IAgent) class ProxyAgent(_AgentBase): """An Agent implementation which will use an HTTP proxy if one was requested @@ -96,6 +116,9 @@ class ProxyAgent(_AgentBase): https_proxy = proxies["https"].encode() if "https" in proxies else None no_proxy = proxies["no"] if "no" in proxies else None + # Parse credentials from https proxy connection string if present + self.https_proxy_creds, https_proxy = parse_username_password(https_proxy) + self.http_proxy_endpoint = _http_proxy_endpoint( http_proxy, self.proxy_reactor, **self._endpoint_kwargs ) @@ -175,11 +198,22 @@ class ProxyAgent(_AgentBase): and self.https_proxy_endpoint and not should_skip_proxy ): + connect_headers = Headers() + + # Determine whether we need to set Proxy-Authorization headers + if self.https_proxy_creds: + # Set a Proxy-Authorization header + connect_headers.addRawHeader( + b"Proxy-Authorization", + self.https_proxy_creds.as_proxy_authorization_value(), + ) + endpoint = HTTPConnectProxyEndpoint( self.proxy_reactor, self.https_proxy_endpoint, parsed_uri.host, parsed_uri.port, + headers=connect_headers, ) else: # not using a proxy @@ -208,12 +242,16 @@ class ProxyAgent(_AgentBase): ) -def _http_proxy_endpoint(proxy, reactor, **kwargs): +def _http_proxy_endpoint(proxy: Optional[bytes], reactor, **kwargs): """Parses an http proxy setting and returns an endpoint for the proxy Args: - proxy (bytes|None): the proxy setting + proxy: the proxy setting in the form: [:@][:] + Note that compared to other apps, this function currently lacks support + for specifying a protocol schema (i.e. protocol://...). + reactor: reactor to be used to connect to the proxy + kwargs: other args to be passed to HostnameEndpoint Returns: @@ -223,16 +261,43 @@ def _http_proxy_endpoint(proxy, reactor, **kwargs): if proxy is None: return None - # currently we only support hostname:port. Some apps also support - # protocol://[:port], which allows a way of requiring a TLS connection to the - # proxy. - + # Parse the connection string host, port = parse_host_port(proxy, default_port=1080) return HostnameEndpoint(reactor, host, port, **kwargs) -def parse_host_port(hostport, default_port=None): - # could have sworn we had one of these somewhere else... +def parse_username_password(proxy: bytes) -> Tuple[Optional[ProxyCredentials], bytes]: + """ + Parses the username and password from a proxy declaration e.g + username:password@hostname:port. + + Args: + proxy: The proxy connection string. + + Returns + An instance of ProxyCredentials and the proxy connection string with any credentials + stripped, i.e u:p@host:port -> host:port. If no credentials were found, the + ProxyCredentials instance is replaced with None. + """ + if proxy and b"@" in proxy: + # We use rsplit here as the password could contain an @ character + credentials, proxy_without_credentials = proxy.rsplit(b"@", 1) + return ProxyCredentials(credentials), proxy_without_credentials + + return None, proxy + + +def parse_host_port(hostport: bytes, default_port: int = None) -> Tuple[bytes, int]: + """ + Parse the hostname and port from a proxy connection byte string. + + Args: + hostport: The proxy connection string. Must be in the form 'host[:port]'. + default_port: The default port to return if one is not found in `hostport`. + + Returns: + A tuple containing the hostname and port. Uses `default_port` if one was not found. + """ if b":" in hostport: host, port = hostport.rsplit(b":", 1) try: diff --git a/tests/http/test_proxyagent.py b/tests/http/test_proxyagent.py index 505ffcd300..3ea8b5bec7 100644 --- a/tests/http/test_proxyagent.py +++ b/tests/http/test_proxyagent.py @@ -12,8 +12,10 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. +import base64 import logging import os +from typing import Optional from unittest.mock import patch import treq @@ -242,6 +244,21 @@ class MatrixFederationAgentTests(TestCase): @patch.dict(os.environ, {"https_proxy": "proxy.com", "no_proxy": "unused.com"}) def test_https_request_via_proxy(self): + """Tests that TLS-encrypted requests can be made through a proxy""" + self._do_https_request_via_proxy(auth_credentials=None) + + @patch.dict( + os.environ, + {"https_proxy": "bob:pinkponies@proxy.com", "no_proxy": "unused.com"}, + ) + def test_https_request_via_proxy_with_auth(self): + """Tests that authenticated, TLS-encrypted requests can be made through a proxy""" + self._do_https_request_via_proxy(auth_credentials="bob:pinkponies") + + def _do_https_request_via_proxy( + self, + auth_credentials: Optional[str] = None, + ): agent = ProxyAgent( self.reactor, contextFactory=get_test_https_policy(), @@ -278,6 +295,22 @@ class MatrixFederationAgentTests(TestCase): self.assertEqual(request.method, b"CONNECT") self.assertEqual(request.path, b"test.com:443") + # Check whether auth credentials have been supplied to the proxy + proxy_auth_header_values = request.requestHeaders.getRawHeaders( + b"Proxy-Authorization" + ) + + if auth_credentials is not None: + # Compute the correct header value for Proxy-Authorization + encoded_credentials = base64.b64encode(b"bob:pinkponies") + expected_header_value = b"Basic " + encoded_credentials + + # Validate the header's value + self.assertIn(expected_header_value, proxy_auth_header_values) + else: + # Check that the Proxy-Authorization header has not been supplied to the proxy + self.assertIsNone(proxy_auth_header_values) + # tell the proxy server not to close the connection proxy_server.persistent = True @@ -312,6 +345,13 @@ class MatrixFederationAgentTests(TestCase): self.assertEqual(request.method, b"GET") self.assertEqual(request.path, b"/abc") self.assertEqual(request.requestHeaders.getRawHeaders(b"host"), [b"test.com"]) + + # Check that the destination server DID NOT receive proxy credentials + proxy_auth_header_values = request.requestHeaders.getRawHeaders( + b"Proxy-Authorization" + ) + self.assertIsNone(proxy_auth_header_values) + request.write(b"result") request.finish() -- cgit 1.5.1 From b7748d3c00e87df8c49346e67d643916487254e4 Mon Sep 17 00:00:00 2001 From: Patrick Cloke Date: Tue, 23 Mar 2021 07:12:48 -0400 Subject: Import HomeServer from the proper module. (#9665) --- changelog.d/9665.misc | 1 + synapse/crypto/keyring.py | 2 +- synapse/federation/federation_client.py | 2 +- synapse/groups/attestations.py | 2 +- synapse/groups/groups_server.py | 2 +- synapse/handlers/_base.py | 2 +- synapse/handlers/account_data.py | 2 +- synapse/handlers/account_validity.py | 2 +- synapse/handlers/acme.py | 2 +- synapse/handlers/admin.py | 2 +- synapse/handlers/appservice.py | 2 +- synapse/handlers/auth.py | 2 +- synapse/handlers/cas_handler.py | 2 +- synapse/handlers/deactivate_account.py | 2 +- synapse/handlers/device.py | 2 +- synapse/handlers/devicemessage.py | 2 +- synapse/handlers/e2e_keys.py | 2 +- synapse/handlers/e2e_room_keys.py | 2 +- synapse/handlers/groups_local.py | 2 +- synapse/handlers/password_policy.py | 2 +- synapse/handlers/profile.py | 2 +- synapse/handlers/read_marker.py | 2 +- synapse/handlers/receipts.py | 2 +- synapse/handlers/register.py | 2 +- synapse/handlers/room_list.py | 2 +- synapse/handlers/room_member_worker.py | 2 +- synapse/handlers/search.py | 2 +- synapse/handlers/set_password.py | 2 +- synapse/handlers/state_deltas.py | 2 +- synapse/handlers/stats.py | 2 +- synapse/handlers/user_directory.py | 2 +- synapse/http/client.py | 2 +- synapse/push/__init__.py | 2 +- synapse/push/action_generator.py | 2 +- synapse/push/bulk_push_rule_evaluator.py | 2 +- synapse/push/emailpusher.py | 2 +- synapse/push/httppusher.py | 2 +- synapse/push/mailer.py | 2 +- synapse/push/pusher.py | 2 +- synapse/replication/slave/storage/pushers.py | 2 +- synapse/replication/tcp/streams/_base.py | 2 +- synapse/rest/admin/media.py | 2 +- synapse/rest/client/v1/room.py | 2 +- synapse/rest/client/v2_alpha/account.py | 2 +- synapse/rest/client/v2_alpha/groups.py | 2 +- synapse/rest/media/v1/config_resource.py | 2 +- synapse/rest/media/v1/download_resource.py | 2 +- synapse/rest/media/v1/media_repository.py | 2 +- synapse/rest/media/v1/preview_url_resource.py | 2 +- synapse/rest/media/v1/storage_provider.py | 2 +- synapse/rest/media/v1/thumbnail_resource.py | 2 +- synapse/rest/media/v1/upload_resource.py | 2 +- synapse/storage/__init__.py | 2 +- synapse/storage/_base.py | 2 +- synapse/storage/background_updates.py | 2 +- synapse/storage/databases/main/appservice.py | 2 +- synapse/storage/databases/main/pusher.py | 2 +- synapse/storage/purge_events.py | 2 +- synapse/storage/state.py | 2 +- 59 files changed, 59 insertions(+), 58 deletions(-) create mode 100644 changelog.d/9665.misc diff --git a/changelog.d/9665.misc b/changelog.d/9665.misc new file mode 100644 index 0000000000..b8bf76c639 --- /dev/null +++ b/changelog.d/9665.misc @@ -0,0 +1 @@ +Import `HomeServer` from the proper module. diff --git a/synapse/crypto/keyring.py b/synapse/crypto/keyring.py index 902128a23c..d5fb51513b 100644 --- a/synapse/crypto/keyring.py +++ b/synapse/crypto/keyring.py @@ -57,7 +57,7 @@ from synapse.util.metrics import Measure from synapse.util.retryutils import NotRetryingDestination if TYPE_CHECKING: - from synapse.app.homeserver import HomeServer + from synapse.server import HomeServer logger = logging.getLogger(__name__) diff --git a/synapse/federation/federation_client.py b/synapse/federation/federation_client.py index bee81fc019..3b2f51baab 100644 --- a/synapse/federation/federation_client.py +++ b/synapse/federation/federation_client.py @@ -62,7 +62,7 @@ from synapse.util.caches.expiringcache import ExpiringCache from synapse.util.retryutils import NotRetryingDestination if TYPE_CHECKING: - from synapse.app.homeserver import HomeServer + from synapse.server import HomeServer logger = logging.getLogger(__name__) diff --git a/synapse/groups/attestations.py b/synapse/groups/attestations.py index a3f8d92d08..368c44708d 100644 --- a/synapse/groups/attestations.py +++ b/synapse/groups/attestations.py @@ -46,7 +46,7 @@ from synapse.metrics.background_process_metrics import run_as_background_process from synapse.types import JsonDict, get_domain_from_id if TYPE_CHECKING: - from synapse.app.homeserver import HomeServer + from synapse.server import HomeServer logger = logging.getLogger(__name__) diff --git a/synapse/groups/groups_server.py b/synapse/groups/groups_server.py index f9a0f40221..4b16a4ac29 100644 --- a/synapse/groups/groups_server.py +++ b/synapse/groups/groups_server.py @@ -25,7 +25,7 @@ from synapse.types import GroupID, JsonDict, RoomID, UserID, get_domain_from_id from synapse.util.async_helpers import concurrently_execute if TYPE_CHECKING: - from synapse.app.homeserver import HomeServer + from synapse.server import HomeServer logger = logging.getLogger(__name__) diff --git a/synapse/handlers/_base.py b/synapse/handlers/_base.py index d29b066a56..aade2c4a3a 100644 --- a/synapse/handlers/_base.py +++ b/synapse/handlers/_base.py @@ -24,7 +24,7 @@ from synapse.api.ratelimiting import Ratelimiter from synapse.types import UserID if TYPE_CHECKING: - from synapse.app.homeserver import HomeServer + from synapse.server import HomeServer logger = logging.getLogger(__name__) diff --git a/synapse/handlers/account_data.py b/synapse/handlers/account_data.py index b1a5df9638..1ce6d697ed 100644 --- a/synapse/handlers/account_data.py +++ b/synapse/handlers/account_data.py @@ -25,7 +25,7 @@ from synapse.replication.http.account_data import ( from synapse.types import JsonDict, UserID if TYPE_CHECKING: - from synapse.app.homeserver import HomeServer + from synapse.server import HomeServer class AccountDataHandler: diff --git a/synapse/handlers/account_validity.py b/synapse/handlers/account_validity.py index 664d09da1c..d781bb251d 100644 --- a/synapse/handlers/account_validity.py +++ b/synapse/handlers/account_validity.py @@ -27,7 +27,7 @@ from synapse.types import UserID from synapse.util import stringutils if TYPE_CHECKING: - from synapse.app.homeserver import HomeServer + from synapse.server import HomeServer logger = logging.getLogger(__name__) diff --git a/synapse/handlers/acme.py b/synapse/handlers/acme.py index 132be238dd..2a25af6288 100644 --- a/synapse/handlers/acme.py +++ b/synapse/handlers/acme.py @@ -24,7 +24,7 @@ from twisted.web.resource import Resource from synapse.app import check_bind_error if TYPE_CHECKING: - from synapse.app.homeserver import HomeServer + from synapse.server import HomeServer logger = logging.getLogger(__name__) diff --git a/synapse/handlers/admin.py b/synapse/handlers/admin.py index db68c94c50..c494de49a3 100644 --- a/synapse/handlers/admin.py +++ b/synapse/handlers/admin.py @@ -25,7 +25,7 @@ from synapse.visibility import filter_events_for_client from ._base import BaseHandler if TYPE_CHECKING: - from synapse.app.homeserver import HomeServer + from synapse.server import HomeServer logger = logging.getLogger(__name__) diff --git a/synapse/handlers/appservice.py b/synapse/handlers/appservice.py index deab8ff2d0..996f9e5deb 100644 --- a/synapse/handlers/appservice.py +++ b/synapse/handlers/appservice.py @@ -38,7 +38,7 @@ from synapse.types import Collection, JsonDict, RoomAlias, RoomStreamToken, User from synapse.util.metrics import Measure if TYPE_CHECKING: - from synapse.app.homeserver import HomeServer + from synapse.server import HomeServer logger = logging.getLogger(__name__) diff --git a/synapse/handlers/auth.py b/synapse/handlers/auth.py index badac8c26c..d537ea8137 100644 --- a/synapse/handlers/auth.py +++ b/synapse/handlers/auth.py @@ -70,7 +70,7 @@ from synapse.util.msisdn import phone_number_to_msisdn from synapse.util.threepids import canonicalise_email if TYPE_CHECKING: - from synapse.app.homeserver import HomeServer + from synapse.server import HomeServer logger = logging.getLogger(__name__) diff --git a/synapse/handlers/cas_handler.py b/synapse/handlers/cas_handler.py index cb67589f7d..5060936f94 100644 --- a/synapse/handlers/cas_handler.py +++ b/synapse/handlers/cas_handler.py @@ -27,7 +27,7 @@ from synapse.http.site import SynapseRequest from synapse.types import UserID, map_username_to_mxid_localpart if TYPE_CHECKING: - from synapse.app.homeserver import HomeServer + from synapse.server import HomeServer logger = logging.getLogger(__name__) diff --git a/synapse/handlers/deactivate_account.py b/synapse/handlers/deactivate_account.py index 3886d3124d..2bcd8f5435 100644 --- a/synapse/handlers/deactivate_account.py +++ b/synapse/handlers/deactivate_account.py @@ -23,7 +23,7 @@ from synapse.types import Requester, UserID, create_requester from ._base import BaseHandler if TYPE_CHECKING: - from synapse.app.homeserver import HomeServer + from synapse.server import HomeServer logger = logging.getLogger(__name__) diff --git a/synapse/handlers/device.py b/synapse/handlers/device.py index 2fc4951df4..54293d0b9c 100644 --- a/synapse/handlers/device.py +++ b/synapse/handlers/device.py @@ -45,7 +45,7 @@ from synapse.util.retryutils import NotRetryingDestination from ._base import BaseHandler if TYPE_CHECKING: - from synapse.app.homeserver import HomeServer + from synapse.server import HomeServer logger = logging.getLogger(__name__) diff --git a/synapse/handlers/devicemessage.py b/synapse/handlers/devicemessage.py index 7db4f48965..eb547743be 100644 --- a/synapse/handlers/devicemessage.py +++ b/synapse/handlers/devicemessage.py @@ -32,7 +32,7 @@ from synapse.util import json_encoder from synapse.util.stringutils import random_string if TYPE_CHECKING: - from synapse.app.homeserver import HomeServer + from synapse.server import HomeServer logger = logging.getLogger(__name__) diff --git a/synapse/handlers/e2e_keys.py b/synapse/handlers/e2e_keys.py index 9a946a3cfe..2ad9b6d930 100644 --- a/synapse/handlers/e2e_keys.py +++ b/synapse/handlers/e2e_keys.py @@ -42,7 +42,7 @@ from synapse.util.caches.expiringcache import ExpiringCache from synapse.util.retryutils import NotRetryingDestination if TYPE_CHECKING: - from synapse.app.homeserver import HomeServer + from synapse.server import HomeServer logger = logging.getLogger(__name__) diff --git a/synapse/handlers/e2e_room_keys.py b/synapse/handlers/e2e_room_keys.py index 622cae23be..a910d246d6 100644 --- a/synapse/handlers/e2e_room_keys.py +++ b/synapse/handlers/e2e_room_keys.py @@ -29,7 +29,7 @@ from synapse.types import JsonDict from synapse.util.async_helpers import Linearizer if TYPE_CHECKING: - from synapse.app.homeserver import HomeServer + from synapse.server import HomeServer logger = logging.getLogger(__name__) diff --git a/synapse/handlers/groups_local.py b/synapse/handlers/groups_local.py index bfb95e3eee..a41ca5df9c 100644 --- a/synapse/handlers/groups_local.py +++ b/synapse/handlers/groups_local.py @@ -21,7 +21,7 @@ from synapse.api.errors import HttpResponseException, RequestSendFailed, Synapse from synapse.types import GroupID, JsonDict, get_domain_from_id if TYPE_CHECKING: - from synapse.app.homeserver import HomeServer + from synapse.server import HomeServer logger = logging.getLogger(__name__) diff --git a/synapse/handlers/password_policy.py b/synapse/handlers/password_policy.py index 6c635cc31b..92cefa11aa 100644 --- a/synapse/handlers/password_policy.py +++ b/synapse/handlers/password_policy.py @@ -21,7 +21,7 @@ from typing import TYPE_CHECKING from synapse.api.errors import Codes, PasswordRefusedError if TYPE_CHECKING: - from synapse.app.homeserver import HomeServer + from synapse.server import HomeServer logger = logging.getLogger(__name__) diff --git a/synapse/handlers/profile.py b/synapse/handlers/profile.py index dd59392bda..a755363c3f 100644 --- a/synapse/handlers/profile.py +++ b/synapse/handlers/profile.py @@ -36,7 +36,7 @@ from synapse.types import ( from ._base import BaseHandler if TYPE_CHECKING: - from synapse.app.homeserver import HomeServer + from synapse.server import HomeServer logger = logging.getLogger(__name__) diff --git a/synapse/handlers/read_marker.py b/synapse/handlers/read_marker.py index 6bb2fd936b..a54fe1968e 100644 --- a/synapse/handlers/read_marker.py +++ b/synapse/handlers/read_marker.py @@ -21,7 +21,7 @@ from synapse.util.async_helpers import Linearizer from ._base import BaseHandler if TYPE_CHECKING: - from synapse.app.homeserver import HomeServer + from synapse.server import HomeServer logger = logging.getLogger(__name__) diff --git a/synapse/handlers/receipts.py b/synapse/handlers/receipts.py index 6a6c528849..dbfe9bfaca 100644 --- a/synapse/handlers/receipts.py +++ b/synapse/handlers/receipts.py @@ -20,7 +20,7 @@ from synapse.handlers._base import BaseHandler from synapse.types import JsonDict, ReadReceipt, get_domain_from_id if TYPE_CHECKING: - from synapse.app.homeserver import HomeServer + from synapse.server import HomeServer logger = logging.getLogger(__name__) diff --git a/synapse/handlers/register.py b/synapse/handlers/register.py index d7f226d589..0fc2bf15d5 100644 --- a/synapse/handlers/register.py +++ b/synapse/handlers/register.py @@ -38,7 +38,7 @@ from synapse.types import RoomAlias, UserID, create_requester from ._base import BaseHandler if TYPE_CHECKING: - from synapse.app.homeserver import HomeServer + from synapse.server import HomeServer logger = logging.getLogger(__name__) diff --git a/synapse/handlers/room_list.py b/synapse/handlers/room_list.py index 8bfc46c654..924b81db7c 100644 --- a/synapse/handlers/room_list.py +++ b/synapse/handlers/room_list.py @@ -29,7 +29,7 @@ from synapse.util.caches.response_cache import ResponseCache from ._base import BaseHandler if TYPE_CHECKING: - from synapse.app.homeserver import HomeServer + from synapse.server import HomeServer logger = logging.getLogger(__name__) diff --git a/synapse/handlers/room_member_worker.py b/synapse/handlers/room_member_worker.py index d75506c75e..3a90fc0c16 100644 --- a/synapse/handlers/room_member_worker.py +++ b/synapse/handlers/room_member_worker.py @@ -26,7 +26,7 @@ from synapse.replication.http.membership import ( from synapse.types import Requester, UserID if TYPE_CHECKING: - from synapse.app.homeserver import HomeServer + from synapse.server import HomeServer logger = logging.getLogger(__name__) diff --git a/synapse/handlers/search.py b/synapse/handlers/search.py index 94062e79cb..d742dfbd53 100644 --- a/synapse/handlers/search.py +++ b/synapse/handlers/search.py @@ -30,7 +30,7 @@ from synapse.visibility import filter_events_for_client from ._base import BaseHandler if TYPE_CHECKING: - from synapse.app.homeserver import HomeServer + from synapse.server import HomeServer logger = logging.getLogger(__name__) diff --git a/synapse/handlers/set_password.py b/synapse/handlers/set_password.py index 04e7c64c94..f98a338ec5 100644 --- a/synapse/handlers/set_password.py +++ b/synapse/handlers/set_password.py @@ -21,7 +21,7 @@ from synapse.types import Requester from ._base import BaseHandler if TYPE_CHECKING: - from synapse.app.homeserver import HomeServer + from synapse.server import HomeServer logger = logging.getLogger(__name__) diff --git a/synapse/handlers/state_deltas.py b/synapse/handlers/state_deltas.py index b3f9875358..ee8f87e59a 100644 --- a/synapse/handlers/state_deltas.py +++ b/synapse/handlers/state_deltas.py @@ -17,7 +17,7 @@ import logging from typing import TYPE_CHECKING, Optional if TYPE_CHECKING: - from synapse.app.homeserver import HomeServer + from synapse.server import HomeServer logger = logging.getLogger(__name__) diff --git a/synapse/handlers/stats.py b/synapse/handlers/stats.py index 924281144c..8730f99d03 100644 --- a/synapse/handlers/stats.py +++ b/synapse/handlers/stats.py @@ -24,7 +24,7 @@ from synapse.metrics.background_process_metrics import run_as_background_process from synapse.types import JsonDict if TYPE_CHECKING: - from synapse.app.homeserver import HomeServer + from synapse.server import HomeServer logger = logging.getLogger(__name__) diff --git a/synapse/handlers/user_directory.py b/synapse/handlers/user_directory.py index 1a8340000a..b121286d95 100644 --- a/synapse/handlers/user_directory.py +++ b/synapse/handlers/user_directory.py @@ -25,7 +25,7 @@ from synapse.types import JsonDict from synapse.util.metrics import Measure if TYPE_CHECKING: - from synapse.app.homeserver import HomeServer + from synapse.server import HomeServer logger = logging.getLogger(__name__) diff --git a/synapse/http/client.py b/synapse/http/client.py index 1e01e0a9f2..a0caba84e4 100644 --- a/synapse/http/client.py +++ b/synapse/http/client.py @@ -77,7 +77,7 @@ from synapse.util import json_decoder from synapse.util.async_helpers import timeout_deferred if TYPE_CHECKING: - from synapse.app.homeserver import HomeServer + from synapse.server import HomeServer logger = logging.getLogger(__name__) diff --git a/synapse/push/__init__.py b/synapse/push/__init__.py index f4f7ec96f8..9fc3da49a2 100644 --- a/synapse/push/__init__.py +++ b/synapse/push/__init__.py @@ -21,7 +21,7 @@ import attr from synapse.types import JsonDict, RoomStreamToken if TYPE_CHECKING: - from synapse.app.homeserver import HomeServer + from synapse.server import HomeServer @attr.s(slots=True) diff --git a/synapse/push/action_generator.py b/synapse/push/action_generator.py index aaed28650d..38a47a600f 100644 --- a/synapse/push/action_generator.py +++ b/synapse/push/action_generator.py @@ -22,7 +22,7 @@ from synapse.push.bulk_push_rule_evaluator import BulkPushRuleEvaluator from synapse.util.metrics import Measure if TYPE_CHECKING: - from synapse.app.homeserver import HomeServer + from synapse.server import HomeServer logger = logging.getLogger(__name__) diff --git a/synapse/push/bulk_push_rule_evaluator.py b/synapse/push/bulk_push_rule_evaluator.py index c016a83909..1897f59153 100644 --- a/synapse/push/bulk_push_rule_evaluator.py +++ b/synapse/push/bulk_push_rule_evaluator.py @@ -33,7 +33,7 @@ from synapse.util.caches.lrucache import LruCache from .push_rule_evaluator import PushRuleEvaluatorForEvent if TYPE_CHECKING: - from synapse.app.homeserver import HomeServer + from synapse.server import HomeServer logger = logging.getLogger(__name__) diff --git a/synapse/push/emailpusher.py b/synapse/push/emailpusher.py index 3dc06a79e8..c0968dc7a1 100644 --- a/synapse/push/emailpusher.py +++ b/synapse/push/emailpusher.py @@ -24,7 +24,7 @@ from synapse.push import Pusher, PusherConfig, ThrottleParams from synapse.push.mailer import Mailer if TYPE_CHECKING: - from synapse.app.homeserver import HomeServer + from synapse.server import HomeServer logger = logging.getLogger(__name__) diff --git a/synapse/push/httppusher.py b/synapse/push/httppusher.py index 026134ae26..26af5309c1 100644 --- a/synapse/push/httppusher.py +++ b/synapse/push/httppusher.py @@ -31,7 +31,7 @@ from synapse.push import Pusher, PusherConfig, PusherConfigException from . import push_rule_evaluator, push_tools if TYPE_CHECKING: - from synapse.app.homeserver import HomeServer + from synapse.server import HomeServer logger = logging.getLogger(__name__) diff --git a/synapse/push/mailer.py b/synapse/push/mailer.py index d10201b6b3..2e5161de2c 100644 --- a/synapse/push/mailer.py +++ b/synapse/push/mailer.py @@ -40,7 +40,7 @@ from synapse.util.async_helpers import concurrently_execute from synapse.visibility import filter_events_for_client if TYPE_CHECKING: - from synapse.app.homeserver import HomeServer + from synapse.server import HomeServer logger = logging.getLogger(__name__) diff --git a/synapse/push/pusher.py b/synapse/push/pusher.py index 2aa7918fb4..cb94127850 100644 --- a/synapse/push/pusher.py +++ b/synapse/push/pusher.py @@ -22,7 +22,7 @@ from synapse.push.httppusher import HttpPusher from synapse.push.mailer import Mailer if TYPE_CHECKING: - from synapse.app.homeserver import HomeServer + from synapse.server import HomeServer logger = logging.getLogger(__name__) diff --git a/synapse/replication/slave/storage/pushers.py b/synapse/replication/slave/storage/pushers.py index 045bd014da..93161c3dfb 100644 --- a/synapse/replication/slave/storage/pushers.py +++ b/synapse/replication/slave/storage/pushers.py @@ -24,7 +24,7 @@ from ._base import BaseSlavedStore from ._slaved_id_tracker import SlavedIdTracker if TYPE_CHECKING: - from synapse.app.homeserver import HomeServer + from synapse.server import HomeServer class SlavedPusherStore(PusherWorkerStore, BaseSlavedStore): diff --git a/synapse/replication/tcp/streams/_base.py b/synapse/replication/tcp/streams/_base.py index 7e8e64d61c..3dfee76743 100644 --- a/synapse/replication/tcp/streams/_base.py +++ b/synapse/replication/tcp/streams/_base.py @@ -33,7 +33,7 @@ import attr from synapse.replication.http.streams import ReplicationGetStreamUpdates if TYPE_CHECKING: - from synapse.app.homeserver import HomeServer + from synapse.server import HomeServer logger = logging.getLogger(__name__) diff --git a/synapse/rest/admin/media.py b/synapse/rest/admin/media.py index 7fcc48a9d7..40646ef241 100644 --- a/synapse/rest/admin/media.py +++ b/synapse/rest/admin/media.py @@ -28,7 +28,7 @@ from synapse.rest.admin._base import ( from synapse.types import JsonDict if TYPE_CHECKING: - from synapse.app.homeserver import HomeServer + from synapse.server import HomeServer logger = logging.getLogger(__name__) diff --git a/synapse/rest/client/v1/room.py b/synapse/rest/client/v1/room.py index 6c722d634d..525efdf221 100644 --- a/synapse/rest/client/v1/room.py +++ b/synapse/rest/client/v1/room.py @@ -58,7 +58,7 @@ from synapse.util import json_decoder from synapse.util.stringutils import parse_and_validate_server_name, random_string if TYPE_CHECKING: - from synapse.app.homeserver import HomeServer + from synapse.server import HomeServer logger = logging.getLogger(__name__) diff --git a/synapse/rest/client/v2_alpha/account.py b/synapse/rest/client/v2_alpha/account.py index adf1d39728..c2ba790bab 100644 --- a/synapse/rest/client/v2_alpha/account.py +++ b/synapse/rest/client/v2_alpha/account.py @@ -45,7 +45,7 @@ from synapse.util.threepids import canonicalise_email, check_3pid_allowed from ._base import client_patterns, interactive_auth_handler if TYPE_CHECKING: - from synapse.app.homeserver import HomeServer + from synapse.server import HomeServer logger = logging.getLogger(__name__) diff --git a/synapse/rest/client/v2_alpha/groups.py b/synapse/rest/client/v2_alpha/groups.py index 5901432fad..08fb6b2b06 100644 --- a/synapse/rest/client/v2_alpha/groups.py +++ b/synapse/rest/client/v2_alpha/groups.py @@ -38,7 +38,7 @@ from synapse.types import GroupID, JsonDict from ._base import client_patterns if TYPE_CHECKING: - from synapse.app.homeserver import HomeServer + from synapse.server import HomeServer logger = logging.getLogger(__name__) diff --git a/synapse/rest/media/v1/config_resource.py b/synapse/rest/media/v1/config_resource.py index 1eff98ef14..c41a7ab412 100644 --- a/synapse/rest/media/v1/config_resource.py +++ b/synapse/rest/media/v1/config_resource.py @@ -23,7 +23,7 @@ from synapse.http.server import DirectServeJsonResource, respond_with_json from synapse.http.site import SynapseRequest if TYPE_CHECKING: - from synapse.app.homeserver import HomeServer + from synapse.server import HomeServer class MediaConfigResource(DirectServeJsonResource): diff --git a/synapse/rest/media/v1/download_resource.py b/synapse/rest/media/v1/download_resource.py index 8a43581f1f..5dadaeaf57 100644 --- a/synapse/rest/media/v1/download_resource.py +++ b/synapse/rest/media/v1/download_resource.py @@ -24,8 +24,8 @@ from synapse.http.servlet import parse_boolean from ._base import parse_media_id, respond_404 if TYPE_CHECKING: - from synapse.app.homeserver import HomeServer from synapse.rest.media.v1.media_repository import MediaRepository + from synapse.server import HomeServer logger = logging.getLogger(__name__) diff --git a/synapse/rest/media/v1/media_repository.py b/synapse/rest/media/v1/media_repository.py index 8b4841ed5d..0c041b542d 100644 --- a/synapse/rest/media/v1/media_repository.py +++ b/synapse/rest/media/v1/media_repository.py @@ -58,7 +58,7 @@ from .thumbnailer import Thumbnailer, ThumbnailError from .upload_resource import UploadResource if TYPE_CHECKING: - from synapse.app.homeserver import HomeServer + from synapse.server import HomeServer logger = logging.getLogger(__name__) diff --git a/synapse/rest/media/v1/preview_url_resource.py b/synapse/rest/media/v1/preview_url_resource.py index b8895aeaa9..e590a0deab 100644 --- a/synapse/rest/media/v1/preview_url_resource.py +++ b/synapse/rest/media/v1/preview_url_resource.py @@ -54,8 +54,8 @@ from ._base import FileInfo if TYPE_CHECKING: from lxml import etree - from synapse.app.homeserver import HomeServer from synapse.rest.media.v1.media_repository import MediaRepository + from synapse.server import HomeServer logger = logging.getLogger(__name__) diff --git a/synapse/rest/media/v1/storage_provider.py b/synapse/rest/media/v1/storage_provider.py index e92006faa9..031947557d 100644 --- a/synapse/rest/media/v1/storage_provider.py +++ b/synapse/rest/media/v1/storage_provider.py @@ -29,7 +29,7 @@ from .media_storage import FileResponder logger = logging.getLogger(__name__) if TYPE_CHECKING: - from synapse.app.homeserver import HomeServer + from synapse.server import HomeServer class StorageProvider(metaclass=abc.ABCMeta): diff --git a/synapse/rest/media/v1/thumbnail_resource.py b/synapse/rest/media/v1/thumbnail_resource.py index fbcd50f1e2..af802bc0b1 100644 --- a/synapse/rest/media/v1/thumbnail_resource.py +++ b/synapse/rest/media/v1/thumbnail_resource.py @@ -34,8 +34,8 @@ from ._base import ( ) if TYPE_CHECKING: - from synapse.app.homeserver import HomeServer from synapse.rest.media.v1.media_repository import MediaRepository + from synapse.server import HomeServer logger = logging.getLogger(__name__) diff --git a/synapse/rest/media/v1/upload_resource.py b/synapse/rest/media/v1/upload_resource.py index ae5aef2f7f..0138b2e2d1 100644 --- a/synapse/rest/media/v1/upload_resource.py +++ b/synapse/rest/media/v1/upload_resource.py @@ -26,8 +26,8 @@ from synapse.http.site import SynapseRequest from synapse.rest.media.v1.media_storage import SpamMediaException if TYPE_CHECKING: - from synapse.app.homeserver import HomeServer from synapse.rest.media.v1.media_repository import MediaRepository + from synapse.server import HomeServer logger = logging.getLogger(__name__) diff --git a/synapse/storage/__init__.py b/synapse/storage/__init__.py index a3c52695e9..0b9007e51f 100644 --- a/synapse/storage/__init__.py +++ b/synapse/storage/__init__.py @@ -36,7 +36,7 @@ from synapse.storage.purge_events import PurgeEventsStorage from synapse.storage.state import StateGroupStorage if TYPE_CHECKING: - from synapse.app.homeserver import HomeServer + from synapse.server import HomeServer __all__ = ["Databases", "DataStore"] diff --git a/synapse/storage/_base.py b/synapse/storage/_base.py index a25c4093bc..240905329f 100644 --- a/synapse/storage/_base.py +++ b/synapse/storage/_base.py @@ -27,7 +27,7 @@ from synapse.types import Collection, StreamToken, get_domain_from_id from synapse.util import json_decoder if TYPE_CHECKING: - from synapse.app.homeserver import HomeServer + from synapse.server import HomeServer logger = logging.getLogger(__name__) diff --git a/synapse/storage/background_updates.py b/synapse/storage/background_updates.py index 329660cf0f..ccb06aab39 100644 --- a/synapse/storage/background_updates.py +++ b/synapse/storage/background_updates.py @@ -23,7 +23,7 @@ from synapse.util import json_encoder from . import engines if TYPE_CHECKING: - from synapse.app.homeserver import HomeServer + from synapse.server import HomeServer from synapse.storage.database import DatabasePool, LoggingTransaction logger = logging.getLogger(__name__) diff --git a/synapse/storage/databases/main/appservice.py b/synapse/storage/databases/main/appservice.py index 03a38422a1..85bb853d33 100644 --- a/synapse/storage/databases/main/appservice.py +++ b/synapse/storage/databases/main/appservice.py @@ -32,7 +32,7 @@ from synapse.types import JsonDict from synapse.util import json_encoder if TYPE_CHECKING: - from synapse.app.homeserver import HomeServer + from synapse.server import HomeServer logger = logging.getLogger(__name__) diff --git a/synapse/storage/databases/main/pusher.py b/synapse/storage/databases/main/pusher.py index 85f1ebac98..c65558c280 100644 --- a/synapse/storage/databases/main/pusher.py +++ b/synapse/storage/databases/main/pusher.py @@ -27,7 +27,7 @@ from synapse.util import json_encoder from synapse.util.caches.descriptors import cached, cachedList if TYPE_CHECKING: - from synapse.app.homeserver import HomeServer + from synapse.server import HomeServer logger = logging.getLogger(__name__) diff --git a/synapse/storage/purge_events.py b/synapse/storage/purge_events.py index 4dcd848c59..ad954990a7 100644 --- a/synapse/storage/purge_events.py +++ b/synapse/storage/purge_events.py @@ -20,7 +20,7 @@ from typing import TYPE_CHECKING, Set from synapse.storage.databases import Databases if TYPE_CHECKING: - from synapse.app.homeserver import HomeServer + from synapse.server import HomeServer logger = logging.getLogger(__name__) diff --git a/synapse/storage/state.py b/synapse/storage/state.py index d179a41884..aa25bd8350 100644 --- a/synapse/storage/state.py +++ b/synapse/storage/state.py @@ -32,7 +32,7 @@ from synapse.events import EventBase from synapse.types import MutableStateMap, StateMap if TYPE_CHECKING: - from synapse.app.homeserver import HomeServer + from synapse.server import HomeServer from synapse.storage.databases import Databases logger = logging.getLogger(__name__) -- cgit 1.5.1 From 4ecba9bd5cb9c094da864d05e3a10456c30bc409 Mon Sep 17 00:00:00 2001 From: Richard van der Hoff <1389908+richvdh@users.noreply.github.com> Date: Tue, 23 Mar 2021 11:51:12 +0000 Subject: Federation API for Space summary (#9652) Builds on the work done in #9643 to add a federation API for space summaries. There's a bit of refactoring of the existing client-server code first, to avoid too much duplication. --- changelog.d/9652.feature | 1 + synapse/federation/transport/server.py | 67 ++++++++++-- synapse/handlers/space_summary.py | 183 +++++++++++++++++++++++++-------- 3 files changed, 197 insertions(+), 54 deletions(-) create mode 100644 changelog.d/9652.feature diff --git a/changelog.d/9652.feature b/changelog.d/9652.feature new file mode 100644 index 0000000000..2f7ccedcfb --- /dev/null +++ b/changelog.d/9652.feature @@ -0,0 +1 @@ +Add initial experimental support for a "space summary" API. diff --git a/synapse/federation/transport/server.py b/synapse/federation/transport/server.py index 2cf935f38d..84e39c5a46 100644 --- a/synapse/federation/transport/server.py +++ b/synapse/federation/transport/server.py @@ -18,7 +18,7 @@ import functools import logging import re -from typing import Optional, Tuple, Type +from typing import Container, Mapping, Optional, Sequence, Tuple, Type import synapse from synapse.api.constants import MAX_GROUP_CATEGORYID_LENGTH, MAX_GROUP_ROLEID_LENGTH @@ -29,7 +29,7 @@ from synapse.api.urls import ( FEDERATION_V1_PREFIX, FEDERATION_V2_PREFIX, ) -from synapse.http.server import JsonResource +from synapse.http.server import HttpServer, JsonResource from synapse.http.servlet import ( parse_boolean_from_args, parse_integer_from_args, @@ -44,7 +44,8 @@ from synapse.logging.opentracing import ( whitelisted_homeserver, ) from synapse.server import HomeServer -from synapse.types import ThirdPartyInstanceID, get_domain_from_id +from synapse.types import JsonDict, ThirdPartyInstanceID, get_domain_from_id +from synapse.util.ratelimitutils import FederationRateLimiter from synapse.util.stringutils import parse_and_validate_server_name from synapse.util.versionstring import get_version_string @@ -1376,6 +1377,40 @@ class FederationGroupsSettingJoinPolicyServlet(BaseFederationServlet): return 200, new_content +class FederationSpaceSummaryServlet(BaseFederationServlet): + PREFIX = FEDERATION_UNSTABLE_PREFIX + "/org.matrix.msc2946" + PATH = "/spaces/(?P[^/]*)" + + async def on_POST( + self, + origin: str, + content: JsonDict, + query: Mapping[bytes, Sequence[bytes]], + room_id: str, + ) -> Tuple[int, JsonDict]: + suggested_only = content.get("suggested_only", False) + if not isinstance(suggested_only, bool): + raise SynapseError( + 400, "'suggested_only' must be a boolean", Codes.BAD_JSON + ) + + exclude_rooms = content.get("exclude_rooms", []) + if not isinstance(exclude_rooms, list) or any( + not isinstance(x, str) for x in exclude_rooms + ): + raise SynapseError(400, "bad value for 'exclude_rooms'", Codes.BAD_JSON) + + max_rooms_per_space = content.get("max_rooms_per_space") + if max_rooms_per_space is not None and not isinstance(max_rooms_per_space, int): + raise SynapseError( + 400, "bad value for 'max_rooms_per_space'", Codes.BAD_JSON + ) + + return 200, await self.handler.federation_space_summary( + room_id, suggested_only, max_rooms_per_space, exclude_rooms + ) + + class RoomComplexityServlet(BaseFederationServlet): """ Indicates to other servers how complex (and therefore likely @@ -1474,18 +1509,24 @@ DEFAULT_SERVLET_GROUPS = ( ) -def register_servlets(hs, resource, authenticator, ratelimiter, servlet_groups=None): +def register_servlets( + hs: HomeServer, + resource: HttpServer, + authenticator: Authenticator, + ratelimiter: FederationRateLimiter, + servlet_groups: Optional[Container[str]] = None, +): """Initialize and register servlet classes. Will by default register all servlets. For custom behaviour, pass in a list of servlet_groups to register. Args: - hs (synapse.server.HomeServer): homeserver - resource (JsonResource): resource class to register to - authenticator (Authenticator): authenticator to use - ratelimiter (util.ratelimitutils.FederationRateLimiter): ratelimiter to use - servlet_groups (list[str], optional): List of servlet groups to register. + hs: homeserver + resource: resource class to register to + authenticator: authenticator to use + ratelimiter: ratelimiter to use + servlet_groups: List of servlet groups to register. Defaults to ``DEFAULT_SERVLET_GROUPS``. """ if not servlet_groups: @@ -1500,6 +1541,14 @@ def register_servlets(hs, resource, authenticator, ratelimiter, servlet_groups=N server_name=hs.hostname, ).register(resource) + if hs.config.experimental.spaces_enabled: + FederationSpaceSummaryServlet( + handler=hs.get_space_summary_handler(), + authenticator=authenticator, + ratelimiter=ratelimiter, + server_name=hs.hostname, + ).register(resource) + if "openid" in servlet_groups: for servletclass in OPENID_SERVLET_CLASSES: servletclass( diff --git a/synapse/handlers/space_summary.py b/synapse/handlers/space_summary.py index 513dc0c71a..f5ead9447f 100644 --- a/synapse/handlers/space_summary.py +++ b/synapse/handlers/space_summary.py @@ -16,7 +16,9 @@ import itertools import logging from collections import deque -from typing import TYPE_CHECKING, Iterable, List, Optional, Set +from typing import TYPE_CHECKING, Iterable, List, Optional, Sequence, Set, Tuple + +import attr from synapse.api.constants import EventContentFields, EventTypes, HistoryVisibility from synapse.api.errors import AuthError @@ -54,7 +56,7 @@ class SpaceSummaryHandler: max_rooms_per_space: Optional[int] = None, ) -> JsonDict: """ - Implementation of the space summary API + Implementation of the space summary C-S API Args: requester: user id of the user making this request @@ -66,7 +68,7 @@ class SpaceSummaryHandler: max_rooms_per_space: an optional limit on the number of child rooms we will return. This does not apply to the root room (ie, room_id), and - is overridden by ROOMS_PER_SPACE_LIMIT. + is overridden by MAX_ROOMS_PER_SPACE. Returns: summary dict to return @@ -76,67 +78,153 @@ class SpaceSummaryHandler: await self._auth.check_user_in_room_or_world_readable(room_id, requester) # the queue of rooms to process - room_queue = deque((room_id,)) + room_queue = deque((_RoomQueueEntry(room_id),)) processed_rooms = set() # type: Set[str] rooms_result = [] # type: List[JsonDict] events_result = [] # type: List[JsonDict] - now = self._clock.time_msec() + while room_queue and len(rooms_result) < MAX_ROOMS: + queue_entry = room_queue.popleft() + room_id = queue_entry.room_id + logger.debug("Processing room %s", room_id) + processed_rooms.add(room_id) + + # The client-specified max_rooms_per_space limit doesn't apply to the + # room_id specified in the request, so we ignore it if this is the + # first room we are processing. + max_children = max_rooms_per_space if processed_rooms else None + + rooms, events = await self._summarize_local_room( + requester, room_id, suggested_only, max_children + ) + + rooms_result.extend(rooms) + events_result.extend(events) + + # add any children that we haven't already processed to the queue + for edge_event in events: + if edge_event["state_key"] not in processed_rooms: + room_queue.append(_RoomQueueEntry(edge_event["state_key"])) + + return {"rooms": rooms_result, "events": events_result} + + async def federation_space_summary( + self, + room_id: str, + suggested_only: bool, + max_rooms_per_space: Optional[int], + exclude_rooms: Iterable[str], + ) -> JsonDict: + """ + Implementation of the space summary Federation API + + Args: + room_id: room id to start the summary at + + suggested_only: whether we should only return children with the "suggested" + flag set. + + max_rooms_per_space: an optional limit on the number of child rooms we will + return. Unlike the C-S API, this applies to the root room (room_id). + It is clipped to MAX_ROOMS_PER_SPACE. + + exclude_rooms: a list of rooms to skip over (presumably because the + calling server has already seen them). + + Returns: + summary dict to return + """ + # the queue of rooms to process + room_queue = deque((room_id,)) + + # the set of rooms that we should not walk further. Initialise it with the + # excluded-rooms list; we will add other rooms as we process them so that + # we do not loop. + processed_rooms = set(exclude_rooms) # type: Set[str] + + rooms_result = [] # type: List[JsonDict] + events_result = [] # type: List[JsonDict] while room_queue and len(rooms_result) < MAX_ROOMS: room_id = room_queue.popleft() logger.debug("Processing room %s", room_id) processed_rooms.add(room_id) - try: - await self._auth.check_user_in_room_or_world_readable( - room_id, requester - ) - except AuthError: - logger.info( - "user %s cannot view room %s, omitting from summary", - requester, - room_id, - ) - continue + rooms, events = await self._summarize_local_room( + None, room_id, suggested_only, max_rooms_per_space + ) - room_entry = await self._build_room_entry(room_id) - rooms_result.append(room_entry) + rooms_result.extend(rooms) + events_result.extend(events) - # look for child rooms/spaces. - child_events = await self._get_child_events(room_id) + # add any children that we haven't already processed to the queue + for edge_event in events: + if edge_event["state_key"] not in processed_rooms: + room_queue.append(edge_event["state_key"]) - if suggested_only: - # we only care about suggested children - child_events = filter(_is_suggested_child_event, child_events) + return {"rooms": rooms_result, "events": events_result} - # The client-specified max_rooms_per_space limit doesn't apply to the - # room_id specified in the request, so we ignore it if this is the - # first room we are processing. Otherwise, apply any client-specified - # limit, capping to our built-in limit. - if max_rooms_per_space is not None and len(processed_rooms) > 1: - max_rooms = min(MAX_ROOMS_PER_SPACE, max_rooms_per_space) - else: - max_rooms = MAX_ROOMS_PER_SPACE - - for edge_event in itertools.islice(child_events, max_rooms): - edge_room_id = edge_event.state_key - - events_result.append( - await self._event_serializer.serialize_event( - edge_event, - time_now=now, - event_format=format_event_for_client_v2, - ) + async def _summarize_local_room( + self, + requester: Optional[str], + room_id: str, + suggested_only: bool, + max_children: Optional[int], + ) -> Tuple[Sequence[JsonDict], Sequence[JsonDict]]: + if not await self._is_room_accessible(room_id, requester): + return (), () + + room_entry = await self._build_room_entry(room_id) + + # look for child rooms/spaces. + child_events = await self._get_child_events(room_id) + + if suggested_only: + # we only care about suggested children + child_events = filter(_is_suggested_child_event, child_events) + + if max_children is None or max_children > MAX_ROOMS_PER_SPACE: + max_children = MAX_ROOMS_PER_SPACE + + now = self._clock.time_msec() + events_result = [] # type: List[JsonDict] + for edge_event in itertools.islice(child_events, max_children): + events_result.append( + await self._event_serializer.serialize_event( + edge_event, + time_now=now, + event_format=format_event_for_client_v2, ) + ) + return (room_entry,), events_result - # if we haven't yet visited the target of this link, add it to the queue - if edge_room_id not in processed_rooms: - room_queue.append(edge_room_id) + async def _is_room_accessible(self, room_id: str, requester: Optional[str]) -> bool: + # if we have an authenticated requesting user, first check if they are in the + # room + if requester: + try: + await self._auth.check_user_in_room(room_id, requester) + return True + except AuthError: + pass - return {"rooms": rooms_result, "events": events_result} + # otherwise, check if the room is peekable + hist_vis_ev = await self._state_handler.get_current_state( + room_id, EventTypes.RoomHistoryVisibility, "" + ) + if hist_vis_ev: + hist_vis = hist_vis_ev.content.get("history_visibility") + if hist_vis == HistoryVisibility.WORLD_READABLE: + return True + + logger.info( + "room %s is unpeekable and user %s is not a member, omitting from summary", + room_id, + requester, + ) + return False async def _build_room_entry(self, room_id: str) -> JsonDict: """Generate en entry suitable for the 'rooms' list in the summary response""" @@ -191,6 +279,11 @@ class SpaceSummaryHandler: return (e for e in events if e.content.get("via")) +@attr.s(frozen=True, slots=True) +class _RoomQueueEntry: + room_id = attr.ib(type=str) + + def _is_suggested_child_event(edge_event: EventBase) -> bool: suggested = edge_event.content.get("suggested") if isinstance(suggested, bool) and suggested: -- cgit 1.5.1