From 079e995ec6e8e1840b4dc0b7d166dcdaf24024d0 Mon Sep 17 00:00:00 2001 From: Andrew Morgan Date: Mon, 5 Oct 2020 19:27:54 +0100 Subject: [PATCH 001/112] Add new experimental room version for knocking --- synapse/api/room_versions.py | 25 +++++++++++++++++++++++-- 1 file changed, 23 insertions(+), 2 deletions(-) diff --git a/synapse/api/room_versions.py b/synapse/api/room_versions.py index f3ecbf36b673..ad26c7a6945f 100644 --- a/synapse/api/room_versions.py +++ b/synapse/api/room_versions.py @@ -57,16 +57,19 @@ class RoomVersion: state_res = attr.ib() # int; one of the StateResolutionVersions enforce_key_validity = attr.ib() # bool - # bool: before MSC2261/MSC2432, m.room.aliases had special auth rules and redaction rules + # Before MSC2432, m.room.aliases had special auth rules and redaction rules special_case_aliases_auth = attr.ib(type=bool) # Strictly enforce canonicaljson, do not allow: # * Integers outside the range of [-2 ^ 53 + 1, 2 ^ 53 - 1] # * Floats # * NaN, Infinity, -Infinity strict_canonicaljson = attr.ib(type=bool) - # bool: MSC2209: Check 'notifications' key while verifying + # MSC2209: Check 'notifications' key while verifying # m.room.power_levels auth rules. limit_notifications_power_levels = attr.ib(type=bool) + # MSC2403: Allows join_rules to be set to 'knock', changes auth rules to allow sending + # m.room.membership event with membership 'knock'. + allow_knocking = attr.ib(type=bool) class RoomVersions: @@ -79,6 +82,7 @@ class RoomVersions: special_case_aliases_auth=True, strict_canonicaljson=False, limit_notifications_power_levels=False, + allow_knocking=False, ) V2 = RoomVersion( "2", @@ -89,6 +93,7 @@ class RoomVersions: special_case_aliases_auth=True, strict_canonicaljson=False, limit_notifications_power_levels=False, + allow_knocking=False, ) V3 = RoomVersion( "3", @@ -99,6 +104,7 @@ class RoomVersions: special_case_aliases_auth=True, strict_canonicaljson=False, limit_notifications_power_levels=False, + allow_knocking=False, ) V4 = RoomVersion( "4", @@ -109,6 +115,7 @@ class RoomVersions: special_case_aliases_auth=True, strict_canonicaljson=False, limit_notifications_power_levels=False, + allow_knocking=False, ) V5 = RoomVersion( "5", @@ -119,6 +126,7 @@ class RoomVersions: special_case_aliases_auth=True, strict_canonicaljson=False, limit_notifications_power_levels=False, + allow_knocking=False, ) V6 = RoomVersion( "6", @@ -129,6 +137,18 @@ class RoomVersions: special_case_aliases_auth=False, strict_canonicaljson=True, limit_notifications_power_levels=True, + allow_knocking=False, + ) + MSC2403_DEV = RoomVersion( + "xyz.amorgan.knock", + RoomDisposition.UNSTABLE, + EventFormatVersions.V3, + StateResolutionVersions.V2, + enforce_key_validity=True, + special_case_aliases_auth=False, + strict_canonicaljson=True, + limit_notifications_power_levels=True, + allow_knocking=True, ) @@ -141,5 +161,6 @@ class RoomVersions: RoomVersions.V4, RoomVersions.V5, RoomVersions.V6, + RoomVersions.MSC2403_DEV, ) } # type: Dict[str, RoomVersion] From f1f5fa7af6974ff6f9b8fd1996cb8507e777ca15 Mon Sep 17 00:00:00 2001 From: Andrew Morgan Date: Wed, 7 Oct 2020 16:49:01 +0100 Subject: [PATCH 002/112] Add `xyz.amorgan.knock` /versions string --- synapse/rest/client/versions.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/synapse/rest/client/versions.py b/synapse/rest/client/versions.py index d24a199318a2..7a5c739b2370 100644 --- a/synapse/rest/client/versions.py +++ b/synapse/rest/client/versions.py @@ -81,6 +81,8 @@ def on_GET(self, request): "io.element.e2ee_forced.public": self.e2ee_forced_public, "io.element.e2ee_forced.private": self.e2ee_forced_private, "io.element.e2ee_forced.trusted_private": self.e2ee_forced_trusted_private, + # Implements additional endpoints and features as described in MSC2403 + "xyz.amorgan.knock": True, }, }, ) From b81617e0df1575de83ae2b6e21ccc9e9d62b3e63 Mon Sep 17 00:00:00 2001 From: Andrew Morgan Date: Tue, 10 Nov 2020 19:35:56 +0000 Subject: [PATCH 003/112] Update the event auth rules for knocking Hopefully most of these changes are explained through the added comments and error messages. The changes are also described conceptually in the MSC: https://github.com/Sorunome/matrix-doc/blob/soru/knock/proposals/2403-knock.md#join-rules --- synapse/event_auth.py | 22 ++++++++++++++++++---- 1 file changed, 18 insertions(+), 4 deletions(-) diff --git a/synapse/event_auth.py b/synapse/event_auth.py index 56f8dc9caf9e..b5848df2514b 100644 --- a/synapse/event_auth.py +++ b/synapse/event_auth.py @@ -161,6 +161,7 @@ def check( if logger.isEnabledFor(logging.DEBUG): logger.debug("Auth events: %s", [a.event_id for a in auth_events.values()]) + # 5. If type is m.room.membership if event.type == EventTypes.Member: _is_membership_change_allowed(event, auth_events) logger.debug("Allowing! %s", event) @@ -247,6 +248,7 @@ def _is_membership_change_allowed( caller_in_room = caller and caller.membership == Membership.JOIN caller_invited = caller and caller.membership == Membership.INVITE + caller_knocked = caller and caller.membership == Membership.KNOCK # get info about the target key = (EventTypes.Member, target_user_id) @@ -289,9 +291,12 @@ def _is_membership_change_allowed( raise AuthError(403, "%s is banned from the room" % (target_user_id,)) return - if Membership.JOIN != membership: + # Require the user to be in the room for membership changes other than join/knocking + if Membership.JOIN != membership and Membership.KNOCK != membership: + # If the user has been invited or has knocked, they are allowed to change their + # membership event to leave if ( - caller_invited + (caller_invited or caller_knocked) and Membership.LEAVE == membership and target_user_id == event.user_id ): @@ -324,7 +329,7 @@ def _is_membership_change_allowed( raise AuthError(403, "You are banned from this room") elif join_rule == JoinRules.PUBLIC: pass - elif join_rule == JoinRules.INVITE: + elif join_rule in [JoinRules.INVITE, JoinRules.KNOCK]: if not caller_in_room and not caller_invited: raise AuthError(403, "You are not invited to this room.") else: @@ -343,6 +348,15 @@ def _is_membership_change_allowed( elif Membership.BAN == membership: if user_level < ban_level or user_level <= target_level: raise AuthError(403, "You don't have permission to ban") + elif Membership.KNOCK == membership: + if join_rule != JoinRules.KNOCK: + raise AuthError(403, "You don't have permission to knock") + elif target_user_id != event.user_id: + raise AuthError(403, "You cannot knock for other users") + elif target_in_room: + raise AuthError(403, "You cannot knock on a room you are already in") + elif target_banned: + raise AuthError(403, "You are banned from this room") else: raise AuthError(500, "Unknown membership %s" % membership) @@ -699,7 +713,7 @@ def auth_types_for_event(event: EventBase) -> Set[Tuple[str, str]]: if event.type == EventTypes.Member: membership = event.content["membership"] - if membership in [Membership.JOIN, Membership.INVITE]: + if membership in [Membership.JOIN, Membership.INVITE, Membership.KNOCK]: auth_types.add((EventTypes.JoinRules, "")) auth_types.add((EventTypes.Member, event.state_key)) From 25a341d28349e521473f875e28e673916d7128b4 Mon Sep 17 00:00:00 2001 From: Andrew Morgan Date: Tue, 10 Nov 2020 19:49:58 +0000 Subject: [PATCH 004/112] Add CS /_matrix/client/r0/knock/{roomIdOrAlias} endpoint We're ditching the usual idea of having two endpoints for each membership-related endpoint as per the MSC. Thus knocking only gets the more powerful variant (the one that supports room aliases as well as IDs. The reason is also optional. The other small change is just to ensure displaynames get added to the content of this particular membership event. --- synapse/handlers/message.py | 2 +- synapse/rest/__init__.py | 2 + synapse/rest/client/v2_alpha/knock.py | 103 ++++++++++++++++++++++++++ 3 files changed, 106 insertions(+), 1 deletion(-) create mode 100644 synapse/rest/client/v2_alpha/knock.py diff --git a/synapse/handlers/message.py b/synapse/handlers/message.py index c6791fb91258..94bdc15dcf36 100644 --- a/synapse/handlers/message.py +++ b/synapse/handlers/message.py @@ -492,7 +492,7 @@ async def create_event( membership = builder.content.get("membership", None) target = UserID.from_string(builder.state_key) - if membership in {Membership.JOIN, Membership.INVITE}: + if membership in {Membership.JOIN, Membership.INVITE, Membership.KNOCK}: # If event doesn't include a display name, add one. profile = self.profile_handler content = builder.content diff --git a/synapse/rest/__init__.py b/synapse/rest/__init__.py index 40f5c32db2df..c44a38af5e9a 100644 --- a/synapse/rest/__init__.py +++ b/synapse/rest/__init__.py @@ -39,6 +39,7 @@ filter, groups, keys, + knock, notifications, openid, password_policy, @@ -121,6 +122,7 @@ def register_servlets(client_resource, hs): account_validity.register_servlets(hs, client_resource) relations.register_servlets(hs, client_resource) password_policy.register_servlets(hs, client_resource) + knock.register_servlets(hs, client_resource) # moving to /_synapse/admin admin.register_servlets_for_client_rest_resource(hs, client_resource) diff --git a/synapse/rest/client/v2_alpha/knock.py b/synapse/rest/client/v2_alpha/knock.py new file mode 100644 index 000000000000..1814683120c2 --- /dev/null +++ b/synapse/rest/client/v2_alpha/knock.py @@ -0,0 +1,103 @@ +# -*- coding: utf-8 -*- +# Copyright 2020 Sorunome +# Copyright 2020 The Matrix.org Foundation C.I.C. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import logging +from typing import TYPE_CHECKING, List, Optional, Tuple + +from twisted.web.server import Request + +from synapse.api.errors import SynapseError +from synapse.http.servlet import RestServlet, parse_json_object_from_request +from synapse.logging.opentracing import set_tag +from synapse.rest.client.transactions import HttpTransactionCache +from synapse.types import JsonDict, RoomAlias, RoomID + +if TYPE_CHECKING: + from synapse.app.homeserver import HomeServer + +from ._base import client_patterns + +logger = logging.getLogger(__name__) + + +class TransactionRestServlet(RestServlet): + def __init__(self, hs: "HomeServer"): + super(TransactionRestServlet, self).__init__() + self.txns = HttpTransactionCache(hs) + + +class KnockRoomAliasServlet(TransactionRestServlet): + """ + POST /knock/{roomIdOrAlias} + """ + + PATTERNS = client_patterns("/knock/(?P[^/]*)") + + def __init__(self, hs: "HomeServer"): + super().__init__(hs) + self.room_member_handler = hs.get_room_member_handler() + self.auth = hs.get_auth() + + async def on_POST( + self, request: Request, room_identifier: str, txn_id: Optional[str] = None, + ) -> Tuple[int, JsonDict]: + requester = await self.auth.get_user_by_req(request) + + content = parse_json_object_from_request(request) + event_content = None + if "reason" in content: + event_content = {"reason": content["reason"]} + + if RoomID.is_valid(room_identifier): + room_id = room_identifier + try: + remote_room_hosts = [ + x.decode("ascii") for x in request.args[b"server_name"] + ] # type: Optional[List[str]] + except Exception: + remote_room_hosts = None + elif RoomAlias.is_valid(room_identifier): + handler = self.room_member_handler + room_alias = RoomAlias.from_string(room_identifier) + room_id_obj, remote_room_hosts = await handler.lookup_room_alias(room_alias) + room_id = room_id_obj.to_string() + else: + raise SynapseError( + 400, "%s was not legal room ID or room alias" % (room_identifier,) + ) + + await self.room_member_handler.update_membership( + requester=requester, + target=requester.user, + room_id=room_id, + action="knock", + txn_id=txn_id, + third_party_signed=None, + remote_room_hosts=remote_room_hosts, + content=event_content, + ) + + return 200, {"room_id": room_id} + + def on_PUT(self, request: Request, room_identifier: str, txn_id: str): + set_tag("txn_id", txn_id) + + return self.txns.fetch_or_execute_request( + request, self.on_POST, request, room_identifier, txn_id + ) + + +def register_servlets(hs, http_server): + KnockRoomAliasServlet(hs).register(http_server) From 66e263bf5453143b429815117db5beda1c8170a3 Mon Sep 17 00:00:00 2001 From: Andrew Morgan Date: Mon, 2 Nov 2020 16:06:38 +0000 Subject: [PATCH 005/112] Add room_knock_state_types config option This option serves the same purpose as the existing room_invite_state_types option, which defines what state events are sent over to a user that is invited to a room. This information is necessary for the user - who isn't in the room yet - to get some metadata about the room in order to display it in a pretty fashion in the user's pending-invites list. It includes information such as the room's name, avatar, topic, canonical alias, room encryption state etc. as well as the invite membership event which the invited user's homeserver can reference. This new option is the exact same, but is sent by a homeserver in the room to the knocker during the knock process. This option will actually be utilised in a later commit. --- docs/sample_config.yaml | 11 +++++++++++ synapse/config/api.py | 31 +++++++++++++++++++++++-------- 2 files changed, 34 insertions(+), 8 deletions(-) diff --git a/docs/sample_config.yaml b/docs/sample_config.yaml index e9e77ca94e98..513c77f86c8c 100644 --- a/docs/sample_config.yaml +++ b/docs/sample_config.yaml @@ -1396,6 +1396,17 @@ metrics_flags: # - "m.room.encryption" # - "m.room.name" +# A list of event types from a room that will be given to users when they +# knock on the room. This allows clients to display information about the +# room that they've knocked on, without actually being in the room yet. +# +#room_knock_state_types: +# - "m.room.join_rules" +# - "m.room.canonical_alias" +# - "m.room.avatar" +# - "m.room.encryption" +# - "m.room.name" + # A list of application service config files to use # diff --git a/synapse/config/api.py b/synapse/config/api.py index 74cd53a8ed34..7604993b637e 100644 --- a/synapse/config/api.py +++ b/synapse/config/api.py @@ -1,4 +1,5 @@ # Copyright 2015, 2016 OpenMarket Ltd +# Copyright 2020 The Matrix.org Foundation C.I.C. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -21,15 +22,18 @@ class ApiConfig(Config): section = "api" def read_config(self, config, **kwargs): + default_room_state_types = [ + EventTypes.JoinRules, + EventTypes.CanonicalAlias, + EventTypes.RoomAvatar, + EventTypes.RoomEncryption, + EventTypes.Name, + ] self.room_invite_state_types = config.get( - "room_invite_state_types", - [ - EventTypes.JoinRules, - EventTypes.CanonicalAlias, - EventTypes.RoomAvatar, - EventTypes.RoomEncryption, - EventTypes.Name, - ], + "room_invite_state_types", default_room_state_types + ) + self.room_knock_state_types = config.get( + "room_knock_state_types", default_room_state_types ) def generate_config_section(cls, **kwargs): @@ -44,6 +48,17 @@ def generate_config_section(cls, **kwargs): # - "{RoomAvatar}" # - "{RoomEncryption}" # - "{Name}" + + # A list of event types from a room that will be given to users when they + # knock on the room. This allows clients to display information about the + # room that they've knocked on, without actually being in the room yet. + # + #room_knock_state_types: + # - "{JoinRules}" + # - "{CanonicalAlias}" + # - "{RoomAvatar}" + # - "{RoomEncryption}" + # - "{Name}" """.format( **vars(EventTypes) ) From 50998c7b3928cc1a028d5e3ffb8cf371ce0a337e Mon Sep 17 00:00:00 2001 From: Andrew Morgan Date: Wed, 11 Nov 2020 17:41:57 +0000 Subject: [PATCH 006/112] Federation: make_knock and send_knock implementations Most of this is explained in the linked MSC (and don't miss the sequence diagram in the MSC comments), but roughly knocking takes inspiration from room joins and room invites. This commit is the room join stuff. First the knocking homeserver POSTs to the make_knock endpoint on another homeserver. The other homeserver will send back a knock event that is valid for the knocking user and the room that they are knocking on. The knocking homeserver will sign the event and send it back, before the other homeserver takes that event and then sends it into the room on the knocking homeserver's behalf. It's worth noting that the accepting/rejecting knocks all happen over existing room invite/leave flows. A homeserver rescinding its knock as well is also just sending a leave. Once the event has been inserted into the room, the homeserver that's in the room will send back a 200 and an empty JSON dict to confirm everything went well to the knocker. In a future commit, this dict will instead be filled with some stripped state events from the room which the knocking homeserver will pass back to the knocking user. And yes, the logging statements in this commit are intentional. They're consistent with the rest of the file :) --- synapse/federation/federation_client.py | 44 +++++- synapse/federation/federation_server.py | 28 ++++ synapse/federation/transport/client.py | 30 ++++- synapse/federation/transport/server.py | 21 ++- synapse/handlers/federation.py | 170 +++++++++++++++++++++++- synapse/handlers/room_member.py | 57 ++++++++ synapse/handlers/room_member_worker.py | 14 ++ 7 files changed, 360 insertions(+), 4 deletions(-) diff --git a/synapse/federation/federation_client.py b/synapse/federation/federation_client.py index 302b2f69bcdd..ffd1781252bd 100644 --- a/synapse/federation/federation_client.py +++ b/synapse/federation/federation_client.py @@ -1,5 +1,7 @@ # -*- coding: utf-8 -*- # Copyright 2015, 2016 OpenMarket Ltd +# Copyrignt 2020 Sorunome +# Copyrignt 2020 The Matrix.org Foundation C.I.C. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -539,7 +541,7 @@ async def make_membership_event( RuntimeError: if no servers were reachable. """ - valid_memberships = {Membership.JOIN, Membership.LEAVE} + valid_memberships = {Membership.JOIN, Membership.LEAVE, Membership.KNOCK} if membership not in valid_memberships: raise RuntimeError( "make_membership_event called with membership='%s', must be one of %s" @@ -879,6 +881,46 @@ async def _do_send_leave(self, destination, pdu): # content. return resp[1] + async def send_knock(self, destinations: List[str], pdu: EventBase): + """Attempts to send a knock event to given a list of servers. Iterates + through the list until one attempt succeeds. + + Doing so will cause the remote server to add the event to the graph, + and send the event out to the rest of the federation. + + Args: + destinations: A list of candidate homeservers which are likely to be + participating in the room. + pdu: The event to be sent. + + Raises: + SynapseError: If the chosen remote server returns a 3xx/4xx code. + RuntimeError: If no servers were reachable. + """ + + async def send_request(destination: str) -> JsonDict: + return await self._do_send_knock(destination, pdu) + + return await self._try_destination_list( + "send_knock", destinations, send_request + ) + + async def _do_send_knock(self, destination: str, pdu: EventBase): + """Send a knock event to a remote homeserver. + + Args: + destination: The homeserver to send to. + pdu: The event to send. + """ + time_now = self._clock.time_msec() + + return await self.transport_layer.send_knock_v1( + destination=destination, + room_id=pdu.room_id, + event_id=pdu.event_id, + content=pdu.get_pdu_json(time_now), + ) + def get_public_rooms( self, remote_server: str, diff --git a/synapse/federation/federation_server.py b/synapse/federation/federation_server.py index 23278e36b733..50c8a64aa86b 100644 --- a/synapse/federation/federation_server.py +++ b/synapse/federation/federation_server.py @@ -565,6 +565,34 @@ async def on_send_leave_request( await self.handler.on_send_leave_request(origin, pdu) return {} + async def on_make_knock_request(self, origin: str, room_id: str, user_id: str): + origin_host, _ = parse_server_name(origin) + await self.check_server_matches_acl(origin_host, room_id) + pdu = await self.handler.on_make_knock_request(origin, room_id, user_id) + + room_version = await self.store.get_room_version_id(room_id) + + time_now = self._clock.time_msec() + return {"event": pdu.get_pdu_json(time_now), "room_version": room_version} + + async def on_send_knock_request(self, origin: str, content: JsonDict, room_id: str): + logger.debug("on_send_knock_request: content: %s", content) + + room_version = await self.store.get_room_version(room_id) + pdu = event_from_pdu_json(content, room_version) + + origin_host, _ = parse_server_name(origin) + await self.check_server_matches_acl(origin_host, pdu.room_id) + + logger.debug("on_send_knock_request: pdu sigs: %s", pdu.signatures) + + pdu = await self._check_sigs_and_hash(room_version, pdu) + + # Handle the event + await self.handler.on_send_knock_request(origin, pdu) + + return {} + async def on_event_auth( self, origin: str, room_id: str, event_id: str ) -> Tuple[int, Dict[str, Any]]: diff --git a/synapse/federation/transport/client.py b/synapse/federation/transport/client.py index 17a10f622eb9..4650c009f922 100644 --- a/synapse/federation/transport/client.py +++ b/synapse/federation/transport/client.py @@ -1,6 +1,8 @@ # -*- coding: utf-8 -*- # Copyright 2014-2016 OpenMarket Ltd # Copyright 2018 New Vector Ltd +# Copyright 2020 Sorunome +# Copyright 2020 The Matrix.org Foundation C.I.C. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -26,6 +28,7 @@ FEDERATION_V2_PREFIX, ) from synapse.logging.utils import log_function +from synapse.types import JsonDict logger = logging.getLogger(__name__) @@ -209,7 +212,7 @@ async def make_membership_event( Fails with ``FederationDeniedError`` if the remote destination is not in our federation whitelist """ - valid_memberships = {Membership.JOIN, Membership.LEAVE} + valid_memberships = {Membership.JOIN, Membership.LEAVE, Membership.KNOCK} if membership not in valid_memberships: raise RuntimeError( "make_membership_event called with membership='%s', must be one of %s" @@ -293,6 +296,31 @@ async def send_leave_v2(self, destination, room_id, event_id, content): return response + @log_function + async def send_knock_v1( + self, destination: str, room_id: str, event_id: str, content: JsonDict, + ) -> JsonDict: + """ + Sends an signed knock membership event to a remote server. This is the second + step knocking after make_knock. + + Args: + destination: The remote homeserver. + room_id: The ID of the room to knock on. + event_id: The ID of the knock membership event that we're sending. + content: The knock membership event that we're sending. Note that this is not the + `content` field of the membership event, but the entire signed membership event + itself represented as a JSON dict. + + Returns: + An empty JSON dictionary. + """ + path = _create_v1_path("/send_knock/%s/%s", room_id, event_id) + + return await self.client.put_json( + destination=destination, path=path, data=content + ) + @log_function async def send_invite_v1(self, destination, room_id, event_id, content): path = _create_v1_path("/invite/%s/%s", room_id, event_id) diff --git a/synapse/federation/transport/server.py b/synapse/federation/transport/server.py index a0933fae88c8..133de2f3a101 100644 --- a/synapse/federation/transport/server.py +++ b/synapse/federation/transport/server.py @@ -1,7 +1,8 @@ # -*- coding: utf-8 -*- # Copyright 2014-2016 OpenMarket Ltd # Copyright 2018 New Vector Ltd -# Copyright 2019 The Matrix.org Foundation C.I.C. +# Copyright 2019-2020 The Matrix.org Foundation C.I.C. +# Copyright 2020 Sorunome # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -542,6 +543,22 @@ async def on_PUT(self, origin, content, query, room_id, event_id): return 200, content +class FederationMakeKnockServlet(BaseFederationServlet): + PATH = "/make_knock/(?P[^/]*)/(?P[^/]*)" + + async def on_GET(self, origin, content, query, context, user_id): + content = await self.handler.on_make_knock_request(origin, context, user_id) + return 200, content + + +class FederationV1MakeKnockServlet(BaseFederationServlet): + PATH = "/send_knock/(?P[^/]*)/(?P[^/]*)" + + async def on_PUT(self, origin, content, query, room_id, event_id): + content = await self.handler.on_send_knock_request(origin, content, room_id) + return 200, content + + class FederationEventAuthServlet(BaseFederationServlet): PATH = "/event_auth/(?P[^/]*)/(?P[^/]*)" @@ -1389,11 +1406,13 @@ async def on_GET(self, origin, content, query, room_id): FederationQueryServlet, FederationMakeJoinServlet, FederationMakeLeaveServlet, + FederationMakeKnockServlet, FederationEventServlet, FederationV1SendJoinServlet, FederationV2SendJoinServlet, FederationV1SendLeaveServlet, FederationV2SendLeaveServlet, + FederationV1MakeKnockServlet, FederationV1InviteServlet, FederationV2InviteServlet, FederationGetMissingEventsServlet, diff --git a/synapse/handlers/federation.py b/synapse/handlers/federation.py index 69bc5ba44db2..ec0514cc678d 100644 --- a/synapse/handlers/federation.py +++ b/synapse/handlers/federation.py @@ -1,7 +1,8 @@ # -*- coding: utf-8 -*- # Copyright 2014-2016 OpenMarket Ltd # Copyright 2017-2018 New Vector Ltd -# Copyright 2019 The Matrix.org Foundation C.I.C. +# Copyright 2019-2020 The Matrix.org Foundation C.I.C. +# Copyright 2020 Sorunome # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -1438,6 +1439,62 @@ async def do_invite_join( run_in_background(self._handle_queued_pdus, room_queue) + @log_function + async def do_knock( + self, target_hosts: List[str], room_id: str, knockee: str, content: JsonDict, + ) -> Tuple[str, int]: + """Sends the knock to the remote server. + + This first triggers a /make_knock request that returns a partial + event that we can fill out and sign. This is then sent to the + remote server via /send_knock. + + Knock events must be signed by the knockee's server before distributing. + + Args: + target_hosts: A list of hosts that we want to try knocking through. + room_id: The ID of the room to knock on. + knockee: The ID of the user who is knocking. + content: The content of the knock event. + + Returns: + A tuple of (event ID, stream ID). + + Raises: + SynapseError: If the chosen remote server returns a 3xx/4xx code. + RuntimeError: If no servers were reachable. + """ + logger.debug("Knocking on room %s on behalf of user %s", room_id, knockee) + + # Ask the remote server to create a valid knock event for us. Once received, + # we sign the event + origin, event, event_format_version = await self._make_and_verify_event( + target_hosts, room_id, knockee, "knock", content, + ) + + # Record the room ID and its version so that we have a record of the room + # TODO: Rename this function as its scope has expanded + await self._maybe_store_room_on_outlier_membership( + room_id=event.room_id, room_version=event_format_version + ) + + # Initially try the host that we successfully called /make_knock on + try: + target_hosts.remove(origin) + target_hosts.insert(0, origin) + except ValueError: + pass + + # Send the signed event back to the room + await self.federation_client.send_knock(target_hosts, event) + + # Store the event locally + context = await self.state_handler.compute_event_context(event) + stream_id = await self.persist_events_and_notify( + event.room_id, [(event, context)] + ) + return event.event_id, stream_id + async def _handle_queued_pdus(self, room_queue): """Process PDUs which got queued up while we were busy send_joining. @@ -1774,6 +1831,117 @@ async def on_send_leave_request(self, origin, pdu): return None + @log_function + async def on_make_knock_request( + self, origin: str, room_id: str, user_id: str + ) -> EventBase: + """We've received a /make_knock/ request, so we create a partial + knock event for the room and return that. We do *not* persist or + process it until the other server has signed it and sent it back. + + Args: + origin: The (verified) server name of the requesting server. + room_id: The room to create the knock event in. + user_id: The user to create the knock for. + + Returns: + The partial knock event. + """ + if get_domain_from_id(user_id) != origin: + logger.info( + "Get /make_knock request for user %r from different origin %s, ignoring", + user_id, + origin, + ) + raise SynapseError(403, "User not from origin", Codes.FORBIDDEN) + + room_version = await self.store.get_room_version_id(room_id) + builder = self.event_builder_factory.new( + room_version, + { + "type": EventTypes.Member, + "content": {"membership": Membership.KNOCK}, + "room_id": room_id, + "sender": user_id, + "state_key": user_id, + }, + ) + + event, context = await self.event_creation_handler.create_new_client_event( + builder=builder + ) + + event_allowed = await self.third_party_event_rules.check_event_allowed( + event, context + ) + if not event_allowed: + logger.warning("Creation of knock %s forbidden by third-party rules", event) + raise SynapseError( + 403, "This event is not allowed in this context", Codes.FORBIDDEN + ) + + try: + # The remote hasn't signed it yet, obviously. We'll do the full checks + # when we get the event back in `on_send_knock_request` + await self.auth.check_from_context( + room_version, event, context, do_sig_check=False + ) + except AuthError as e: + logger.warning("Failed to create new knock %r because %s", event, e) + raise e + + return event + + @log_function + async def on_send_knock_request(self, origin: str, pdu: EventBase) -> EventContext: + """ + We have received a knock event for a room. Verify that event and send it into the room + on the knocking homeserver's behalf. + + Args: + origin: The remote homeserver of the knocking user. + pdu: The knocking member event that has been signed by the remote homeserver. + + Returns: + The context of the event after inserting it into the room graph. + """ + event = pdu + + logger.debug( + "on_send_knock_request: Got event: %s, signatures: %s", + event.event_id, + event.signatures, + ) + + if get_domain_from_id(event.sender) != origin: + logger.info( + "Got /send_knock request for user %r from different origin %s", + event.sender, + origin, + ) + raise SynapseError(403, "User not from origin", Codes.FORBIDDEN) + + event.internal_metadata.outlier = False + + context = await self._handle_new_event(origin, event) + + event_allowed = await self.third_party_event_rules.check_event_allowed( + event, context + ) + if not event_allowed: + logger.info("Sending of knock %s forbidden by third-party rules", event) + raise SynapseError( + 403, "This event is not allowed in this context", Codes.FORBIDDEN + ) + + logger.debug( + "on_send_knock_request: After _handle_new_event: %s, sigs: %s", + event.event_id, + event.signatures, + ) + + return context + async def get_state_for_pdu(self, room_id: str, event_id: str) -> List[EventBase]: """Returns the state at the event. i.e. not including said event. """ diff --git a/synapse/handlers/room_member.py b/synapse/handlers/room_member.py index fd85e08973b4..a8518ac3b616 100644 --- a/synapse/handlers/room_member.py +++ b/synapse/handlers/room_member.py @@ -110,6 +110,20 @@ async def _remote_join( """ raise NotImplementedError() + @abc.abstractmethod + async def _remote_knock( + self, remote_room_hosts: List[str], room_id: str, user: UserID, content: dict, + ) -> Tuple[str, int]: + """Try and knock on a room that this server is not in + + Args: + remote_room_hosts: List of servers that can be used to knock via. + room_id: Room that we are trying to knock on. + user: User who is trying to knock. + content: A dict that should be used as the content of the knock event. + """ + raise NotImplementedError() + @abc.abstractmethod async def remote_reject_invite( self, @@ -552,6 +566,23 @@ async def update_membership_locked( if len(latest_event_ids) == 0: latest_event_ids = [invite.event_id] + elif effective_membership_state == Membership.KNOCK: + if not is_host_in_room: + # The knock needs to be sent over federation instead + remote_room_hosts.append(room_id.split(":", 1)[1]) + + content["membership"] = Membership.KNOCK + + profile = self.profile_handler + if "displayname" not in content: + content["displayname"] = await profile.get_displayname(target) + if "avatar_url" not in content: + content["avatar_url"] = await profile.get_avatar_url(target) + + return await self._remote_knock( + remote_room_hosts, room_id, target, content + ) + return await self._local_membership_update( requester=requester, target=target, @@ -1168,6 +1199,32 @@ async def _generate_local_out_of_band_leave( return result_event.event_id, result_event.internal_metadata.stream_ordering + async def _remote_knock( + self, remote_room_hosts: List[str], room_id: str, user: UserID, content: dict, + ) -> Tuple[str, int]: + """Sends a knock to a room. Attempts to do so via one remote out of a given list. + + Args: + remote_room_hosts: A list of homeservers to try knocking through. + room_id: The ID of the room to knock on. + user: The user to knock on behalf of. + content: The content of the knock event. + + Returns: + A tuple of (event ID, stream ID). + """ + # filter ourselves out of remote_room_hosts + remote_room_hosts = [ + host for host in remote_room_hosts if host != self.hs.hostname + ] + + if len(remote_room_hosts) == 0: + raise SynapseError(404, "No known servers") + + return await self.federation_handler.do_knock( + remote_room_hosts, room_id, user.to_string(), content=content + ) + async def _user_left_room(self, target: UserID, room_id: str) -> None: """Implements RoomMemberHandler._user_left_room """ diff --git a/synapse/handlers/room_member_worker.py b/synapse/handlers/room_member_worker.py index f2e88f6a5b5d..f9cea604ec14 100644 --- a/synapse/handlers/room_member_worker.py +++ b/synapse/handlers/room_member_worker.py @@ -79,6 +79,20 @@ async def remote_reject_invite( ) return ret["event_id"], ret["stream_id"] + async def _remote_knock( + self, remote_room_hosts: List[str], room_id: str, user: UserID, content: dict, + ) -> Tuple[str, int]: + """Sends a knock to a room. + + Implements RoomMemberHandler._remote_knock + """ + return await self._remote_knock( + remote_room_hosts=remote_room_hosts, + room_id=room_id, + user=user, + content=content, + ) + async def _user_left_room(self, target: UserID, room_id: str) -> None: """Implements RoomMemberHandler._user_left_room """ From eea7db98e2b9f0275274f6bee4a12876ba0fb3d2 Mon Sep 17 00:00:00 2001 From: Andrew Morgan Date: Wed, 11 Nov 2020 18:10:40 +0000 Subject: [PATCH 007/112] Send stripped state events back to the knocking homeserver Here we finally send the stripped state events back to the knocking homeserver, which then ingests and stores those events. A future commit will actually start sending those events down /sync to the relevant user. --- synapse/federation/federation_client.py | 20 ++++++++++++++++++-- synapse/federation/federation_server.py | 14 +++++++++++--- synapse/federation/transport/client.py | 7 ++++++- synapse/handlers/federation.py | 14 +++++++++++--- 4 files changed, 46 insertions(+), 9 deletions(-) diff --git a/synapse/federation/federation_client.py b/synapse/federation/federation_client.py index ffd1781252bd..b32b548500b8 100644 --- a/synapse/federation/federation_client.py +++ b/synapse/federation/federation_client.py @@ -881,7 +881,7 @@ async def _do_send_leave(self, destination, pdu): # content. return resp[1] - async def send_knock(self, destinations: List[str], pdu: EventBase): + async def send_knock(self, destinations: List[str], pdu: EventBase) -> JsonDict: """Attempts to send a knock event to given a list of servers. Iterates through the list until one attempt succeeds. @@ -893,6 +893,14 @@ async def send_knock(self, destinations: List[str], pdu: EventBase): participating in the room. pdu: The event to be sent. + Returns: + The remote homeserver return some state from the room. The response + dictionary is in the form: + + {"knock_state_events": [, ...]} + + The list of state events may be empty. + Raises: SynapseError: If the chosen remote server returns a 3xx/4xx code. RuntimeError: If no servers were reachable. @@ -905,12 +913,20 @@ async def send_request(destination: str) -> JsonDict: "send_knock", destinations, send_request ) - async def _do_send_knock(self, destination: str, pdu: EventBase): + async def _do_send_knock(self, destination: str, pdu: EventBase) -> JsonDict: """Send a knock event to a remote homeserver. Args: destination: The homeserver to send to. pdu: The event to send. + + Returns: + The remote homeserver can optionally return some state from the room. The response + dictionary is in the form: + + {"knock_state_events": [, ...]} + + The list of state events may be empty. """ time_now = self._clock.time_msec() diff --git a/synapse/federation/federation_server.py b/synapse/federation/federation_server.py index 50c8a64aa86b..3417f6291639 100644 --- a/synapse/federation/federation_server.py +++ b/synapse/federation/federation_server.py @@ -118,6 +118,8 @@ def __init__(self, hs): hs, "fed_txn_handler", timeout_ms=30000 ) # type: ResponseCache[Tuple[str, str]] + self._room_knock_state_types = hs.config.room_knock_state_types + self.transaction_actions = TransactionActions(self.store) self.registry = hs.get_federation_registry() @@ -588,10 +590,16 @@ async def on_send_knock_request(self, origin: str, content: JsonDict, room_id: s pdu = await self._check_sigs_and_hash(room_version, pdu) - # Handle the event - await self.handler.on_send_knock_request(origin, pdu) + # Handle the event, and retrieve the EventContext + event_context = await self.handler.on_send_knock_request(origin, pdu) - return {} + # Retrieve stripped state events from the room and send them back to the remote + # server. This will allow the remote server's clients to display information + # related to the room while the knock request is pending. + stripped_room_state = await self.store.get_stripped_room_state_from_event_context( + event_context, self._room_knock_state_types + ) + return {"knock_state_events": stripped_room_state} async def on_event_auth( self, origin: str, room_id: str, event_id: str diff --git a/synapse/federation/transport/client.py b/synapse/federation/transport/client.py index 4650c009f922..ca369af2dec5 100644 --- a/synapse/federation/transport/client.py +++ b/synapse/federation/transport/client.py @@ -313,7 +313,12 @@ async def send_knock_v1( itself represented as a JSON dict. Returns: - An empty JSON dictionary. + The remote homeserver can optionally return some state from the room. The response + dictionary is in the form: + + {"knock_state_events": [, ...]} + + The list of state events may be empty. """ path = _create_v1_path("/send_knock/%s/%s", room_id, event_id) diff --git a/synapse/handlers/federation.py b/synapse/handlers/federation.py index ec0514cc678d..2af17974c4ba 100644 --- a/synapse/handlers/federation.py +++ b/synapse/handlers/federation.py @@ -1485,10 +1485,18 @@ async def do_knock( except ValueError: pass - # Send the signed event back to the room - await self.federation_client.send_knock(target_hosts, event) + # Send the signed event back to the room, and potentially receive some + # further information about the room in the form of partial state events + stripped_room_state = await self.federation_client.send_knock( + target_hosts, event + ) + + # Store any stripped room state events in the "unsigned" key of the event. + # This is a bit of a hack and is cribbing off of invites. Basically we + # store the room state here and retrieve it again when this event appears + # in the invitee's sync stream. It is stripped out for all other local users. + event.unsigned["knock_room_state"] = stripped_room_state["knock_state_events"] - # Store the event locally context = await self.state_handler.compute_event_context(event) stream_id = await self.persist_events_and_notify( event.room_id, [(event, context)] From 41f5490f10b14556d3858016e022728708cb50a5 Mon Sep 17 00:00:00 2001 From: Andrew Morgan Date: Thu, 12 Nov 2020 16:41:03 +0000 Subject: [PATCH 008/112] Extend sync to inform clients about the progress of their knocks So we've got federation so that homeservers can communicate knocking information between them - but how does that information actually get down to the client? The client knows that it knocked successfully from a 200 in its original request, but what else does it need? This commit adds a new "knock" section to /sync (in addition to "invite", "join", and "leave") all help give the client the information it needs. The new "knock" section is used for sending down the stripped state events we collected earlier. The client will use these to display the room and its metadata in a little "pending knocks" section or similar. This is all this commit adds. If the user's knock has been accepted or rejected, they will receive that information in the "join" or "leave" sections of /sync. Most of this code is just cribbing off the invite and join sync code yet again, with some minor differences. For instance, we don't need to exclude knock events from sync if the sender is in your ignore list, as you are the only ones that can send knocks for yourself. The structure of the "knock" dict in sync is modeled after "invite", as clients also receive stripped state in that. The structure can be viewed in the linked MSC. --- synapse/events/utils.py | 5 ++ synapse/handlers/sync.py | 83 +++++++++++++++++++++------- synapse/rest/client/v2_alpha/sync.py | 62 ++++++++++++++++++++- 3 files changed, 127 insertions(+), 23 deletions(-) diff --git a/synapse/events/utils.py b/synapse/events/utils.py index 14f7f1156f38..589fa877d276 100644 --- a/synapse/events/utils.py +++ b/synapse/events/utils.py @@ -228,6 +228,7 @@ def format_event_for_client_v1(d): "replaces_state", "prev_content", "invite_room_state", + "knock_room_state", ) for key in copy_keys: if key in d["unsigned"]: @@ -311,6 +312,10 @@ def serialize_event( # If this is an invite for somebody else, then we don't care about the # invite_room_state as that's meant solely for the invitee. Other clients # will already have the state since they're in the room. + # + # Note that is_invite only seems to be set to False in the case of appservices + # that are not interested in the target user of the invite. + # TODO: Figure out what to do here w.r.t knocking if not is_invite: d["unsigned"].pop("invite_room_state", None) diff --git a/synapse/handlers/sync.py b/synapse/handlers/sync.py index 32e53c2d2566..71685cc9f6f5 100644 --- a/synapse/handlers/sync.py +++ b/synapse/handlers/sync.py @@ -149,6 +149,16 @@ def __bool__(self) -> bool: return True +@attr.s(slots=True, frozen=True) +class KnockedSyncResult: + room_id = attr.ib(type=str) + knock = attr.ib(type=EventBase) + + def __bool__(self) -> bool: + """Knocked rooms should always be reported to the client""" + return True + + @attr.s(slots=True, frozen=True) class GroupsSyncResult: join = attr.ib(type=JsonDict) @@ -182,6 +192,7 @@ class _RoomChanges: room_entries = attr.ib(type=List["RoomSyncResultBuilder"]) invited = attr.ib(type=List[InvitedSyncResult]) + knocked = attr.ib(type=List[KnockedSyncResult]) newly_joined_rooms = attr.ib(type=List[str]) newly_left_rooms = attr.ib(type=List[str]) @@ -195,6 +206,7 @@ class SyncResult: account_data: List of account_data events for the user. joined: JoinedSyncResult for each joined room. invited: InvitedSyncResult for each invited room. + knocked: KnockedSyncResult for each knocked on room. archived: ArchivedSyncResult for each archived room. to_device: List of direct messages for the device. device_lists: List of user_ids whose devices have changed @@ -210,6 +222,7 @@ class SyncResult: account_data = attr.ib(type=List[JsonDict]) joined = attr.ib(type=List[JoinedSyncResult]) invited = attr.ib(type=List[InvitedSyncResult]) + knocked = attr.ib(type=List[KnockedSyncResult]) archived = attr.ib(type=List[ArchivedSyncResult]) to_device = attr.ib(type=List[JsonDict]) device_lists = attr.ib(type=DeviceLists) @@ -226,6 +239,7 @@ def __bool__(self) -> bool: self.presence or self.joined or self.invited + or self.knocked or self.archived or self.account_data or self.to_device @@ -997,7 +1011,7 @@ async def generate_sync_result( res = await self._generate_sync_entry_for_rooms( sync_result_builder, account_data_by_room ) - newly_joined_rooms, newly_joined_or_invited_users, _, _ = res + newly_joined_rooms, newly_joined_or_invited_or_knocking_users, _, _ = res _, _, newly_left_rooms, newly_left_users = res block_all_presence_data = ( @@ -1006,7 +1020,9 @@ async def generate_sync_result( if self.hs_config.use_presence and not block_all_presence_data: logger.debug("Fetching presence data") await self._generate_sync_entry_for_presence( - sync_result_builder, newly_joined_rooms, newly_joined_or_invited_users + sync_result_builder, + newly_joined_rooms, + newly_joined_or_invited_or_knocking_users, ) logger.debug("Fetching to-device data") @@ -1015,7 +1031,7 @@ async def generate_sync_result( device_lists = await self._generate_sync_entry_for_device_list( sync_result_builder, newly_joined_rooms=newly_joined_rooms, - newly_joined_or_invited_users=newly_joined_or_invited_users, + newly_joined_or_invited_or_knocking_users=newly_joined_or_invited_or_knocking_users, newly_left_rooms=newly_left_rooms, newly_left_users=newly_left_users, ) @@ -1049,6 +1065,7 @@ async def generate_sync_result( account_data=sync_result_builder.account_data, joined=sync_result_builder.joined, invited=sync_result_builder.invited, + knocked=sync_result_builder.knocked, archived=sync_result_builder.archived, to_device=sync_result_builder.to_device, device_lists=device_lists, @@ -1108,7 +1125,7 @@ async def _generate_sync_entry_for_device_list( self, sync_result_builder: "SyncResultBuilder", newly_joined_rooms: Set[str], - newly_joined_or_invited_users: Set[str], + newly_joined_or_invited_or_knocking_users: Set[str], newly_left_rooms: Set[str], newly_left_users: Set[str], ) -> DeviceLists: @@ -1117,8 +1134,9 @@ async def _generate_sync_entry_for_device_list( Args: sync_result_builder newly_joined_rooms: Set of rooms user has joined since previous sync - newly_joined_or_invited_users: Set of users that have joined or - been invited to a room since previous sync. + newly_joined_or_invited_or_knocking_users: Set of users that have joined, + been invited to a room or are knocking on a room since + previous sync. newly_left_rooms: Set of rooms user has left since previous sync newly_left_users: Set of users that have left a room we're in since previous sync @@ -1129,7 +1147,9 @@ async def _generate_sync_entry_for_device_list( # We're going to mutate these fields, so lets copy them rather than # assume they won't get used later. - newly_joined_or_invited_users = set(newly_joined_or_invited_users) + newly_joined_or_invited_or_knocking_users = set( + newly_joined_or_invited_or_knocking_users + ) newly_left_users = set(newly_left_users) if since_token and since_token.device_list_key: @@ -1168,11 +1188,11 @@ async def _generate_sync_entry_for_device_list( # Step 1b, check for newly joined rooms for room_id in newly_joined_rooms: joined_users = await self.state.get_current_users_in_room(room_id) - newly_joined_or_invited_users.update(joined_users) + newly_joined_or_invited_or_knocking_users.update(joined_users) # TODO: Check that these users are actually new, i.e. either they # weren't in the previous sync *or* they left and rejoined. - users_that_have_changed.update(newly_joined_or_invited_users) + users_that_have_changed.update(newly_joined_or_invited_or_knocking_users) user_signatures_changed = await self.store.get_users_whose_signatures_changed( user_id, since_token.device_list_key @@ -1417,6 +1437,7 @@ async def _generate_sync_entry_for_rooms( room_entries = room_changes.room_entries invited = room_changes.invited + knocked = room_changes.knocked newly_joined_rooms = room_changes.newly_joined_rooms newly_left_rooms = room_changes.newly_left_rooms @@ -1437,9 +1458,10 @@ async def handle_room_entries(room_entry): await concurrently_execute(handle_room_entries, room_entries, 10) sync_result_builder.invited.extend(invited) + sync_result_builder.knocked.extend(knocked) - # Now we want to get any newly joined or invited users - newly_joined_or_invited_users = set() + # Now we want to get any newly joined, invited or knocking users + newly_joined_or_invited_or_knocking_users = set() newly_left_users = set() if since_token: for joined_sync in sync_result_builder.joined: @@ -1451,19 +1473,22 @@ async def handle_room_entries(room_entry): if ( event.membership == Membership.JOIN or event.membership == Membership.INVITE + or event.membership == Membership.KNOCK ): - newly_joined_or_invited_users.add(event.state_key) + newly_joined_or_invited_or_knocking_users.add( + event.state_key + ) else: prev_content = event.unsigned.get("prev_content", {}) prev_membership = prev_content.get("membership", None) if prev_membership == Membership.JOIN: newly_left_users.add(event.state_key) - newly_left_users -= newly_joined_or_invited_users + newly_left_users -= newly_joined_or_invited_or_knocking_users return ( set(newly_joined_rooms), - newly_joined_or_invited_users, + newly_joined_or_invited_or_knocking_users, set(newly_left_rooms), newly_left_users, ) @@ -1519,6 +1544,7 @@ async def _get_rooms_changed( newly_left_rooms = [] room_entries = [] invited = [] + knocked = [] for room_id, events in mem_change_events_by_room_id.items(): logger.debug( "Membership changes in %s: [%s]", @@ -1598,9 +1624,17 @@ async def _get_rooms_changed( should_invite = non_joins[-1].membership == Membership.INVITE if should_invite: if event.sender not in ignored_users: - room_sync = InvitedSyncResult(room_id, invite=non_joins[-1]) - if room_sync: - invited.append(room_sync) + invite_room_sync = InvitedSyncResult(room_id, invite=non_joins[-1]) + if invite_room_sync: + invited.append(invite_room_sync) + + # Only bother if our latest membership in the room is knock (and we haven't + # been accepted/rejected in the meantime). + should_knock = non_joins[-1].membership == Membership.KNOCK + if should_knock: + knock_room_sync = KnockedSyncResult(room_id, knock=non_joins[-1]) + if knock_room_sync: + knocked.append(knock_room_sync) # Always include leave/ban events. Just take the last one. # TODO: How do we handle ban -> leave in same batch? @@ -1704,7 +1738,9 @@ async def _get_rooms_changed( ) room_entries.append(entry) - return _RoomChanges(room_entries, invited, newly_joined_rooms, newly_left_rooms) + return _RoomChanges( + room_entries, invited, knocked, newly_joined_rooms, newly_left_rooms, + ) async def _get_all_rooms( self, sync_result_builder: "SyncResultBuilder", ignored_users: FrozenSet[str] @@ -1724,6 +1760,7 @@ async def _get_all_rooms( membership_list = ( Membership.INVITE, + Membership.KNOCK, Membership.JOIN, Membership.LEAVE, Membership.BAN, @@ -1735,6 +1772,7 @@ async def _get_all_rooms( room_entries = [] invited = [] + knocked = [] for event in room_list: if event.membership == Membership.JOIN: @@ -1754,8 +1792,11 @@ async def _get_all_rooms( continue invite = await self.store.get_event(event.event_id) invited.append(InvitedSyncResult(room_id=event.room_id, invite=invite)) + elif event.membership == Membership.KNOCK: + knock = await self.store.get_event(event.event_id) + knocked.append(KnockedSyncResult(room_id=event.room_id, knock=knock)) elif event.membership in (Membership.LEAVE, Membership.BAN): - # Always send down rooms we were banned or kicked from. + # Always send down rooms we were banned from or kicked from. if not sync_config.filter_collection.include_leave: if event.membership == Membership.LEAVE: if user_id == event.sender: @@ -1776,7 +1817,7 @@ async def _get_all_rooms( ) ) - return _RoomChanges(room_entries, invited, [], []) + return _RoomChanges(room_entries, invited, knocked, [], []) async def _generate_room_entry( self, @@ -2065,6 +2106,7 @@ class SyncResultBuilder: account_data (list) joined (list[JoinedSyncResult]) invited (list[InvitedSyncResult]) + knocked (list[KnockedSyncResult]) archived (list[ArchivedSyncResult]) groups (GroupsSyncResult|None) to_device (list) @@ -2080,6 +2122,7 @@ class SyncResultBuilder: account_data = attr.ib(type=List[JsonDict], default=attr.Factory(list)) joined = attr.ib(type=List[JoinedSyncResult], default=attr.Factory(list)) invited = attr.ib(type=List[InvitedSyncResult], default=attr.Factory(list)) + knocked = attr.ib(type=List[KnockedSyncResult], default=attr.Factory(list)) archived = attr.ib(type=List[ArchivedSyncResult], default=attr.Factory(list)) groups = attr.ib(type=Optional[GroupsSyncResult], default=None) to_device = attr.ib(type=List[JsonDict], default=attr.Factory(list)) diff --git a/synapse/rest/client/v2_alpha/sync.py b/synapse/rest/client/v2_alpha/sync.py index 2b84eb89c02c..2c9c5add727e 100644 --- a/synapse/rest/client/v2_alpha/sync.py +++ b/synapse/rest/client/v2_alpha/sync.py @@ -15,6 +15,7 @@ import itertools import logging +from typing import Any, Callable, Dict, List from synapse.api.constants import PresenceState from synapse.api.errors import Codes, StoreError, SynapseError @@ -24,7 +25,7 @@ format_event_raw, ) from synapse.handlers.presence import format_user_presence_state -from synapse.handlers.sync import SyncConfig +from synapse.handlers.sync import KnockedSyncResult, SyncConfig from synapse.http.servlet import RestServlet, parse_boolean, parse_integer, parse_string from synapse.types import StreamToken from synapse.util import json_decoder @@ -212,6 +213,10 @@ async def encode_response(self, time_now, sync_result, access_token_id, filter): sync_result.invited, time_now, access_token_id, event_formatter ) + knocked = await self.encode_knocked( + sync_result.knocked, time_now, access_token_id, event_formatter + ) + archived = await self.encode_archived( sync_result.archived, time_now, @@ -229,7 +234,12 @@ async def encode_response(self, time_now, sync_result, access_token_id, filter): "left": list(sync_result.device_lists.left), }, "presence": SyncRestServlet.encode_presence(sync_result.presence, time_now), - "rooms": {"join": joined, "invite": invited, "leave": archived}, + "rooms": { + "join": joined, + "invite": invited, + "knock": knocked, + "leave": archived, + }, "groups": { "join": sync_result.groups.join, "invite": sync_result.groups.invite, @@ -295,7 +305,7 @@ async def encode_invited(self, rooms, time_now, token_id, event_formatter): Args: rooms(list[synapse.handlers.sync.InvitedSyncResult]): list of - sync results for rooms this user is joined to + sync results for rooms this user is invited to time_now(int): current time - used as a baseline for age calculations token_id(int): ID of the user's auth token - used for namespacing @@ -324,6 +334,52 @@ async def encode_invited(self, rooms, time_now, token_id, event_formatter): return invited + async def encode_knocked( + self, + rooms: List[KnockedSyncResult], + time_now: int, + token_id: int, + event_formatter: Callable[[Dict], Dict], + ) -> Dict[str, Dict[str, Any]]: + """ + Encode the rooms we've knocked on in a sync result. + + Args: + rooms: list of sync results for rooms this user is knocking on + time_now: current time - used as a baseline for age calculations + token_id: ID of the user's auth token - used for namespacing of transaction IDs + event_formatter: function to convert from federation format to client format + + Returns: + The list of rooms the user has knocked on, in our response format. + """ + knocked = {} + for room in rooms: + knock = await self._event_serializer.serialize_event( + room.knock, time_now, token_id=token_id, event_format=event_formatter, + ) + + # Extract the `unsigned` key from the knock event. + # This is where we (cheekily) store the knock state events + unsigned = knock.setdefault("unsigned", {}) + + # Extract the stripped room state from the unsigned dict + # This is for clients to get a little bit of information about + # the room they've knocked on, without revealing any sensitive information + knocked_state = list(unsigned.pop("knock_room_state", [])) + + # Append the actual knock membership event itself as well + # TODO: I *believe* this is just for the client's sake of track its membership + # state in each room, but I could be wrong. This certainly doesn't seem like it + # could have any negative effects besides resource usage + knocked_state.append(knock) + + # Build the `knock_state` dictionary, which will contain the state of the + # room that the client has knocked on + knocked[room.room_id] = {"knock_state": {"events": knocked_state}} + + return knocked + async def encode_archived( self, rooms, time_now, token_id, event_fields, event_formatter ): From 8e443ddf9252b6c88c10430a055d26ca680ae1cc Mon Sep 17 00:00:00 2001 From: Andrew Morgan Date: Thu, 12 Nov 2020 17:04:03 +0000 Subject: [PATCH 009/112] Auto-add displaynames to knock events if they're missing Tiny commit to just bring knocking up to feature parity. --- synapse/handlers/message.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/synapse/handlers/message.py b/synapse/handlers/message.py index 94bdc15dcf36..4d332f9af174 100644 --- a/synapse/handlers/message.py +++ b/synapse/handlers/message.py @@ -1,7 +1,8 @@ # -*- coding: utf-8 -*- # Copyright 2014-2016 OpenMarket Ltd # Copyright 2017-2018 New Vector Ltd -# Copyright 2019 The Matrix.org Foundation C.I.C. +# Copyright 2019-2020 The Matrix.org Foundation C.I.C. +# Copyrignt 2020 Sorunome # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -914,8 +915,8 @@ async def handle_new_client_event( room_version = await self.store.get_room_version_id(event.room_id) if event.internal_metadata.is_out_of_band_membership(): - # the only sort of out-of-band-membership events we expect to see here - # are invite rejections we have generated ourselves. + # the only sort of out-of-band-membership events we expect to see here are + # invite rejections and rescinded knocks that we have generated ourselves. assert event.type == EventTypes.Member assert event.content["membership"] == Membership.LEAVE else: From 082f04b6de499999afcd3ef1ed4ac0044159203b Mon Sep 17 00:00:00 2001 From: Andrew Morgan Date: Thu, 12 Nov 2020 17:06:52 +0000 Subject: [PATCH 010/112] Add some handy db methods for knocking This just adds some database methods that we'll need to use when implementing rescinding of knocks of clients. They are equivalent to their invite-related counterparts. --- synapse/storage/databases/main/roommember.py | 33 ++++++++++++++++++++ 1 file changed, 33 insertions(+) diff --git a/synapse/storage/databases/main/roommember.py b/synapse/storage/databases/main/roommember.py index 01d9dbb36f44..d880c50f76f9 100644 --- a/synapse/storage/databases/main/roommember.py +++ b/synapse/storage/databases/main/roommember.py @@ -272,6 +272,21 @@ async def get_invited_rooms_for_local_user(self, user_id: str) -> RoomsForUser: user_id, [Membership.INVITE] ) + @cached() + async def get_knocked_rooms_for_local_user(self, user_id: str) -> RoomsForUser: + """Get all the rooms the *local* user currently has the membership of knock for. + + Args: + user_id: The user ID. + + Returns: + A list of RoomsForUser. + """ + + return await self.get_rooms_for_local_user_where_membership_is( + user_id, [Membership.KNOCK] + ) + async def get_invite_for_local_user_in_room( self, user_id: str, room_id: str ) -> Optional[RoomsForUser]: @@ -290,6 +305,24 @@ async def get_invite_for_local_user_in_room( return invite return None + async def get_knock_for_local_user_in_room( + self, user_id: str, room_id: str + ) -> Optional[RoomsForUser]: + """Gets the knock for the given *local* user and room. + + Args: + user_id: The user ID to find the knock of. + room_id: The room to user knocked on. + + Returns: + Either a RoomsForUser or None if no knock was found. + """ + knocks = await self.get_knocked_rooms_for_local_user(user_id) + for knock in knocks: + if knock.room_id == room_id: + return knock + return None + async def get_rooms_for_local_user_where_membership_is( self, user_id: str, membership_list: Collection[str] ) -> List[RoomsForUser]: From e46ceb10a1e3d15686ed6950e8690277a86afbc9 Mon Sep 17 00:00:00 2001 From: Andrew Morgan Date: Thu, 12 Nov 2020 18:35:13 +0000 Subject: [PATCH 011/112] Implement locally rescinding a federated knock As mentioned in the MSC, a user can rescind (take back) a knock while it is pending by sending a leave event to the room. This will set their membership to leave instead of knock. Now, this functionality already worked before this commit for rooms that the homeserver was already in. What didn't work was: * Rescinding a knock over federation to a room with active homeservers * Rescinding a knock over federation to a room with inactive homeservers This commit addresses the second bullet point, and leaves the first bullet point as a TODO (as it is an edge case an not immediately obvious how it would be done). What this commit does is crib off the same functionality as locally rejecting an invite. That occurs when we are unable to contact the homeserver that originally sent us an invite. Instead an out-of-band leave membership event will be generated and sent to clients locally. The same is happening here. You can mostly ignore the new generate_local_out_of_band_membership methods, those are just some structural bits to allow us to call that method from RoomMemberHandler. The real meat of this commit is moving about and adding some logic in `update_membership_locked`, specifically for when we're updating a user's membership to "leave". There was already some code in there to check whether the room to send the leave to was a room the homeserver is not currently a part of. In that case, we'd remote reject the knock. This commit just extends that to also rescind knocks if the user's membership in the room is currently "knock". We skip the remote attempt for now and go straight to generating a local leave membership event. --- synapse/handlers/room_member.py | 97 +++++++++++++++++++------- synapse/handlers/room_member_worker.py | 31 +++++++- 2 files changed, 100 insertions(+), 28 deletions(-) diff --git a/synapse/handlers/room_member.py b/synapse/handlers/room_member.py index a8518ac3b616..499b7cd47b40 100644 --- a/synapse/handlers/room_member.py +++ b/synapse/handlers/room_member.py @@ -1,5 +1,6 @@ # -*- coding: utf-8 -*- # Copyright 2016-2020 The Matrix.org Foundation C.I.C. +# Copyright 2020 Sorunome # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -147,6 +148,28 @@ async def remote_reject_invite( """ raise NotImplementedError() + @abc.abstractmethod + async def generate_local_out_of_band_leave( + self, + previous_membership_event: EventBase, + txn_id: Optional[str], + requester: Requester, + content: JsonDict, + ): + """Generate a local leave event for a room + + Args: + previous_membership_event: the previous membership event for this user + txn_id: optional transaction ID supplied by the client + requester: user making the request, according to the access token + content: additional content to include in the leave event. + Normally an empty dict. + + Returns: + A tuple containing (event_id, stream_id of the leave event) + """ + raise NotImplementedError() + @abc.abstractmethod async def _user_left_room(self, target: UserID, room_id: str) -> None: """Notifies distributor on master process that the user has left the @@ -532,39 +555,59 @@ async def update_membership_locked( invite = await self.store.get_invite_for_local_user_in_room( user_id=target.to_string(), room_id=room_id ) # type: Optional[RoomsForUser] - if not invite: + if invite: logger.info( - "%s sent a leave request to %s, but that is not an active room " - "on this server, and there is no pending invite", + "%s rejects invite to %s from %s", target, room_id, + invite.sender, ) - raise SynapseError(404, "Not a known room") + if not self.hs.is_mine_id(invite.sender): + # send the rejection to the inviter's HS (with fallback to + # local event) + return await self.remote_reject_invite( + invite.event_id, txn_id, requester, content, + ) + + # the inviter was on our server, but has now left. Carry on + # with the normal rejection codepath, which will also send the + # rejection out to any other servers we believe are still in the room. + + # thanks to overzealous cleaning up of event_forward_extremities in + # `delete_old_current_state_events`, it's possible to end up with no + # forward extremities here. If that happens, let's just hang the + # rejection off the invite event. + # + # see: https://github.com/matrix-org/synapse/issues/7139 + if len(latest_event_ids) == 0: + latest_event_ids = [invite.event_id] + + else: + # or perhaps this is a remote room that a local user has knocked on + knock = await self.store.get_knock_for_local_user_in_room( + user_id=target.to_string(), room_id=room_id + ) # type: Optional[RoomsForUser] + if knock: + # TODO: We don't yet support rescinding knocks over federation + # as we don't know which homeserver to send it to. An obvious + # candidate is the remote homeserver we originally knocked through, + # however we don't currently store that information. + + # Just rescind the knock locally + knock_event = await self.store.get_event(knock.event_id) + return await self.generate_local_out_of_band_leave( + knock_event, txn_id, requester, content + ) - logger.info( - "%s rejects invite to %s from %s", target, room_id, invite.sender - ) - - if not self.hs.is_mine_id(invite.sender): - # send the rejection to the inviter's HS (with fallback to - # local event) - return await self.remote_reject_invite( - invite.event_id, txn_id, requester, content, + logger.info( + "%s sent a leave request to %s, but that is not an active room " + "on this server, and there is no pending knock", + target, + room_id, ) - # the inviter was on our server, but has now left. Carry on - # with the normal rejection codepath, which will also send the - # rejection out to any other servers we believe are still in the room. - - # thanks to overzealous cleaning up of event_forward_extremities in - # `delete_old_current_state_events`, it's possible to end up with no - # forward extremities here. If that happens, let's just hang the - # rejection off the invite event. - # - # see: https://github.com/matrix-org/synapse/issues/7139 - if len(latest_event_ids) == 0: - latest_event_ids = [invite.event_id] + raise SynapseError(404, "Not a known room") elif effective_membership_state == Membership.KNOCK: if not is_host_in_room: @@ -1135,11 +1178,11 @@ async def remote_reject_invite( # logger.warning("Failed to reject invite: %s", e) - return await self._generate_local_out_of_band_leave( + return await self.generate_local_out_of_band_leave( invite_event, txn_id, requester, content ) - async def _generate_local_out_of_band_leave( + async def generate_local_out_of_band_leave( self, previous_membership_event: EventBase, txn_id: Optional[str], diff --git a/synapse/handlers/room_member_worker.py b/synapse/handlers/room_member_worker.py index f9cea604ec14..a21ca8c8fcee 100644 --- a/synapse/handlers/room_member_worker.py +++ b/synapse/handlers/room_member_worker.py @@ -1,5 +1,6 @@ # -*- coding: utf-8 -*- # Copyright 2018 New Vector Ltd +# Copyright 2020 The Matrix.org Foundation C.I.C. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -17,13 +18,14 @@ from typing import List, Optional, Tuple from synapse.api.errors import SynapseError +from synapse.events import EventBase from synapse.handlers.room_member import RoomMemberHandler from synapse.replication.http.membership import ( ReplicationRemoteJoinRestServlet as ReplRemoteJoin, ReplicationRemoteRejectInviteRestServlet as ReplRejectInvite, ReplicationUserJoinedLeftRoomRestServlet as ReplJoinedLeft, ) -from synapse.types import Requester, UserID +from synapse.types import JsonDict, Requester, UserID logger = logging.getLogger(__name__) @@ -79,6 +81,33 @@ async def remote_reject_invite( ) return ret["event_id"], ret["stream_id"] + async def generate_local_out_of_band_leave( + self, + previous_membership_event: EventBase, + txn_id: Optional[str], + requester: Requester, + content: JsonDict, + ): + """Generate a local leave event for a room + + Args: + previous_membership_event: the previous membership event for this user + txn_id: optional transaction ID supplied by the client + requester: user making the request, according to the access token + content: additional content to include in the leave event. + Normally an empty dict. + + Returns: + A tuple containing (event_id, stream_id of the leave event) + """ + ret = await self.generate_local_out_of_band_leave( + previous_membership_event=previous_membership_event, + txn_id=txn_id, + requester=requester, + content=content, + ) + return ret["event_id"], ret["stream_id"] + async def _remote_knock( self, remote_room_hosts: List[str], room_id: str, user: UserID, content: dict, ) -> Tuple[str, int]: From 56cde8b12c08de22a38c66e0ea4f4061a590c401 Mon Sep 17 00:00:00 2001 From: Andrew Morgan Date: Thu, 12 Nov 2020 19:01:18 +0000 Subject: [PATCH 012/112] Add knock membership events to stats generation --- synapse/handlers/stats.py | 5 +++++ .../delta/58/24add_knock_members_to_stats.sql | 17 +++++++++++++++++ synapse/storage/databases/main/stats.py | 1 + 3 files changed, 23 insertions(+) create mode 100644 synapse/storage/databases/main/schema/delta/58/24add_knock_members_to_stats.sql diff --git a/synapse/handlers/stats.py b/synapse/handlers/stats.py index dc62b21c06f9..3961f727aa50 100644 --- a/synapse/handlers/stats.py +++ b/synapse/handlers/stats.py @@ -1,5 +1,6 @@ # -*- coding: utf-8 -*- # Copyright 2018 New Vector Ltd +# Copyright 2020 Sorunome # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -225,6 +226,8 @@ async def _handle_deltas(self, deltas): room_stats_delta["left_members"] -= 1 elif prev_membership == Membership.BAN: room_stats_delta["banned_members"] -= 1 + elif prev_membership == Membership.KNOCK: + room_stats_delta["knocked_members"] -= 1 else: raise ValueError( "%r is not a valid prev_membership" % (prev_membership,) @@ -246,6 +249,8 @@ async def _handle_deltas(self, deltas): room_stats_delta["left_members"] += 1 elif membership == Membership.BAN: room_stats_delta["banned_members"] += 1 + elif membership == Membership.KNOCK: + room_stats_delta["knocked_members"] += 1 else: raise ValueError("%r is not a valid membership" % (membership,)) diff --git a/synapse/storage/databases/main/schema/delta/58/24add_knock_members_to_stats.sql b/synapse/storage/databases/main/schema/delta/58/24add_knock_members_to_stats.sql new file mode 100644 index 000000000000..658f55a38452 --- /dev/null +++ b/synapse/storage/databases/main/schema/delta/58/24add_knock_members_to_stats.sql @@ -0,0 +1,17 @@ +/* Copyright 2020 Sorunome + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +ALTER TABLE room_stats_current ADD knocked_members INT NOT NULL DEFAULT '0'; +ALTER TABLE room_stats_historical ADD knocked_members BIGINT NOT NULL DEFAULT '0'; diff --git a/synapse/storage/databases/main/stats.py b/synapse/storage/databases/main/stats.py index 0cdb3ec1f7ff..62b75515170d 100644 --- a/synapse/storage/databases/main/stats.py +++ b/synapse/storage/databases/main/stats.py @@ -41,6 +41,7 @@ "current_state_events", "joined_members", "invited_members", + "knocked_members", "left_members", "banned_members", "local_users_in_room", From 8e0ac8cc451636d0c8ca835766c6f2f07ccc0a64 Mon Sep 17 00:00:00 2001 From: Andrew Morgan Date: Thu, 12 Nov 2020 19:03:08 +0000 Subject: [PATCH 013/112] Changelog --- changelog.d/6739.feature | 1 + 1 file changed, 1 insertion(+) create mode 100644 changelog.d/6739.feature diff --git a/changelog.d/6739.feature b/changelog.d/6739.feature new file mode 100644 index 000000000000..7ca96e0ce87e --- /dev/null +++ b/changelog.d/6739.feature @@ -0,0 +1 @@ +Implement "room knocking" as per MSC2403. Contributed by Sorunome and anoa. \ No newline at end of file From 392b315207e50c03dcd203fe4220029d03aec7f1 Mon Sep 17 00:00:00 2001 From: Andrew Morgan Date: Mon, 23 Nov 2020 15:04:27 +0000 Subject: [PATCH 014/112] Update changelog to include link to MSC --- changelog.d/6739.feature | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/changelog.d/6739.feature b/changelog.d/6739.feature index 7ca96e0ce87e..9c41140194b2 100644 --- a/changelog.d/6739.feature +++ b/changelog.d/6739.feature @@ -1 +1 @@ -Implement "room knocking" as per MSC2403. Contributed by Sorunome and anoa. \ No newline at end of file +Implement "room knocking" as per [MSC2403](https://github.com/matrix-org/matrix-doc/pull/2403). Contributed by Sorunome and anoa. \ No newline at end of file From 665e8638d15cabae84f3cd5d8afd2fd4066775ed Mon Sep 17 00:00:00 2001 From: Andrew Morgan Date: Mon, 23 Nov 2020 15:17:00 +0000 Subject: [PATCH 015/112] Update documentation comment for config option room_invite_state_types --- docs/sample_config.yaml | 4 +++- synapse/config/api.py | 4 +++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/docs/sample_config.yaml b/docs/sample_config.yaml index 513c77f86c8c..e6b37281e819 100644 --- a/docs/sample_config.yaml +++ b/docs/sample_config.yaml @@ -1387,7 +1387,9 @@ metrics_flags: ## API Configuration ## -# A list of event types that will be included in the room_invite_state +# A list of event types from a room that will be given to users when they +# are invited to a room. This allows clients to display information about the +# room that they've been invited to, without actually being in the room yet. # #room_invite_state_types: # - "m.room.join_rules" diff --git a/synapse/config/api.py b/synapse/config/api.py index 7604993b637e..26817cfe125a 100644 --- a/synapse/config/api.py +++ b/synapse/config/api.py @@ -40,7 +40,9 @@ def generate_config_section(cls, **kwargs): return """\ ## API Configuration ## - # A list of event types that will be included in the room_invite_state + # A list of event types from a room that will be given to users when they + # are invited to a room. This allows clients to display information about the + # room that they've been invited to, without actually being in the room yet. # #room_invite_state_types: # - "{JoinRules}" From f08a20b1a60e6cafca71a7f56f94f787fb7300ea Mon Sep 17 00:00:00 2001 From: Andrew Morgan Date: Mon, 23 Nov 2020 15:25:57 +0000 Subject: [PATCH 016/112] Move default_room_state_types to a module-level constant --- synapse/config/api.py | 20 +++++++++++--------- 1 file changed, 11 insertions(+), 9 deletions(-) diff --git a/synapse/config/api.py b/synapse/config/api.py index 26817cfe125a..a160680d5175 100644 --- a/synapse/config/api.py +++ b/synapse/config/api.py @@ -17,23 +17,25 @@ from ._base import Config +# The default types of room state to send to users to are invited to or knock on a room. +DEFAULT_ROOM_STATE_TYPES = [ + EventTypes.JoinRules, + EventTypes.CanonicalAlias, + EventTypes.RoomAvatar, + EventTypes.RoomEncryption, + EventTypes.Name, +] + class ApiConfig(Config): section = "api" def read_config(self, config, **kwargs): - default_room_state_types = [ - EventTypes.JoinRules, - EventTypes.CanonicalAlias, - EventTypes.RoomAvatar, - EventTypes.RoomEncryption, - EventTypes.Name, - ] self.room_invite_state_types = config.get( - "room_invite_state_types", default_room_state_types + "room_invite_state_types", DEFAULT_ROOM_STATE_TYPES ) self.room_knock_state_types = config.get( - "room_knock_state_types", default_room_state_types + "room_knock_state_types", DEFAULT_ROOM_STATE_TYPES ) def generate_config_section(cls, **kwargs): From f8fa8fabcfd025643c745cbb6f831c09ba803957 Mon Sep 17 00:00:00 2001 From: Andrew Morgan Date: Mon, 23 Nov 2020 17:08:16 +0000 Subject: [PATCH 017/112] Explain and rename is_invite --- synapse/appservice/api.py | 11 ++++++++--- synapse/events/utils.py | 22 +++++++++++----------- synapse/rest/client/v2_alpha/sync.py | 2 +- 3 files changed, 20 insertions(+), 15 deletions(-) diff --git a/synapse/appservice/api.py b/synapse/appservice/api.py index e366a982b801..5dcd9ea2908a 100644 --- a/synapse/appservice/api.py +++ b/synapse/appservice/api.py @@ -18,7 +18,7 @@ from prometheus_client import Counter -from synapse.api.constants import EventTypes, ThirdPartyEntityKind +from synapse.api.constants import EventTypes, Membership, ThirdPartyEntityKind from synapse.api.errors import CodeMessageException from synapse.events import EventBase from synapse.events.utils import serialize_event @@ -249,9 +249,14 @@ def _serialize(self, service, events): e, time_now, as_client_event=True, - is_invite=( + # If this is an invite or a knock membership event, and we're interested + # in this user, then include any stripped state alongside the event. + include_stripped_room_state=( e.type == EventTypes.Member - and e.membership == "invite" + and ( + e.membership == Membership.INVITE + or e.membership == Membership.KNOCK + ) and service.is_interested_in_user(e.state_key) ), ) diff --git a/synapse/events/utils.py b/synapse/events/utils.py index 589fa877d276..7d95ddda5170 100644 --- a/synapse/events/utils.py +++ b/synapse/events/utils.py @@ -265,7 +265,7 @@ def serialize_event( event_format=format_event_for_client_v1, token_id=None, only_event_fields=None, - is_invite=False, + include_stripped_room_state=False, ): """Serialize event for clients @@ -276,8 +276,10 @@ def serialize_event( event_format token_id only_event_fields - is_invite (bool): Whether this is an invite that is being sent to the - invitee + include_stripped_room_state (bool): Some events can have stripped room state + stored in the `unsigned` field. This is required for invite and knock + functionality. If this option is False, that state will be removed from the + event before it is returned. Otherwise, it will be kept. Returns: dict @@ -309,15 +311,13 @@ def serialize_event( if txn_id is not None: d["unsigned"]["transaction_id"] = txn_id - # If this is an invite for somebody else, then we don't care about the - # invite_room_state as that's meant solely for the invitee. Other clients - # will already have the state since they're in the room. - # - # Note that is_invite only seems to be set to False in the case of appservices - # that are not interested in the target user of the invite. - # TODO: Figure out what to do here w.r.t knocking - if not is_invite: + # invite_room_state is a list of stripped room state events that are meant to + # provide metadata about a room to an invitee. knock_room_state is the same, + # but for user knocking on a room. They are intended to only be included in specific + # circumstances, such as down sync, and should not be included in any other case. + if not include_stripped_room_state: d["unsigned"].pop("invite_room_state", None) + d["unsigned"].pop("knock_room_state", None) if as_client_event: d = event_format(d) diff --git a/synapse/rest/client/v2_alpha/sync.py b/synapse/rest/client/v2_alpha/sync.py index 2c9c5add727e..09da07152734 100644 --- a/synapse/rest/client/v2_alpha/sync.py +++ b/synapse/rest/client/v2_alpha/sync.py @@ -324,7 +324,7 @@ async def encode_invited(self, rooms, time_now, token_id, event_formatter): time_now, token_id=token_id, event_format=event_formatter, - is_invite=True, + include_stripped_room_state=True, ) unsigned = dict(invite.get("unsigned", {})) invite["unsigned"] = unsigned From aa6652564b40430514bf81e7f38153e4a4af9178 Mon Sep 17 00:00:00 2001 From: Andrew Morgan Date: Mon, 23 Nov 2020 17:59:40 +0000 Subject: [PATCH 018/112] Add docstrings to FederationServer knock methods --- synapse/federation/federation_server.py | 34 +++++++++++++++++++++++-- 1 file changed, 32 insertions(+), 2 deletions(-) diff --git a/synapse/federation/federation_server.py b/synapse/federation/federation_server.py index 3417f6291639..c3012911659f 100644 --- a/synapse/federation/federation_server.py +++ b/synapse/federation/federation_server.py @@ -567,7 +567,22 @@ async def on_send_leave_request( await self.handler.on_send_leave_request(origin, pdu) return {} - async def on_make_knock_request(self, origin: str, room_id: str, user_id: str): + async def on_make_knock_request( + self, origin: str, room_id: str, user_id: str, + ) -> Dict[str, Union[EventBase, str]]: + """We've received a /make_knock/ request, so we create a partial knock event + for the room and hand that back, along with the room version, to the knocking + homeserver. We do *not* persist or process this event until the other server has + signed it and sent it back. + + Args: + origin: The (verified) server name of the requesting server. + room_id: The room to create the knock event in. + user_id: The user to create the knock for. + + Returns: + The partial knock event. + """ origin_host, _ = parse_server_name(origin) await self.check_server_matches_acl(origin_host, room_id) pdu = await self.handler.on_make_knock_request(origin, room_id, user_id) @@ -577,7 +592,22 @@ async def on_make_knock_request(self, origin: str, room_id: str, user_id: str): time_now = self._clock.time_msec() return {"event": pdu.get_pdu_json(time_now), "room_version": room_version} - async def on_send_knock_request(self, origin: str, content: JsonDict, room_id: str): + async def on_send_knock_request( + self, origin: str, content: JsonDict, room_id: str, + ) -> Dict[str, List[JsonDict]]: + """ + We have received a knock event for a room. Verify and send the event into the room + on the knocking homeserver's behalf. Then reply with some stripped state from the + room for the knockee. + + Args: + origin: The remote homeserver of the knocking user. + content: The content of the request. + room_id: The ID of the room to knock on. + + Returns: + The stripped room state. + """ logger.debug("on_send_knock_request: content: %s", content) room_version = await self.store.get_room_version(room_id) From d4960c81422c3ddc19bb88bb7c024c899be57f93 Mon Sep 17 00:00:00 2001 From: Andrew Morgan Date: Mon, 23 Nov 2020 18:19:22 +0000 Subject: [PATCH 019/112] context -> room_id --- synapse/federation/transport/server.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/synapse/federation/transport/server.py b/synapse/federation/transport/server.py index 133de2f3a101..4f1a0baeb4e1 100644 --- a/synapse/federation/transport/server.py +++ b/synapse/federation/transport/server.py @@ -544,10 +544,10 @@ async def on_PUT(self, origin, content, query, room_id, event_id): class FederationMakeKnockServlet(BaseFederationServlet): - PATH = "/make_knock/(?P[^/]*)/(?P[^/]*)" + PATH = "/make_knock/(?P[^/]*)/(?P[^/]*)" - async def on_GET(self, origin, content, query, context, user_id): - content = await self.handler.on_make_knock_request(origin, context, user_id) + async def on_GET(self, origin, content, query, room_id, user_id): + content = await self.handler.on_make_knock_request(origin, room_id, user_id) return 200, content From 51f6c5d0c8b99177c8a3b5739098f908379fb14a Mon Sep 17 00:00:00 2001 From: Andrew Morgan Date: Mon, 23 Nov 2020 18:22:39 +0000 Subject: [PATCH 020/112] Remove old TODO --- synapse/handlers/federation.py | 1 - 1 file changed, 1 deletion(-) diff --git a/synapse/handlers/federation.py b/synapse/handlers/federation.py index 2af17974c4ba..377180886f56 100644 --- a/synapse/handlers/federation.py +++ b/synapse/handlers/federation.py @@ -1473,7 +1473,6 @@ async def do_knock( ) # Record the room ID and its version so that we have a record of the room - # TODO: Rename this function as its scope has expanded await self._maybe_store_room_on_outlier_membership( room_id=event.room_id, room_version=event_format_version ) From 98f96ec1970d756242d7ed9cc39b19de1196af6a Mon Sep 17 00:00:00 2001 From: Andrew Morgan Date: Mon, 23 Nov 2020 18:51:25 +0000 Subject: [PATCH 021/112] Abstract method _remote_knock -> remote_knock --- synapse/handlers/room_member.py | 6 +++--- synapse/handlers/room_member_worker.py | 6 +++--- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/synapse/handlers/room_member.py b/synapse/handlers/room_member.py index 499b7cd47b40..b9c896eca143 100644 --- a/synapse/handlers/room_member.py +++ b/synapse/handlers/room_member.py @@ -112,7 +112,7 @@ async def _remote_join( raise NotImplementedError() @abc.abstractmethod - async def _remote_knock( + async def remote_knock( self, remote_room_hosts: List[str], room_id: str, user: UserID, content: dict, ) -> Tuple[str, int]: """Try and knock on a room that this server is not in @@ -622,7 +622,7 @@ async def update_membership_locked( if "avatar_url" not in content: content["avatar_url"] = await profile.get_avatar_url(target) - return await self._remote_knock( + return await self.remote_knock( remote_room_hosts, room_id, target, content ) @@ -1242,7 +1242,7 @@ async def generate_local_out_of_band_leave( return result_event.event_id, result_event.internal_metadata.stream_ordering - async def _remote_knock( + async def remote_knock( self, remote_room_hosts: List[str], room_id: str, user: UserID, content: dict, ) -> Tuple[str, int]: """Sends a knock to a room. Attempts to do so via one remote out of a given list. diff --git a/synapse/handlers/room_member_worker.py b/synapse/handlers/room_member_worker.py index a21ca8c8fcee..e7d7a49b7cd6 100644 --- a/synapse/handlers/room_member_worker.py +++ b/synapse/handlers/room_member_worker.py @@ -108,14 +108,14 @@ async def generate_local_out_of_band_leave( ) return ret["event_id"], ret["stream_id"] - async def _remote_knock( + async def remote_knock( self, remote_room_hosts: List[str], room_id: str, user: UserID, content: dict, ) -> Tuple[str, int]: """Sends a knock to a room. - Implements RoomMemberHandler._remote_knock + Implements RoomMemberHandler.remote_knock """ - return await self._remote_knock( + return await self.remote_knock( remote_room_hosts=remote_room_hosts, room_id=room_id, user=user, From 9560d0a9b65ae673f8f3dc4e7b40fe4b24bb1222 Mon Sep 17 00:00:00 2001 From: Andrew Morgan Date: Mon, 23 Nov 2020 18:57:40 +0000 Subject: [PATCH 022/112] Use get_domain_from_id instead of manually parsing --- synapse/handlers/room_member.py | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/synapse/handlers/room_member.py b/synapse/handlers/room_member.py index b9c896eca143..a39c960bf784 100644 --- a/synapse/handlers/room_member.py +++ b/synapse/handlers/room_member.py @@ -13,7 +13,6 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. - import abc import logging import random @@ -33,7 +32,15 @@ from synapse.events import EventBase from synapse.events.snapshot import EventContext from synapse.storage.roommember import RoomsForUser -from synapse.types import JsonDict, Requester, RoomAlias, RoomID, StateMap, UserID +from synapse.types import ( + JsonDict, + Requester, + RoomAlias, + RoomID, + StateMap, + UserID, + get_domain_from_id, +) from synapse.util.async_helpers import Linearizer from synapse.util.distributor import user_left_room @@ -612,7 +619,7 @@ async def update_membership_locked( elif effective_membership_state == Membership.KNOCK: if not is_host_in_room: # The knock needs to be sent over federation instead - remote_room_hosts.append(room_id.split(":", 1)[1]) + remote_room_hosts.append(get_domain_from_id(room_id)) content["membership"] = Membership.KNOCK From e427bd24fc5f4096066eb0b90443a7cf89e16d7c Mon Sep 17 00:00:00 2001 From: Andrew Morgan Date: Mon, 23 Nov 2020 19:15:18 +0000 Subject: [PATCH 023/112] Add some missing documented return types --- synapse/handlers/room_member.py | 2 +- synapse/handlers/room_member_worker.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/synapse/handlers/room_member.py b/synapse/handlers/room_member.py index a39c960bf784..54fa80ccfd80 100644 --- a/synapse/handlers/room_member.py +++ b/synapse/handlers/room_member.py @@ -162,7 +162,7 @@ async def generate_local_out_of_band_leave( txn_id: Optional[str], requester: Requester, content: JsonDict, - ): + ) -> Tuple[str, int]: """Generate a local leave event for a room Args: diff --git a/synapse/handlers/room_member_worker.py b/synapse/handlers/room_member_worker.py index e7d7a49b7cd6..a4bb262b44ae 100644 --- a/synapse/handlers/room_member_worker.py +++ b/synapse/handlers/room_member_worker.py @@ -87,7 +87,7 @@ async def generate_local_out_of_band_leave( txn_id: Optional[str], requester: Requester, content: JsonDict, - ): + ) -> Tuple[str, int]: """Generate a local leave event for a room Args: From 8cec504071becc5cf66e957177bec633b2c48a2e Mon Sep 17 00:00:00 2001 From: Andrew Morgan <1342360+anoadragon453@users.noreply.github.com> Date: Mon, 23 Nov 2020 19:25:16 +0000 Subject: [PATCH 024/112] Apply suggestions from code review Co-authored-by: Patrick Cloke --- synapse/event_auth.py | 2 +- synapse/federation/transport/client.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/synapse/event_auth.py b/synapse/event_auth.py index b5848df2514b..5edd610d0922 100644 --- a/synapse/event_auth.py +++ b/synapse/event_auth.py @@ -329,7 +329,7 @@ def _is_membership_change_allowed( raise AuthError(403, "You are banned from this room") elif join_rule == JoinRules.PUBLIC: pass - elif join_rule in [JoinRules.INVITE, JoinRules.KNOCK]: + elif join_rule in (JoinRules.INVITE, JoinRules.KNOCK): if not caller_in_room and not caller_invited: raise AuthError(403, "You are not invited to this room.") else: diff --git a/synapse/federation/transport/client.py b/synapse/federation/transport/client.py index ca369af2dec5..83ce560e7773 100644 --- a/synapse/federation/transport/client.py +++ b/synapse/federation/transport/client.py @@ -301,8 +301,8 @@ async def send_knock_v1( self, destination: str, room_id: str, event_id: str, content: JsonDict, ) -> JsonDict: """ - Sends an signed knock membership event to a remote server. This is the second - step knocking after make_knock. + Sends a signed knock membership event to a remote server. This is the second + step for knocking after make_knock. Args: destination: The remote homeserver. From 162677b04bae90d9d732ff61cffc393cef4d6f16 Mon Sep 17 00:00:00 2001 From: Andrew Morgan Date: Tue, 24 Nov 2020 15:04:52 +0000 Subject: [PATCH 025/112] Fix locally rescinding knocks on worker setups Turns out there was some extra boilerplate around replication that is necessary to rescind knocks on worker setups. Workers need to call out to the master process to help remotely accept/reject invites, and the same is true for accepting/rescinding knocks. This commit adds replication client functions and handlers for remote_knock and remote_rescind_knock. As part of this, I turned generate_local_out_of_band_leave back into a private function. It was originally being called by RoomMemberHandler.update_membership_locked as it was simply a shortcut to generating a local leave event (aka rescinding a knock locally, but not informing any other servers about it). This shortcut was temporary while actually remote rescinding is left as a TODO. Instead of setting up all the boilerplate code for generate_local_out_of_band_leave, only to later replace it with remote_rescind_knock (which would call generate_local_out_of_band_leave if it failed to rescind a knock remotely), we just set up replication for remote_rescind_knock now. In the future, remote_rescind_knock will do more than just immediately giving up and rescinding the knock locally. --- synapse/handlers/room_member.py | 61 +++++++---- synapse/handlers/room_member_worker.py | 19 ++-- synapse/replication/http/membership.py | 141 ++++++++++++++++++++++++- 3 files changed, 192 insertions(+), 29 deletions(-) diff --git a/synapse/handlers/room_member.py b/synapse/handlers/room_member.py index 54fa80ccfd80..bfe6236ae6d8 100644 --- a/synapse/handlers/room_member.py +++ b/synapse/handlers/room_member.py @@ -156,24 +156,23 @@ async def remote_reject_invite( raise NotImplementedError() @abc.abstractmethod - async def generate_local_out_of_band_leave( + async def remote_rescind_knock( self, - previous_membership_event: EventBase, + knock_event_id: str, txn_id: Optional[str], requester: Requester, content: JsonDict, ) -> Tuple[str, int]: - """Generate a local leave event for a room + """Rescind a local knock made on a remote room. Args: - previous_membership_event: the previous membership event for this user - txn_id: optional transaction ID supplied by the client - requester: user making the request, according to the access token - content: additional content to include in the leave event. - Normally an empty dict. + knock_event_id: The ID of the knock event to rescind. + txn_id: An optional transaction ID supplied by the client. + requester: The user making the request, according to the access token. + content: The content of the generated leave event. Returns: - A tuple containing (event_id, stream_id of the leave event) + A tuple containing (event_id, stream_id of the leave event). """ raise NotImplementedError() @@ -596,15 +595,8 @@ async def update_membership_locked( user_id=target.to_string(), room_id=room_id ) # type: Optional[RoomsForUser] if knock: - # TODO: We don't yet support rescinding knocks over federation - # as we don't know which homeserver to send it to. An obvious - # candidate is the remote homeserver we originally knocked through, - # however we don't currently store that information. - - # Just rescind the knock locally - knock_event = await self.store.get_event(knock.event_id) - return await self.generate_local_out_of_band_leave( - knock_event, txn_id, requester, content + return await self.remote_rescind_knock( + knock.event_id, txn_id, requester, content ) logger.info( @@ -1185,11 +1177,40 @@ async def remote_reject_invite( # logger.warning("Failed to reject invite: %s", e) - return await self.generate_local_out_of_band_leave( + return await self._generate_local_out_of_band_leave( invite_event, txn_id, requester, content ) - async def generate_local_out_of_band_leave( + async def remote_rescind_knock( + self, + knock_event_id: str, + txn_id: Optional[str], + requester: Requester, + content: JsonDict, + ) -> Tuple[str, int]: + """ + Rescinds a local knock made on a remote room + + Args: + knock_event_id: The ID of the knock event to rescind. + txn_id: The transaction ID to use. + requester: The originator of the request. + content: The content of the leave event. + + Implements RoomMemberHandler.remote_rescind_knock + """ + # TODO: We don't yet support rescinding knocks over federation + # as we don't know which homeserver to send it to. An obvious + # candidate is the remote homeserver we originally knocked through, + # however we don't currently store that information. + + # Just rescind the knock locally + knock_event = await self.store.get_event(knock_event_id) + return await self._generate_local_out_of_band_leave( + knock_event, txn_id, requester, content + ) + + async def _generate_local_out_of_band_leave( self, previous_membership_event: EventBase, txn_id: Optional[str], diff --git a/synapse/handlers/room_member_worker.py b/synapse/handlers/room_member_worker.py index a4bb262b44ae..67c513b285a1 100644 --- a/synapse/handlers/room_member_worker.py +++ b/synapse/handlers/room_member_worker.py @@ -22,7 +22,9 @@ from synapse.handlers.room_member import RoomMemberHandler from synapse.replication.http.membership import ( ReplicationRemoteJoinRestServlet as ReplRemoteJoin, + ReplicationRemoteKnockRestServlet as ReplRemoteKnock, ReplicationRemoteRejectInviteRestServlet as ReplRejectInvite, + ReplicationRemoteRescindKnockRestServlet as ReplRescindKnock, ReplicationUserJoinedLeftRoomRestServlet as ReplJoinedLeft, ) from synapse.types import JsonDict, Requester, UserID @@ -35,7 +37,9 @@ def __init__(self, hs): super().__init__(hs) self._remote_join_client = ReplRemoteJoin.make_client(hs) + self._remote_knock_client = ReplRemoteKnock.make_client(hs) self._remote_reject_client = ReplRejectInvite.make_client(hs) + self._remote_rescind_client = ReplRescindKnock.make_client(hs) self._notify_change_client = ReplJoinedLeft.make_client(hs) async def _remote_join( @@ -81,17 +85,18 @@ async def remote_reject_invite( ) return ret["event_id"], ret["stream_id"] - async def generate_local_out_of_band_leave( + async def remote_rescind_knock( self, - previous_membership_event: EventBase, + knock_event_id: str, txn_id: Optional[str], requester: Requester, content: JsonDict, ) -> Tuple[str, int]: - """Generate a local leave event for a room + """ + Rescinds a local knock made on a remote room Args: - previous_membership_event: the previous membership event for this user + knock_event_id: the knock event txn_id: optional transaction ID supplied by the client requester: user making the request, according to the access token content: additional content to include in the leave event. @@ -100,8 +105,8 @@ async def generate_local_out_of_band_leave( Returns: A tuple containing (event_id, stream_id of the leave event) """ - ret = await self.generate_local_out_of_band_leave( - previous_membership_event=previous_membership_event, + ret = await self._remote_rescind_client( + knock_event_id=knock_event_id, txn_id=txn_id, requester=requester, content=content, @@ -115,7 +120,7 @@ async def remote_knock( Implements RoomMemberHandler.remote_knock """ - return await self.remote_knock( + return await self._remote_knock_client( remote_room_hosts=remote_room_hosts, room_id=room_id, user=user, diff --git a/synapse/replication/http/membership.py b/synapse/replication/http/membership.py index f0c37eaf5e1b..15047f4246ae 100644 --- a/synapse/replication/http/membership.py +++ b/synapse/replication/http/membership.py @@ -12,9 +12,10 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. - import logging -from typing import TYPE_CHECKING, Optional +from typing import TYPE_CHECKING, List, Optional + +from twisted.web.http import Request from synapse.http.servlet import parse_json_object_from_request from synapse.replication.http._base import ReplicationEndpoint @@ -88,6 +89,76 @@ async def _handle_request(self, request, room_id, user_id): return 200, {"event_id": event_id, "stream_id": stream_id} +class ReplicationRemoteKnockRestServlet(ReplicationEndpoint): + """Perform a remote knock for the given user on the given room + + Request format: + + POST /_synapse/replication/remote_knock/:room_id/:user_id + + { + "requester": ..., + "remote_room_hosts": [...], + "content": { ... } + } + """ + + NAME = "remote_knock" + PATH_ARGS = ("room_id", "user_id") + + def __init__(self, hs): + super().__init__(hs) + + self.federation_handler = hs.get_federation_handler() + self.store = hs.get_datastore() + self.clock = hs.get_clock() + + @staticmethod + async def _serialize_payload( # type: ignore + requester: Requester, + room_id: str, + user_id: str, + remote_room_hosts: List[str], + content: JsonDict, + ): + """ + Args: + requester: The user making the request, according to the access token. + room_id: The ID of the room to knock on. + user_id: The ID of the knocking user. + remote_room_hosts: Servers to try and send the knock via. + content: The event content to use for the knock event. + """ + return { + "requester": requester.serialize(), + "remote_room_hosts": remote_room_hosts, + "content": content, + } + + async def _handle_request( # type: ignore + self, + request: Request, + room_id: str, + user_id: str, + ): + content = parse_json_object_from_request(request) + + remote_room_hosts = content["remote_room_hosts"] + event_content = content["content"] + + requester = Requester.deserialize(self.store, content["requester"]) + + request.requester = requester + + logger.debug("remote_knock: %s on room: %s", user_id, room_id) + + event_id, stream_id = await self.federation_handler.do_knock( + remote_room_hosts, room_id, user_id, event_content + ) + + return 200, {"event_id": event_id, "stream_id": stream_id} + + class ReplicationRemoteRejectInviteRestServlet(ReplicationEndpoint): """Rejects an out-of-band invite we have received from a remote server @@ -151,6 +222,72 @@ async def _handle_request(self, request, invite_event_id): return 200, {"event_id": event_id, "stream_id": stream_id} +class ReplicationRemoteRescindKnockRestServlet(ReplicationEndpoint): + """Rescinds a local knock made on a remote room + + Request format: + + POST /_synapse/replication/remote_rescind_knock/:event_id + + { + "txn_id": ..., + "requester": ..., + "content": { ... } + } + """ + + NAME = "remote_rescind_knock" + PATH_ARGS = ("knock_event_id",) + + def __init__(self, hs: "HomeServer"): + super().__init__(hs) + + self.store = hs.get_datastore() + self.clock = hs.get_clock() + self.member_handler = hs.get_room_member_handler() + + @staticmethod + async def _serialize_payload( # type: ignore + knock_event_id: str, + txn_id: Optional[str], + requester: Requester, + content: JsonDict, + ): + """ + Args: + knock_event_id: The ID of the knock to be rescinded. + txn_id: An optional transaction ID supplied by the client. + requester: The user making the rescind request, according to the access token. + content: The content to include in the rescind event. + """ + return { + "txn_id": txn_id, + "requester": requester.serialize(), + "content": content, + } + + async def _handle_request( # type: ignore + self, + request: Request, + knock_event_id: str, + ): + content = parse_json_object_from_request(request) + + txn_id = content["txn_id"] + event_content = content["content"] + + requester = Requester.deserialize(self.store, content["requester"]) + + request.requester = requester + + # hopefully we're now on the master, so this won't recurse! + event_id, stream_id = await self.member_handler.remote_rescind_knock( + knock_event_id, txn_id, requester, event_content, + ) + + return 200, {"event_id": event_id, "stream_id": stream_id} + + class ReplicationUserJoinedLeftRoomRestServlet(ReplicationEndpoint): """Notifies that a user has joined or left the room From 4b3686243996203f35c9a14f667c8c71f8b68fd2 Mon Sep 17 00:00:00 2001 From: Andrew Morgan Date: Tue, 24 Nov 2020 15:23:31 +0000 Subject: [PATCH 026/112] Fix tense on newly_joined_or_invited_or_knocking_users --- synapse/handlers/sync.py | 26 +++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/synapse/handlers/sync.py b/synapse/handlers/sync.py index 71685cc9f6f5..45c7c4833b40 100644 --- a/synapse/handlers/sync.py +++ b/synapse/handlers/sync.py @@ -1011,7 +1011,7 @@ async def generate_sync_result( res = await self._generate_sync_entry_for_rooms( sync_result_builder, account_data_by_room ) - newly_joined_rooms, newly_joined_or_invited_or_knocking_users, _, _ = res + newly_joined_rooms, newly_joined_or_invited_or_knocked_users, _, _ = res _, _, newly_left_rooms, newly_left_users = res block_all_presence_data = ( @@ -1022,7 +1022,7 @@ async def generate_sync_result( await self._generate_sync_entry_for_presence( sync_result_builder, newly_joined_rooms, - newly_joined_or_invited_or_knocking_users, + newly_joined_or_invited_or_knocked_users, ) logger.debug("Fetching to-device data") @@ -1031,7 +1031,7 @@ async def generate_sync_result( device_lists = await self._generate_sync_entry_for_device_list( sync_result_builder, newly_joined_rooms=newly_joined_rooms, - newly_joined_or_invited_or_knocking_users=newly_joined_or_invited_or_knocking_users, + newly_joined_or_invited_or_knocked_users=newly_joined_or_invited_or_knocked_users, newly_left_rooms=newly_left_rooms, newly_left_users=newly_left_users, ) @@ -1125,7 +1125,7 @@ async def _generate_sync_entry_for_device_list( self, sync_result_builder: "SyncResultBuilder", newly_joined_rooms: Set[str], - newly_joined_or_invited_or_knocking_users: Set[str], + newly_joined_or_invited_or_knocked_users: Set[str], newly_left_rooms: Set[str], newly_left_users: Set[str], ) -> DeviceLists: @@ -1134,7 +1134,7 @@ async def _generate_sync_entry_for_device_list( Args: sync_result_builder newly_joined_rooms: Set of rooms user has joined since previous sync - newly_joined_or_invited_or_knocking_users: Set of users that have joined, + newly_joined_or_invited_or_knocked_users: Set of users that have joined, been invited to a room or are knocking on a room since previous sync. newly_left_rooms: Set of rooms user has left since previous sync @@ -1147,8 +1147,8 @@ async def _generate_sync_entry_for_device_list( # We're going to mutate these fields, so lets copy them rather than # assume they won't get used later. - newly_joined_or_invited_or_knocking_users = set( - newly_joined_or_invited_or_knocking_users + newly_joined_or_invited_or_knocked_users = set( + newly_joined_or_invited_or_knocked_users ) newly_left_users = set(newly_left_users) @@ -1188,11 +1188,11 @@ async def _generate_sync_entry_for_device_list( # Step 1b, check for newly joined rooms for room_id in newly_joined_rooms: joined_users = await self.state.get_current_users_in_room(room_id) - newly_joined_or_invited_or_knocking_users.update(joined_users) + newly_joined_or_invited_or_knocked_users.update(joined_users) # TODO: Check that these users are actually new, i.e. either they # weren't in the previous sync *or* they left and rejoined. - users_that_have_changed.update(newly_joined_or_invited_or_knocking_users) + users_that_have_changed.update(newly_joined_or_invited_or_knocked_users) user_signatures_changed = await self.store.get_users_whose_signatures_changed( user_id, since_token.device_list_key @@ -1461,7 +1461,7 @@ async def handle_room_entries(room_entry): sync_result_builder.knocked.extend(knocked) # Now we want to get any newly joined, invited or knocking users - newly_joined_or_invited_or_knocking_users = set() + newly_joined_or_invited_or_knocked_users = set() newly_left_users = set() if since_token: for joined_sync in sync_result_builder.joined: @@ -1475,7 +1475,7 @@ async def handle_room_entries(room_entry): or event.membership == Membership.INVITE or event.membership == Membership.KNOCK ): - newly_joined_or_invited_or_knocking_users.add( + newly_joined_or_invited_or_knocked_users.add( event.state_key ) else: @@ -1484,11 +1484,11 @@ async def handle_room_entries(room_entry): if prev_membership == Membership.JOIN: newly_left_users.add(event.state_key) - newly_left_users -= newly_joined_or_invited_or_knocking_users + newly_left_users -= newly_joined_or_invited_or_knocked_users return ( set(newly_joined_rooms), - newly_joined_or_invited_or_knocking_users, + newly_joined_or_invited_or_knocked_users, set(newly_left_rooms), newly_left_users, ) From ebc683ab63b8f656d3242c41605d3e1bb201f667 Mon Sep 17 00:00:00 2001 From: Andrew Morgan Date: Tue, 24 Nov 2020 15:29:40 +0000 Subject: [PATCH 027/112] Inline TransactionRestServlet --- synapse/rest/client/v2_alpha/knock.py | 11 +++-------- 1 file changed, 3 insertions(+), 8 deletions(-) diff --git a/synapse/rest/client/v2_alpha/knock.py b/synapse/rest/client/v2_alpha/knock.py index 1814683120c2..509ee8257548 100644 --- a/synapse/rest/client/v2_alpha/knock.py +++ b/synapse/rest/client/v2_alpha/knock.py @@ -32,13 +32,7 @@ logger = logging.getLogger(__name__) -class TransactionRestServlet(RestServlet): - def __init__(self, hs: "HomeServer"): - super(TransactionRestServlet, self).__init__() - self.txns = HttpTransactionCache(hs) - - -class KnockRoomAliasServlet(TransactionRestServlet): +class KnockRoomAliasServlet(RestServlet): """ POST /knock/{roomIdOrAlias} """ @@ -46,7 +40,8 @@ class KnockRoomAliasServlet(TransactionRestServlet): PATTERNS = client_patterns("/knock/(?P[^/]*)") def __init__(self, hs: "HomeServer"): - super().__init__(hs) + super().__init__() + self.txns = HttpTransactionCache(hs) self.room_member_handler = hs.get_room_member_handler() self.auth = hs.get_auth() From a63022201946075d07e4fdc44707d72f27ad9489 Mon Sep 17 00:00:00 2001 From: Andrew Morgan Date: Tue, 24 Nov 2020 15:41:52 +0000 Subject: [PATCH 028/112] lint --- synapse/handlers/room_member_worker.py | 1 - synapse/replication/http/membership.py | 9 ++------- 2 files changed, 2 insertions(+), 8 deletions(-) diff --git a/synapse/handlers/room_member_worker.py b/synapse/handlers/room_member_worker.py index 67c513b285a1..5c3094a57abe 100644 --- a/synapse/handlers/room_member_worker.py +++ b/synapse/handlers/room_member_worker.py @@ -18,7 +18,6 @@ from typing import List, Optional, Tuple from synapse.api.errors import SynapseError -from synapse.events import EventBase from synapse.handlers.room_member import RoomMemberHandler from synapse.replication.http.membership import ( ReplicationRemoteJoinRestServlet as ReplRemoteJoin, diff --git a/synapse/replication/http/membership.py b/synapse/replication/http/membership.py index 15047f4246ae..98ab24f24d7c 100644 --- a/synapse/replication/http/membership.py +++ b/synapse/replication/http/membership.py @@ -136,10 +136,7 @@ async def _serialize_payload( # type: ignore } async def _handle_request( # type: ignore - self, - request: Request, - room_id: str, - user_id: str, + self, request: Request, room_id: str, user_id: str, ): content = parse_json_object_from_request(request) @@ -267,9 +264,7 @@ async def _serialize_payload( # type: ignore } async def _handle_request( # type: ignore - self, - request: Request, - knock_event_id: str, + self, request: Request, knock_event_id: str, ): content = parse_json_object_from_request(request) From a34be45d5e7e55637adf2627d6fc22f946a31248 Mon Sep 17 00:00:00 2001 From: Andrew Morgan Date: Tue, 24 Nov 2020 17:24:28 +0000 Subject: [PATCH 029/112] Create and user parse_list_from_args, add docstring and types to parse_string_from_args --- synapse/http/servlet.py | 67 +++++++++++++++++++++++---- synapse/rest/admin/rooms.py | 8 ++-- synapse/rest/client/v1/room.py | 10 ++-- synapse/rest/client/v2_alpha/knock.py | 14 +++--- 4 files changed, 74 insertions(+), 25 deletions(-) diff --git a/synapse/http/servlet.py b/synapse/http/servlet.py index b361b7cbaf43..86f6ecba345e 100644 --- a/synapse/http/servlet.py +++ b/synapse/http/servlet.py @@ -14,8 +14,8 @@ # limitations under the License. """ This module contains base REST classes for constructing REST servlets. """ - import logging +from typing import Dict, List, Optional, Union from synapse.api.errors import Codes, SynapseError from synapse.util import json_decoder @@ -147,16 +147,67 @@ def parse_string( ) +def parse_list_from_args( + args: Dict[bytes, List[bytes]], + name: Union[bytes, str], + encoding: Optional[str] = "ascii", +): + """Parse and optionally decode a list of values from request query parameters. + + Args: + args: A dictionary of query parameters from a request. + name: The name of the query parameter to extract values from. If given as bytes, + will be decoded as "ascii". + encoding: An optional encoding that is used to decode each parameter value with. + + Raises: + KeyError: If the given `name` does not exist in `args`. + SynapseError: If an argument was not encoded with the specified `encoding`. + """ + if not isinstance(name, bytes): + name = name.encode("ascii") + args_list = args[name] + + if encoding: + # Decode each argument value + try: + args_list = [value.decode(encoding) for value in args_list] + except ValueError: + raise SynapseError(400, "Query parameter %r must be %s" % (name, encoding)) + + return args_list + + def parse_string_from_args( - args, - name, - default=None, - required=False, - allowed_values=None, - param_type="string", - encoding="ascii", + args: Dict[bytes, List[bytes]], + name: Union[bytes, str], + default: Optional[str] = None, + required: Optional[bool] = False, + allowed_values: Optional[List[bytes]] = None, + param_type: Optional[str] = "string", + encoding: Optional[str] = "ascii", ): + """Parse a single + Args: + args: A dictionary of query parameters from a request. + name: The name of the query parameter to extract values from. If given as bytes, + will be decoded as "ascii". + default: A default value to return if the given argument `name` was not found. + required: If this is True, no `default` is provided and the given argument `name` + was not found then a SynapseError is raised. + allowed_values: A list of allowed values. If specified and the found str is + not in this list, a SynapseError is raised. + param_type: The expected type of the query parameter's value. + encoding: An optional encoding that is used to decode each parameter value with. + + Returns: + The found argument value. + + Raises: + SynapseError: If the given name was not found in the request arguments, + the argument's values were encoded incorrectly or a required value was missing. + """ if not isinstance(name, bytes): name = name.encode("ascii") diff --git a/synapse/rest/admin/rooms.py b/synapse/rest/admin/rooms.py index f5304ff43dd4..4283891d4797 100644 --- a/synapse/rest/admin/rooms.py +++ b/synapse/rest/admin/rooms.py @@ -14,7 +14,6 @@ # limitations under the License. import logging from http import HTTPStatus -from typing import List, Optional from synapse.api.constants import EventTypes, JoinRules from synapse.api.errors import Codes, NotFoundError, SynapseError @@ -23,6 +22,7 @@ assert_params_in_dict, parse_integer, parse_json_object_from_request, + parse_list_from_args, parse_string, ) from synapse.rest.admin._base import ( @@ -294,10 +294,8 @@ async def on_POST(self, request, room_identifier): if RoomID.is_valid(room_identifier): room_id = room_identifier try: - remote_room_hosts = [ - x.decode("ascii") for x in request.args[b"server_name"] - ] # type: Optional[List[str]] - except Exception: + remote_room_hosts = parse_list_from_args(request.args, "server_name") + except KeyError: remote_room_hosts = None elif RoomAlias.is_valid(room_identifier): handler = self.room_member_handler diff --git a/synapse/rest/client/v1/room.py b/synapse/rest/client/v1/room.py index 25d3cc614806..c1272d756ecb 100644 --- a/synapse/rest/client/v1/room.py +++ b/synapse/rest/client/v1/room.py @@ -15,10 +15,9 @@ # limitations under the License. """ This module contains REST servlets to do with rooms: /rooms/ """ - import logging import re -from typing import List, Optional +from typing import Optional from urllib import parse as urlparse from synapse.api.constants import EventTypes, Membership @@ -37,6 +36,7 @@ assert_params_in_dict, parse_integer, parse_json_object_from_request, + parse_list_from_args, parse_string, ) from synapse.logging.opentracing import set_tag @@ -284,10 +284,8 @@ async def on_POST(self, request, room_identifier, txn_id=None): if RoomID.is_valid(room_identifier): room_id = room_identifier try: - remote_room_hosts = [ - x.decode("ascii") for x in request.args[b"server_name"] - ] # type: Optional[List[str]] - except Exception: + remote_room_hosts = parse_list_from_args(request.args, "server_name") + except KeyError: remote_room_hosts = None elif RoomAlias.is_valid(room_identifier): handler = self.room_member_handler diff --git a/synapse/rest/client/v2_alpha/knock.py b/synapse/rest/client/v2_alpha/knock.py index 509ee8257548..f1c23e0096f1 100644 --- a/synapse/rest/client/v2_alpha/knock.py +++ b/synapse/rest/client/v2_alpha/knock.py @@ -14,12 +14,16 @@ # See the License for the specific language governing permissions and # limitations under the License. import logging -from typing import TYPE_CHECKING, List, Optional, Tuple +from typing import TYPE_CHECKING, Optional, Tuple from twisted.web.server import Request from synapse.api.errors import SynapseError -from synapse.http.servlet import RestServlet, parse_json_object_from_request +from synapse.http.servlet import ( + RestServlet, + parse_json_object_from_request, + parse_list_from_args, +) from synapse.logging.opentracing import set_tag from synapse.rest.client.transactions import HttpTransactionCache from synapse.types import JsonDict, RoomAlias, RoomID @@ -58,10 +62,8 @@ async def on_POST( if RoomID.is_valid(room_identifier): room_id = room_identifier try: - remote_room_hosts = [ - x.decode("ascii") for x in request.args[b"server_name"] - ] # type: Optional[List[str]] - except Exception: + remote_room_hosts = parse_list_from_args(request.args, "server_name") + except KeyError: remote_room_hosts = None elif RoomAlias.is_valid(room_identifier): handler = self.room_member_handler From 9090465779421ffa72a886edbfb893eca8e47b31 Mon Sep 17 00:00:00 2001 From: Andrew Morgan Date: Tue, 24 Nov 2020 17:31:39 +0000 Subject: [PATCH 030/112] Remove knock unstable_features Clients can tell if knocking is available by checking for the presence of a room version containing knocking in /_matrix/client/r0/capabilities. --- synapse/rest/client/versions.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/synapse/rest/client/versions.py b/synapse/rest/client/versions.py index 7a5c739b2370..d24a199318a2 100644 --- a/synapse/rest/client/versions.py +++ b/synapse/rest/client/versions.py @@ -81,8 +81,6 @@ def on_GET(self, request): "io.element.e2ee_forced.public": self.e2ee_forced_public, "io.element.e2ee_forced.private": self.e2ee_forced_private, "io.element.e2ee_forced.trusted_private": self.e2ee_forced_trusted_private, - # Implements additional endpoints and features as described in MSC2403 - "xyz.amorgan.knock": True, }, }, ) From ad929c3bf02ae95a2d4abfb2ee4e21c875b513a6 Mon Sep 17 00:00:00 2001 From: Andrew Morgan Date: Tue, 24 Nov 2020 18:07:56 +0000 Subject: [PATCH 031/112] Optimise retrieving previous invite or knock event during membership update. It was pointed out that the method in RoomMemberHandler.update_membership_locked for retrieving the previous invite or knock event was overly complex. We already knew the ID of the room, yet got a list of all rooms that the user has been invited to/knocked on from the database. We then filtered that list, and extracted some metadata from it that allowed us to get the invite/knock event ID. This was quite ineffecient, and I've updated both methods to simply get the current state of the room given the room ID, extract the latest m.room.member event for the user, and check that it has invite/knock membership. There's still some inefficiency between pulling out the knock event, passing the event ID to remote_rescind_knock, to which remote_rescind_knock then pulls the associated event out of the database again - however I think this is necessary given that we may need to call remote_rescind_knock over replication, where passing an event ID is easier than a serialised event. It also more closely resembles remote_reject_invite's function signature. --- synapse/handlers/room_member.py | 79 +++++++++++--------- synapse/storage/databases/main/roommember.py | 33 -------- 2 files changed, 44 insertions(+), 68 deletions(-) diff --git a/synapse/handlers/room_member.py b/synapse/handlers/room_member.py index bfe6236ae6d8..3390538b5d98 100644 --- a/synapse/handlers/room_member.py +++ b/synapse/handlers/room_member.py @@ -31,7 +31,6 @@ from synapse.api.ratelimiting import Ratelimiter from synapse.events import EventBase from synapse.events.snapshot import EventContext -from synapse.storage.roommember import RoomsForUser from synapse.types import ( JsonDict, Requester, @@ -558,50 +557,60 @@ async def update_membership_locked( elif effective_membership_state == Membership.LEAVE: if not is_host_in_room: # perhaps we've been invited - invite = await self.store.get_invite_for_local_user_in_room( - user_id=target.to_string(), room_id=room_id - ) # type: Optional[RoomsForUser] - if invite: - logger.info( - "%s rejects invite to %s from %s", - target, - room_id, - invite.sender, + room_state_or_invite = await self.state_handler.get_current_state( + room_id, event_type=EventTypes.Member, state_key=target.to_string() + ) + if room_state_or_invite: + invite = room_state_or_invite.get( + (EventTypes.Member, target.to_string()) ) - - if not self.hs.is_mine_id(invite.sender): - # send the rejection to the inviter's HS (with fallback to - # local event) - return await self.remote_reject_invite( - invite.event_id, txn_id, requester, content, + if invite and invite.membership == Membership.KNOCK: + logger.info( + "%s rejects invite to %s from %s", + target, + room_id, + invite.sender, ) - # the inviter was on our server, but has now left. Carry on - # with the normal rejection codepath, which will also send the - # rejection out to any other servers we believe are still in the room. - - # thanks to overzealous cleaning up of event_forward_extremities in - # `delete_old_current_state_events`, it's possible to end up with no - # forward extremities here. If that happens, let's just hang the - # rejection off the invite event. - # - # see: https://github.com/matrix-org/synapse/issues/7139 - if len(latest_event_ids) == 0: - latest_event_ids = [invite.event_id] + if not self.hs.is_mine_id(invite.sender): + # send the rejection to the inviter's HS (with fallback to + # local event) + return await self.remote_reject_invite( + invite.event_id, txn_id, requester, content, + ) + + # the inviter was on our server, but has now left. Carry on + # with the normal rejection codepath, which will also send the + # rejection out to any other servers we believe are still in the room. + + # thanks to overzealous cleaning up of event_forward_extremities in + # `delete_old_current_state_events`, it's possible to end up with no + # forward extremities here. If that happens, let's just hang the + # rejection off the invite event. + # + # see: https://github.com/matrix-org/synapse/issues/7139 + if len(latest_event_ids) == 0: + latest_event_ids = [invite.event_id] else: # or perhaps this is a remote room that a local user has knocked on - knock = await self.store.get_knock_for_local_user_in_room( - user_id=target.to_string(), room_id=room_id - ) # type: Optional[RoomsForUser] - if knock: - return await self.remote_rescind_knock( - knock.event_id, txn_id, requester, content + room_state_or_knock = await self.state_handler.get_current_state( + room_id, + event_type=EventTypes.Member, + state_key=target.to_string(), + ) + if room_state_or_knock: + knock = room_state_or_knock.get( + (EventTypes.Member, target.to_string()) ) + if knock and knock.membership == Membership.KNOCK: + return await self.remote_rescind_knock( + knock.event_id, txn_id, requester, content + ) logger.info( "%s sent a leave request to %s, but that is not an active room " - "on this server, and there is no pending knock", + "on this server, or there is no pending knock", target, room_id, ) diff --git a/synapse/storage/databases/main/roommember.py b/synapse/storage/databases/main/roommember.py index d880c50f76f9..01d9dbb36f44 100644 --- a/synapse/storage/databases/main/roommember.py +++ b/synapse/storage/databases/main/roommember.py @@ -272,21 +272,6 @@ async def get_invited_rooms_for_local_user(self, user_id: str) -> RoomsForUser: user_id, [Membership.INVITE] ) - @cached() - async def get_knocked_rooms_for_local_user(self, user_id: str) -> RoomsForUser: - """Get all the rooms the *local* user currently has the membership of knock for. - - Args: - user_id: The user ID. - - Returns: - A list of RoomsForUser. - """ - - return await self.get_rooms_for_local_user_where_membership_is( - user_id, [Membership.KNOCK] - ) - async def get_invite_for_local_user_in_room( self, user_id: str, room_id: str ) -> Optional[RoomsForUser]: @@ -305,24 +290,6 @@ async def get_invite_for_local_user_in_room( return invite return None - async def get_knock_for_local_user_in_room( - self, user_id: str, room_id: str - ) -> Optional[RoomsForUser]: - """Gets the knock for the given *local* user and room. - - Args: - user_id: The user ID to find the knock of. - room_id: The room to user knocked on. - - Returns: - Either a RoomsForUser or None if no knock was found. - """ - knocks = await self.get_knocked_rooms_for_local_user(user_id) - for knock in knocks: - if knock.room_id == room_id: - return knock - return None - async def get_rooms_for_local_user_where_membership_is( self, user_id: str, membership_list: Collection[str] ) -> List[RoomsForUser]: From 42ecc285db9ded49c40ee1a73dda932f2504de5a Mon Sep 17 00:00:00 2001 From: Andrew Morgan Date: Tue, 24 Nov 2020 18:30:34 +0000 Subject: [PATCH 032/112] Modify knock join_rules and membership states to use unstable prefix --- synapse/api/constants.py | 4 ++-- synapse/handlers/federation.py | 2 +- synapse/rest/client/v2_alpha/knock.py | 3 ++- synapse/rest/client/v2_alpha/sync.py | 16 ++++++++-------- 4 files changed, 13 insertions(+), 12 deletions(-) diff --git a/synapse/api/constants.py b/synapse/api/constants.py index 592abd844b8a..05fc84f6e12a 100644 --- a/synapse/api/constants.py +++ b/synapse/api/constants.py @@ -34,7 +34,7 @@ class Membership: INVITE = "invite" JOIN = "join" - KNOCK = "knock" + KNOCK = "xyz.amorgan.knock" LEAVE = "leave" BAN = "ban" LIST = (INVITE, JOIN, KNOCK, LEAVE, BAN) @@ -50,7 +50,7 @@ class PresenceState: class JoinRules: PUBLIC = "public" - KNOCK = "knock" + KNOCK = "xyz.amorgan.knock" INVITE = "invite" PRIVATE = "private" diff --git a/synapse/handlers/federation.py b/synapse/handlers/federation.py index 377180886f56..56afdf9383b0 100644 --- a/synapse/handlers/federation.py +++ b/synapse/handlers/federation.py @@ -1469,7 +1469,7 @@ async def do_knock( # Ask the remote server to create a valid knock event for us. Once received, # we sign the event origin, event, event_format_version = await self._make_and_verify_event( - target_hosts, room_id, knockee, "knock", content, + target_hosts, room_id, knockee, Membership.KNOCK, content, ) # Record the room ID and its version so that we have a record of the room diff --git a/synapse/rest/client/v2_alpha/knock.py b/synapse/rest/client/v2_alpha/knock.py index f1c23e0096f1..818236935ff5 100644 --- a/synapse/rest/client/v2_alpha/knock.py +++ b/synapse/rest/client/v2_alpha/knock.py @@ -18,6 +18,7 @@ from twisted.web.server import Request +from synapse.api.constants import Membership from synapse.api.errors import SynapseError from synapse.http.servlet import ( RestServlet, @@ -79,7 +80,7 @@ async def on_POST( requester=requester, target=requester.user, room_id=room_id, - action="knock", + action=Membership.KNOCK, txn_id=txn_id, third_party_signed=None, remote_room_hosts=remote_room_hosts, diff --git a/synapse/rest/client/v2_alpha/sync.py b/synapse/rest/client/v2_alpha/sync.py index 09da07152734..19f01ba02440 100644 --- a/synapse/rest/client/v2_alpha/sync.py +++ b/synapse/rest/client/v2_alpha/sync.py @@ -17,7 +17,7 @@ import logging from typing import Any, Callable, Dict, List -from synapse.api.constants import PresenceState +from synapse.api.constants import PresenceState, Membership from synapse.api.errors import Codes, StoreError, SynapseError from synapse.api.filtering import DEFAULT_FILTER_COLLECTION, FilterCollection from synapse.events.utils import ( @@ -235,15 +235,15 @@ async def encode_response(self, time_now, sync_result, access_token_id, filter): }, "presence": SyncRestServlet.encode_presence(sync_result.presence, time_now), "rooms": { - "join": joined, - "invite": invited, - "knock": knocked, - "leave": archived, + Membership.JOIN: joined, + Membership.INVITE: invited, + Membership.KNOCK: knocked, + Membership.LEAVE: archived, }, "groups": { - "join": sync_result.groups.join, - "invite": sync_result.groups.invite, - "leave": sync_result.groups.leave, + Membership.JOIN: sync_result.groups.join, + Membership.INVITE: sync_result.groups.invite, + Membership.LEAVE: sync_result.groups.leave, }, "device_one_time_keys_count": sync_result.device_one_time_keys_count, "org.matrix.msc2732.device_unused_fallback_key_types": sync_result.device_unused_fallback_key_types, From 7f65f01a9b59ce3e3586dc6ccbbd1c18d1b7ae37 Mon Sep 17 00:00:00 2001 From: Andrew Morgan Date: Tue, 24 Nov 2020 18:33:58 +0000 Subject: [PATCH 033/112] Update endpoint to use unstable prefix I assume the same does not need to happen to replication endpoints as they are not client or federation facing. --- synapse/rest/client/v2_alpha/knock.py | 4 ++-- synapse/rest/client/v2_alpha/sync.py | 3 +-- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/synapse/rest/client/v2_alpha/knock.py b/synapse/rest/client/v2_alpha/knock.py index 818236935ff5..9045ca97a7d0 100644 --- a/synapse/rest/client/v2_alpha/knock.py +++ b/synapse/rest/client/v2_alpha/knock.py @@ -39,10 +39,10 @@ class KnockRoomAliasServlet(RestServlet): """ - POST /knock/{roomIdOrAlias} + POST /xyz.amorgan.knock/{roomIdOrAlias} """ - PATTERNS = client_patterns("/knock/(?P[^/]*)") + PATTERNS = client_patterns("/xyz.amorgan.knock/(?P[^/]*)") def __init__(self, hs: "HomeServer"): super().__init__() diff --git a/synapse/rest/client/v2_alpha/sync.py b/synapse/rest/client/v2_alpha/sync.py index 19f01ba02440..fe21d8f9b2af 100644 --- a/synapse/rest/client/v2_alpha/sync.py +++ b/synapse/rest/client/v2_alpha/sync.py @@ -12,12 +12,11 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. - import itertools import logging from typing import Any, Callable, Dict, List -from synapse.api.constants import PresenceState, Membership +from synapse.api.constants import Membership, PresenceState from synapse.api.errors import Codes, StoreError, SynapseError from synapse.api.filtering import DEFAULT_FILTER_COLLECTION, FilterCollection from synapse.events.utils import ( From a49cd83ba9c92001a8da90ed835b1320740aa564 Mon Sep 17 00:00:00 2001 From: Andrew Morgan <1342360+anoadragon453@users.noreply.github.com> Date: Tue, 24 Nov 2020 18:42:25 +0000 Subject: [PATCH 034/112] Update synapse/http/servlet.py Co-authored-by: Patrick Cloke --- synapse/http/servlet.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/synapse/http/servlet.py b/synapse/http/servlet.py index 86f6ecba345e..9bfe151b5f5d 100644 --- a/synapse/http/servlet.py +++ b/synapse/http/servlet.py @@ -187,7 +187,7 @@ def parse_string_from_args( param_type: Optional[str] = "string", encoding: Optional[str] = "ascii", ): - """Parse a single + """Parse and optionally decode a single value from request query parameters. Args: args: A dictionary of query parameters from a request. From c097c52a160e364ba252805a90c08bb0aba0eaf8 Mon Sep 17 00:00:00 2001 From: Andrew Morgan <1342360+anoadragon453@users.noreply.github.com> Date: Tue, 24 Nov 2020 20:29:10 +0000 Subject: [PATCH 035/112] Apply suggestions from code review Co-authored-by: Patrick Cloke --- synapse/event_auth.py | 2 +- synapse/events/utils.py | 8 ++++---- synapse/federation/transport/server.py | 2 +- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/synapse/event_auth.py b/synapse/event_auth.py index 5edd610d0922..4629515a59b8 100644 --- a/synapse/event_auth.py +++ b/synapse/event_auth.py @@ -291,7 +291,7 @@ def _is_membership_change_allowed( raise AuthError(403, "%s is banned from the room" % (target_user_id,)) return - # Require the user to be in the room for membership changes other than join/knocking + # Require the user to be in the room for membership changes other than join/knock. if Membership.JOIN != membership and Membership.KNOCK != membership: # If the user has been invited or has knocked, they are allowed to change their # membership event to leave diff --git a/synapse/events/utils.py b/synapse/events/utils.py index 7d95ddda5170..72c8ad4fe584 100644 --- a/synapse/events/utils.py +++ b/synapse/events/utils.py @@ -311,10 +311,10 @@ def serialize_event( if txn_id is not None: d["unsigned"]["transaction_id"] = txn_id - # invite_room_state is a list of stripped room state events that are meant to - # provide metadata about a room to an invitee. knock_room_state is the same, - # but for user knocking on a room. They are intended to only be included in specific - # circumstances, such as down sync, and should not be included in any other case. + # invite_room_state and knock_room_state are a list of stripped room state events + # that are meant to provide metadata about a room to an invitee/knocker. They are + # intended to only be included in specific circumstances, such as down sync, and + # should not be included in any other case. if not include_stripped_room_state: d["unsigned"].pop("invite_room_state", None) d["unsigned"].pop("knock_room_state", None) diff --git a/synapse/federation/transport/server.py b/synapse/federation/transport/server.py index 4f1a0baeb4e1..2d5aba151021 100644 --- a/synapse/federation/transport/server.py +++ b/synapse/federation/transport/server.py @@ -551,7 +551,7 @@ async def on_GET(self, origin, content, query, room_id, user_id): return 200, content -class FederationV1MakeKnockServlet(BaseFederationServlet): +class FederationV2MakeKnockServlet(BaseFederationServlet): PATH = "/send_knock/(?P[^/]*)/(?P[^/]*)" async def on_PUT(self, origin, content, query, room_id, event_id): From ab934b40e3f3491f061cd1e0a994d68b24cbb315 Mon Sep 17 00:00:00 2001 From: Andrew Morgan Date: Tue, 24 Nov 2020 20:57:06 +0000 Subject: [PATCH 036/112] Fix FederationV2SendKnockServlet naming --- synapse/federation/transport/server.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/synapse/federation/transport/server.py b/synapse/federation/transport/server.py index 5454e667c55c..cb45af48c292 100644 --- a/synapse/federation/transport/server.py +++ b/synapse/federation/transport/server.py @@ -551,7 +551,7 @@ async def on_GET(self, origin, content, query, room_id, user_id): return 200, content -class FederationV2MakeKnockServlet(BaseFederationServlet): +class FederationV2SendKnockServlet(BaseFederationServlet): PATH = "/send_knock/(?P[^/]*)/(?P[^/]*)" async def on_PUT(self, origin, content, query, room_id, event_id): @@ -1410,7 +1410,7 @@ async def on_GET(self, origin, content, query, room_id): FederationV2SendJoinServlet, FederationV1SendLeaveServlet, FederationV2SendLeaveServlet, - FederationV1MakeKnockServlet, + FederationV2SendKnockServlet, FederationV1InviteServlet, FederationV2InviteServlet, FederationGetMissingEventsServlet, From 16da4251cfa7959e13e9047ef2abbdfe66388362 Mon Sep 17 00:00:00 2001 From: Andrew Morgan Date: Tue, 24 Nov 2020 21:05:08 +0000 Subject: [PATCH 037/112] Remove config option room_knock_state_types We introduced this option in this PR to mirror room_knock_invite_types, but ended up questioning the worth of both, and ultimately decided that neither were very useful. Context: https://github.com/matrix-org/synapse/pull/6739/files#r527027645 and https://github.com/matrix-org/synapse/issues/8807 --- docs/sample_config.yaml | 11 ----------- synapse/config/api.py | 14 -------------- synapse/federation/federation_server.py | 5 ++--- 3 files changed, 2 insertions(+), 28 deletions(-) diff --git a/docs/sample_config.yaml b/docs/sample_config.yaml index 8773967dfd31..66f6b53f06f3 100644 --- a/docs/sample_config.yaml +++ b/docs/sample_config.yaml @@ -1398,17 +1398,6 @@ metrics_flags: # - "m.room.encryption" # - "m.room.name" -# A list of event types from a room that will be given to users when they -# knock on the room. This allows clients to display information about the -# room that they've knocked on, without actually being in the room yet. -# -#room_knock_state_types: -# - "m.room.join_rules" -# - "m.room.canonical_alias" -# - "m.room.avatar" -# - "m.room.encryption" -# - "m.room.name" - # A list of application service config files to use # diff --git a/synapse/config/api.py b/synapse/config/api.py index a160680d5175..0638ed8d2ecc 100644 --- a/synapse/config/api.py +++ b/synapse/config/api.py @@ -34,9 +34,6 @@ def read_config(self, config, **kwargs): self.room_invite_state_types = config.get( "room_invite_state_types", DEFAULT_ROOM_STATE_TYPES ) - self.room_knock_state_types = config.get( - "room_knock_state_types", DEFAULT_ROOM_STATE_TYPES - ) def generate_config_section(cls, **kwargs): return """\ @@ -52,17 +49,6 @@ def generate_config_section(cls, **kwargs): # - "{RoomAvatar}" # - "{RoomEncryption}" # - "{Name}" - - # A list of event types from a room that will be given to users when they - # knock on the room. This allows clients to display information about the - # room that they've knocked on, without actually being in the room yet. - # - #room_knock_state_types: - # - "{JoinRules}" - # - "{CanonicalAlias}" - # - "{RoomAvatar}" - # - "{RoomEncryption}" - # - "{Name}" """.format( **vars(EventTypes) ) diff --git a/synapse/federation/federation_server.py b/synapse/federation/federation_server.py index 47f84fd8255b..b350e82aa2ea 100644 --- a/synapse/federation/federation_server.py +++ b/synapse/federation/federation_server.py @@ -43,6 +43,7 @@ SynapseError, UnsupportedRoomVersionError, ) +from synapse.config.api import DEFAULT_ROOM_STATE_TYPES from synapse.api.room_versions import KNOWN_ROOM_VERSIONS from synapse.events import EventBase from synapse.federation.federation_base import FederationBase, event_from_pdu_json @@ -119,8 +120,6 @@ def __init__(self, hs): hs, "fed_txn_handler", timeout_ms=30000 ) # type: ResponseCache[Tuple[str, str]] - self._room_knock_state_types = hs.config.room_knock_state_types - self.transaction_actions = TransactionActions(self.store) self.registry = hs.get_federation_registry() @@ -628,7 +627,7 @@ async def on_send_knock_request( # server. This will allow the remote server's clients to display information # related to the room while the knock request is pending. stripped_room_state = await self.store.get_stripped_room_state_from_event_context( - event_context, self._room_knock_state_types + event_context, DEFAULT_ROOM_STATE_TYPES ) return {"knock_state_events": stripped_room_state} From acdfd4bab20b9f782f294848369d1bafc379dd63 Mon Sep 17 00:00:00 2001 From: Andrew Morgan Date: Tue, 24 Nov 2020 21:09:50 +0000 Subject: [PATCH 038/112] Update send_knock federation endpoint to v2 --- synapse/federation/federation_client.py | 2 +- synapse/federation/federation_server.py | 2 +- synapse/federation/transport/client.py | 4 ++-- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/synapse/federation/federation_client.py b/synapse/federation/federation_client.py index b32b548500b8..7076652b2bbb 100644 --- a/synapse/federation/federation_client.py +++ b/synapse/federation/federation_client.py @@ -930,7 +930,7 @@ async def _do_send_knock(self, destination: str, pdu: EventBase) -> JsonDict: """ time_now = self._clock.time_msec() - return await self.transport_layer.send_knock_v1( + return await self.transport_layer.send_knock_v2( destination=destination, room_id=pdu.room_id, event_id=pdu.event_id, diff --git a/synapse/federation/federation_server.py b/synapse/federation/federation_server.py index b350e82aa2ea..a422ab2241db 100644 --- a/synapse/federation/federation_server.py +++ b/synapse/federation/federation_server.py @@ -43,8 +43,8 @@ SynapseError, UnsupportedRoomVersionError, ) -from synapse.config.api import DEFAULT_ROOM_STATE_TYPES from synapse.api.room_versions import KNOWN_ROOM_VERSIONS +from synapse.config.api import DEFAULT_ROOM_STATE_TYPES from synapse.events import EventBase from synapse.federation.federation_base import FederationBase, event_from_pdu_json from synapse.federation.persistence import TransactionActions diff --git a/synapse/federation/transport/client.py b/synapse/federation/transport/client.py index 83ce560e7773..6a85450ed687 100644 --- a/synapse/federation/transport/client.py +++ b/synapse/federation/transport/client.py @@ -297,7 +297,7 @@ async def send_leave_v2(self, destination, room_id, event_id, content): return response @log_function - async def send_knock_v1( + async def send_knock_v2( self, destination: str, room_id: str, event_id: str, content: JsonDict, ) -> JsonDict: """ @@ -320,7 +320,7 @@ async def send_knock_v1( The list of state events may be empty. """ - path = _create_v1_path("/send_knock/%s/%s", room_id, event_id) + path = _create_v2_path("/send_knock/%s/%s", room_id, event_id) return await self.client.put_json( destination=destination, path=path, data=content From afe242cc30b16ba7fb10224ead34a4489f97515e Mon Sep 17 00:00:00 2001 From: Andrew Morgan Date: Tue, 24 Nov 2020 21:29:46 +0000 Subject: [PATCH 039/112] Remove event = pdu silliness --- synapse/handlers/federation.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/synapse/handlers/federation.py b/synapse/handlers/federation.py index 0fe45029b6dd..4c1d6ab3350f 100644 --- a/synapse/handlers/federation.py +++ b/synapse/handlers/federation.py @@ -1901,20 +1901,20 @@ async def on_make_knock_request( return event @log_function - async def on_send_knock_request(self, origin: str, pdu: EventBase) -> EventContext: + async def on_send_knock_request( + self, origin: str, event: EventBase + ) -> EventContext: """ We have received a knock event for a room. Verify that event and send it into the room on the knocking homeserver's behalf. Args: origin: The remote homeserver of the knocking user. - pdu: The knocking member event that has been signed by the remote homeserver. + event: The knocking member event that has been signed by the remote homeserver. Returns: The context of the event after inserting it into the room graph. """ - event = pdu - logger.debug( "on_send_knock_request: Got event: %s, signatures: %s", event.event_id, From 7f3c188aac83c00d2bd9ec4ff43607da9f3d2537 Mon Sep 17 00:00:00 2001 From: Andrew Morgan Date: Tue, 24 Nov 2020 21:31:39 +0000 Subject: [PATCH 040/112] Correct invite.membership == Membership.KNOCK typo --- synapse/handlers/room_member.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/synapse/handlers/room_member.py b/synapse/handlers/room_member.py index 4095cc378b64..e7721d14f0b8 100644 --- a/synapse/handlers/room_member.py +++ b/synapse/handlers/room_member.py @@ -564,7 +564,7 @@ async def update_membership_locked( invite = room_state_or_invite.get( (EventTypes.Member, target.to_string()) ) - if invite and invite.membership == Membership.KNOCK: + if invite and invite.membership == Membership.INVITE: logger.info( "%s rejects invite to %s from %s", target, From 5d3365e2f83bd9cd967dbae908357f1164983225 Mon Sep 17 00:00:00 2001 From: Andrew Morgan Date: Tue, 24 Nov 2020 21:35:10 +0000 Subject: [PATCH 041/112] Fix feturn value of RoomMemberWorkerHandler.remote_knock --- synapse/handlers/room_member_worker.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/synapse/handlers/room_member_worker.py b/synapse/handlers/room_member_worker.py index 5c3094a57abe..3de63e885e37 100644 --- a/synapse/handlers/room_member_worker.py +++ b/synapse/handlers/room_member_worker.py @@ -119,12 +119,13 @@ async def remote_knock( Implements RoomMemberHandler.remote_knock """ - return await self._remote_knock_client( + ret = await self._remote_knock_client( remote_room_hosts=remote_room_hosts, room_id=room_id, user=user, content=content, ) + return ret["event_id"], ret["stream_id"] async def _user_left_room(self, target: UserID, room_id: str) -> None: """Implements RoomMemberHandler._user_left_room From 72d16b0def84f69b9b2edd852946565beda32940 Mon Sep 17 00:00:00 2001 From: Andrew Morgan Date: Tue, 24 Nov 2020 21:38:05 +0000 Subject: [PATCH 042/112] Add Matrix.org Foundation copyright to synapse/handlers/stats.py This file had a minor edit to update the wording. --- synapse/handlers/stats.py | 1 + 1 file changed, 1 insertion(+) diff --git a/synapse/handlers/stats.py b/synapse/handlers/stats.py index 3961f727aa50..8c390c9f4d18 100644 --- a/synapse/handlers/stats.py +++ b/synapse/handlers/stats.py @@ -1,6 +1,7 @@ # -*- coding: utf-8 -*- # Copyright 2018 New Vector Ltd # Copyright 2020 Sorunome +# Copyright 2020 The Matrix.org Foundation C.I.C. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. From 3fb055c92adbf12e5c5a0388c1d7e934b5b89e33 Mon Sep 17 00:00:00 2001 From: Andrew Morgan Date: Tue, 24 Nov 2020 21:45:48 +0000 Subject: [PATCH 043/112] Remove unnecessary arguments from knocking _serialize_payload functions --- synapse/replication/http/membership.py | 14 ++------------ 1 file changed, 2 insertions(+), 12 deletions(-) diff --git a/synapse/replication/http/membership.py b/synapse/replication/http/membership.py index 98ab24f24d7c..7631a7f280b6 100644 --- a/synapse/replication/http/membership.py +++ b/synapse/replication/http/membership.py @@ -115,17 +115,11 @@ def __init__(self, hs): @staticmethod async def _serialize_payload( # type: ignore - requester: Requester, - room_id: str, - user_id: str, - remote_room_hosts: List[str], - content: JsonDict, + requester: Requester, remote_room_hosts: List[str], content: JsonDict, ): """ Args: requester: The user making the request, according to the access token. - room_id: The ID of the room to knock on. - user_id: The ID of the knocking user. remote_room_hosts: Servers to try and send the knock via. content: The event content to use for the knock event. """ @@ -245,14 +239,10 @@ def __init__(self, hs: "HomeServer"): @staticmethod async def _serialize_payload( # type: ignore - knock_event_id: str, - txn_id: Optional[str], - requester: Requester, - content: JsonDict, + txn_id: Optional[str], requester: Requester, content: JsonDict, ): """ Args: - knock_event_id: The ID of the knock to be rescinded. txn_id: An optional transaction ID supplied by the client. requester: The user making the rescind request, according to the access token. content: The content to include in the rescind event. From 13b8e4af8af165ae33f35a70a4cc38eeb877102b Mon Sep 17 00:00:00 2001 From: Andrew Morgan Date: Wed, 25 Nov 2020 11:51:45 +0000 Subject: [PATCH 044/112] Use unstable prefix on fed endpoints, fix send_knock v2 path --- synapse/federation/federation_client.py | 2 +- synapse/federation/federation_server.py | 4 ++-- synapse/federation/transport/client.py | 4 ++-- synapse/federation/transport/server.py | 6 ++++-- synapse/handlers/federation.py | 13 +++++++------ 5 files changed, 16 insertions(+), 13 deletions(-) diff --git a/synapse/federation/federation_client.py b/synapse/federation/federation_client.py index 7076652b2bbb..fbd1bd2153f7 100644 --- a/synapse/federation/federation_client.py +++ b/synapse/federation/federation_client.py @@ -910,7 +910,7 @@ async def send_request(destination: str) -> JsonDict: return await self._do_send_knock(destination, pdu) return await self._try_destination_list( - "send_knock", destinations, send_request + "send_xyz.amorgan.knock", destinations, send_request ) async def _do_send_knock(self, destination: str, pdu: EventBase) -> JsonDict: diff --git a/synapse/federation/federation_server.py b/synapse/federation/federation_server.py index a422ab2241db..6006b3860762 100644 --- a/synapse/federation/federation_server.py +++ b/synapse/federation/federation_server.py @@ -570,8 +570,8 @@ async def on_send_leave_request(self, origin: str, content: JsonDict) -> dict: async def on_make_knock_request( self, origin: str, room_id: str, user_id: str, ) -> Dict[str, Union[EventBase, str]]: - """We've received a /make_knock/ request, so we create a partial knock event - for the room and hand that back, along with the room version, to the knocking + """We've received a /make_xyz.amorgan.knock/ request, so we create a partial knock + event for the room and hand that back, along with the room version, to the knocking homeserver. We do *not* persist or process this event until the other server has signed it and sent it back. diff --git a/synapse/federation/transport/client.py b/synapse/federation/transport/client.py index 6a85450ed687..9efcfe9a4b9b 100644 --- a/synapse/federation/transport/client.py +++ b/synapse/federation/transport/client.py @@ -302,7 +302,7 @@ async def send_knock_v2( ) -> JsonDict: """ Sends a signed knock membership event to a remote server. This is the second - step for knocking after make_knock. + step for knocking after /make_xyz.amorgan.knock. Args: destination: The remote homeserver. @@ -320,7 +320,7 @@ async def send_knock_v2( The list of state events may be empty. """ - path = _create_v2_path("/send_knock/%s/%s", room_id, event_id) + path = _create_v2_path("/send_xyz.amorgan.knock/%s/%s", room_id, event_id) return await self.client.put_json( destination=destination, path=path, data=content diff --git a/synapse/federation/transport/server.py b/synapse/federation/transport/server.py index cb45af48c292..dacea9014890 100644 --- a/synapse/federation/transport/server.py +++ b/synapse/federation/transport/server.py @@ -544,7 +544,7 @@ async def on_PUT(self, origin, content, query, room_id, event_id): class FederationMakeKnockServlet(BaseFederationServlet): - PATH = "/make_knock/(?P[^/]*)/(?P[^/]*)" + PATH = "/make_xyz.amorgan.knock/(?P[^/]*)/(?P[^/]*)" async def on_GET(self, origin, content, query, room_id, user_id): content = await self.handler.on_make_knock_request(origin, room_id, user_id) @@ -552,7 +552,9 @@ async def on_GET(self, origin, content, query, room_id, user_id): class FederationV2SendKnockServlet(BaseFederationServlet): - PATH = "/send_knock/(?P[^/]*)/(?P[^/]*)" + PATH = "/send_xyz.amorgan.knock/(?P[^/]*)/(?P[^/]*)" + + PREFIX = FEDERATION_V2_PREFIX async def on_PUT(self, origin, content, query, room_id, event_id): content = await self.handler.on_send_knock_request(origin, content, room_id) diff --git a/synapse/handlers/federation.py b/synapse/handlers/federation.py index 4c1d6ab3350f..2982c9fbe8e3 100644 --- a/synapse/handlers/federation.py +++ b/synapse/handlers/federation.py @@ -1446,9 +1446,9 @@ async def do_knock( ) -> Tuple[str, int]: """Sends the knock to the remote server. - This first triggers a /make_knock request that returns a partial + This first triggers a /make_xyz.amorgan.knock request that returns a partial event that we can fill out and sign. This is then sent to the - remote server via /send_knock. + remote server via /send_xyz.amorgan.knock. Knock events must be signed by the knockee's server before distributing. @@ -1478,7 +1478,7 @@ async def do_knock( room_id=event.room_id, room_version=event_format_version ) - # Initially try the host that we successfully called /make_knock on + # Initially try the host that we successfully called /make_xyz.amorgan.knock on try: target_hosts.remove(origin) target_hosts.insert(0, origin) @@ -1843,7 +1843,7 @@ async def on_send_leave_request(self, origin, pdu): async def on_make_knock_request( self, origin: str, room_id: str, user_id: str ) -> EventBase: - """We've received a /make_knock/ request, so we create a partial + """We've received a /make_xyz.amorgan.knock/ request, so we create a partial knock event for the room and return that. We do *not* persist or process it until the other server has signed it and sent it back. @@ -1857,7 +1857,8 @@ async def on_make_knock_request( """ if get_domain_from_id(user_id) != origin: logger.info( - "Get /make_knock request for user %r from different origin %s, ignoring", + "Get /make_xyz.amorgan.knock request for user %r" + "from different origin %s, ignoring", user_id, origin, ) @@ -1923,7 +1924,7 @@ async def on_send_knock_request( if get_domain_from_id(event.sender) != origin: logger.info( - "Got /send_knock request for user %r from different origin %s", + "Got /send_xyz.amorgan.knock request for user %r from different origin %s", event.sender, origin, ) From 8e43e63755ff545e6164c01a1c26c593995cba68 Mon Sep 17 00:00:00 2001 From: Andrew Morgan Date: Wed, 25 Nov 2020 19:48:37 +0000 Subject: [PATCH 045/112] Change CS and Federation endpoints to use unstable prefixes --- synapse/federation/transport/client.py | 16 ++++++++++++++-- synapse/federation/transport/server.py | 4 +++- synapse/rest/client/v2_alpha/knock.py | 4 +++- 3 files changed, 20 insertions(+), 4 deletions(-) diff --git a/synapse/federation/transport/client.py b/synapse/federation/transport/client.py index 9efcfe9a4b9b..8aab18c96f64 100644 --- a/synapse/federation/transport/client.py +++ b/synapse/federation/transport/client.py @@ -218,7 +218,14 @@ async def make_membership_event( "make_membership_event called with membership='%s', must be one of %s" % (membership, ",".join(valid_memberships)) ) - path = _create_v1_path("/make_%s/%s/%s", membership, room_id, user_id) + + # Knock currently uses an unstable prefix + if membership == Membership.KNOCK: + prefix = FEDERATION_UNSTABLE_PREFIX + else: + prefix = FEDERATION_V1_PREFIX + + path = _create_path(prefix, "/make_%s/%s/%s", membership, room_id, user_id) ignore_backoff = False retry_on_dns_fail = False @@ -320,7 +327,12 @@ async def send_knock_v2( The list of state events may be empty. """ - path = _create_v2_path("/send_xyz.amorgan.knock/%s/%s", room_id, event_id) + path = _create_path( + FEDERATION_UNSTABLE_PREFIX, + "/send_xyz.amorgan.knock/%s/%s", + room_id, + event_id, + ) return await self.client.put_json( destination=destination, path=path, data=content diff --git a/synapse/federation/transport/server.py b/synapse/federation/transport/server.py index dacea9014890..a55225eaa83b 100644 --- a/synapse/federation/transport/server.py +++ b/synapse/federation/transport/server.py @@ -546,6 +546,8 @@ async def on_PUT(self, origin, content, query, room_id, event_id): class FederationMakeKnockServlet(BaseFederationServlet): PATH = "/make_xyz.amorgan.knock/(?P[^/]*)/(?P[^/]*)" + PREFIX = FEDERATION_UNSTABLE_PREFIX + async def on_GET(self, origin, content, query, room_id, user_id): content = await self.handler.on_make_knock_request(origin, room_id, user_id) return 200, content @@ -554,7 +556,7 @@ async def on_GET(self, origin, content, query, room_id, user_id): class FederationV2SendKnockServlet(BaseFederationServlet): PATH = "/send_xyz.amorgan.knock/(?P[^/]*)/(?P[^/]*)" - PREFIX = FEDERATION_V2_PREFIX + PREFIX = FEDERATION_UNSTABLE_PREFIX async def on_PUT(self, origin, content, query, room_id, event_id): content = await self.handler.on_send_knock_request(origin, content, room_id) diff --git a/synapse/rest/client/v2_alpha/knock.py b/synapse/rest/client/v2_alpha/knock.py index 9045ca97a7d0..8439da447e2c 100644 --- a/synapse/rest/client/v2_alpha/knock.py +++ b/synapse/rest/client/v2_alpha/knock.py @@ -42,7 +42,9 @@ class KnockRoomAliasServlet(RestServlet): POST /xyz.amorgan.knock/{roomIdOrAlias} """ - PATTERNS = client_patterns("/xyz.amorgan.knock/(?P[^/]*)") + PATTERNS = client_patterns( + "/xyz.amorgan.knock/(?P[^/]*)", releases=() + ) def __init__(self, hs: "HomeServer"): super().__init__() From 86048613d203ac9c8cf0a1859513614c48d29580 Mon Sep 17 00:00:00 2001 From: Andrew Morgan <1342360+anoadragon453@users.noreply.github.com> Date: Wed, 25 Nov 2020 20:06:13 +0000 Subject: [PATCH 046/112] Speed up remote invite rejection database call (#8815) This is another PR that grew out of #6739. The existing code for checking whether a user is currently invited to a room when they want to leave the room looks like the following: https://github.com/matrix-org/synapse/blob/f737368a26bb9eea401fcc3a5bdd7e0b59e91f09/synapse/handlers/room_member.py#L518-L540 It calls `get_invite_for_local_user_in_room`, which will actually query *all* rooms the user has been invited to, before iterating over them and matching via the room ID. It will then return a tuple of a lot of information which we pull the event ID out of. I need to do a similar check for knocking, but this code wasn't very efficient. I then tried to write a different implementation using `StateHandler.get_current_state` but this actually didn't work as we haven't *joined* the room yet - we've only been invited to it. That means that only certain tables in Synapse have our desired `invite` membership state. One of those tables is `local_current_membership`. So I wrote a store method that just queries that table instead --- changelog.d/8815.misc | 1 + synapse/handlers/room_member.py | 63 ++++++++++---------- synapse/storage/databases/main/roommember.py | 34 ++++++++++- 3 files changed, 67 insertions(+), 31 deletions(-) create mode 100644 changelog.d/8815.misc diff --git a/changelog.d/8815.misc b/changelog.d/8815.misc new file mode 100644 index 000000000000..647edeb56851 --- /dev/null +++ b/changelog.d/8815.misc @@ -0,0 +1 @@ +Optimise the lookup for an invite from another homeserver when trying to reject it. \ No newline at end of file diff --git a/synapse/handlers/room_member.py b/synapse/handlers/room_member.py index e7721d14f0b8..3c0a0919c32a 100644 --- a/synapse/handlers/room_member.py +++ b/synapse/handlers/room_member.py @@ -557,40 +557,43 @@ async def update_membership_locked( elif effective_membership_state == Membership.LEAVE: if not is_host_in_room: # perhaps we've been invited - room_state_or_invite = await self.state_handler.get_current_state( - room_id, event_type=EventTypes.Member, state_key=target.to_string() + ( + current_membership_type, + current_membership_event_id, + ) = await self.store.get_local_current_membership_for_user_in_room( + target.to_string(), room_id ) - if room_state_or_invite: - invite = room_state_or_invite.get( - (EventTypes.Member, target.to_string()) + if ( + current_membership_type == Membership.INVITE + and current_membership_event_id + ): + invite = await self.store.get_event(current_membership_event_id) + logger.info( + "%s rejects invite to %s from %s", + target, + room_id, + invite.sender, ) - if invite and invite.membership == Membership.INVITE: - logger.info( - "%s rejects invite to %s from %s", - target, - room_id, - invite.sender, - ) - if not self.hs.is_mine_id(invite.sender): - # send the rejection to the inviter's HS (with fallback to - # local event) - return await self.remote_reject_invite( - invite.event_id, txn_id, requester, content, - ) + if not self.hs.is_mine_id(invite.sender): + # send the rejection to the inviter's HS (with fallback to + # local event) + return await self.remote_reject_invite( + invite.event_id, txn_id, requester, content, + ) - # the inviter was on our server, but has now left. Carry on - # with the normal rejection codepath, which will also send the - # rejection out to any other servers we believe are still in the room. - - # thanks to overzealous cleaning up of event_forward_extremities in - # `delete_old_current_state_events`, it's possible to end up with no - # forward extremities here. If that happens, let's just hang the - # rejection off the invite event. - # - # see: https://github.com/matrix-org/synapse/issues/7139 - if len(latest_event_ids) == 0: - latest_event_ids = [invite.event_id] + # the inviter was on our server, but has now left. Carry on + # with the normal rejection codepath, which will also send the + # rejection out to any other servers we believe are still in the room. + + # thanks to overzealous cleaning up of event_forward_extremities in + # `delete_old_current_state_events`, it's possible to end up with no + # forward extremities here. If that happens, let's just hang the + # rejection off the invite event. + # + # see: https://github.com/matrix-org/synapse/issues/7139 + if len(latest_event_ids) == 0: + latest_event_ids = [invite.event_id] else: # or perhaps this is a remote room that a local user has knocked on diff --git a/synapse/storage/databases/main/roommember.py b/synapse/storage/databases/main/roommember.py index 01d9dbb36f44..dcdaf09682b6 100644 --- a/synapse/storage/databases/main/roommember.py +++ b/synapse/storage/databases/main/roommember.py @@ -14,7 +14,7 @@ # See the License for the specific language governing permissions and # limitations under the License. import logging -from typing import TYPE_CHECKING, Dict, FrozenSet, Iterable, List, Optional, Set +from typing import TYPE_CHECKING, Dict, FrozenSet, Iterable, List, Optional, Set, Tuple from synapse.api.constants import EventTypes, Membership from synapse.events import EventBase @@ -350,6 +350,38 @@ def _get_rooms_for_local_user_where_membership_is_txn( return results + async def get_local_current_membership_for_user_in_room( + self, user_id: str, room_id: str + ) -> Tuple[Optional[str], Optional[str]]: + """Retrieve the current local membership state and event ID for a user in a room. + + Args: + user_id: The ID of the user. + room_id: The ID of the room. + + Returns: + A tuple of (membership_type, event_id). Both will be None if a + room_id/user_id pair is not found. + """ + # Paranoia check. + if not self.hs.is_mine_id(user_id): + raise Exception( + "Cannot call 'get_local_current_membership_for_user_in_room' on " + "non-local user %s" % (user_id,), + ) + + results_dict = await self.db_pool.simple_select_one( + "local_current_membership", + {"room_id": room_id, "user_id": user_id}, + ("membership", "event_id"), + allow_none=True, + desc="get_local_current_membership_for_user_in_room", + ) + if not results_dict: + return None, None + + return results_dict.get("membership"), results_dict.get("event_id") + @cached(max_entries=500000, iterable=True) async def get_rooms_for_user_with_stream_ordering( self, user_id: str From 280eed304e12fc979b0c26de3524ec939efefaa1 Mon Sep 17 00:00:00 2001 From: Andrew Morgan Date: Thu, 26 Nov 2020 16:37:44 +0000 Subject: [PATCH 047/112] Use get_local_current_membership_for_user_in_room for knock code Additionally clean up the logic of this invite/knock code a bit, and it was originally a little tricky to follow the flow. --- synapse/handlers/room_member.py | 46 +++++++++++++-------------------- 1 file changed, 18 insertions(+), 28 deletions(-) diff --git a/synapse/handlers/room_member.py b/synapse/handlers/room_member.py index 3c0a0919c32a..d491d6291fe6 100644 --- a/synapse/handlers/room_member.py +++ b/synapse/handlers/room_member.py @@ -556,17 +556,25 @@ async def update_membership_locked( elif effective_membership_state == Membership.LEAVE: if not is_host_in_room: - # perhaps we've been invited + # Figure out the user's current membership state for the room ( current_membership_type, current_membership_event_id, ) = await self.store.get_local_current_membership_for_user_in_room( target.to_string(), room_id ) - if ( - current_membership_type == Membership.INVITE - and current_membership_event_id - ): + if not current_membership_type or not current_membership_event_id: + logger.info( + "%s sent a leave request to %s, but that is not an active room " + "on this server, or there is no pending invite or knock", + target, + room_id, + ) + + raise SynapseError(404, "Not a known room") + + # perhaps we've been invited + if current_membership_type == Membership.INVITE: invite = await self.store.get_event(current_membership_event_id) logger.info( "%s rejects invite to %s from %s", @@ -595,31 +603,13 @@ async def update_membership_locked( if len(latest_event_ids) == 0: latest_event_ids = [invite.event_id] - else: - # or perhaps this is a remote room that a local user has knocked on - room_state_or_knock = await self.state_handler.get_current_state( - room_id, - event_type=EventTypes.Member, - state_key=target.to_string(), - ) - if room_state_or_knock: - knock = room_state_or_knock.get( - (EventTypes.Member, target.to_string()) - ) - if knock and knock.membership == Membership.KNOCK: - return await self.remote_rescind_knock( - knock.event_id, txn_id, requester, content - ) - - logger.info( - "%s sent a leave request to %s, but that is not an active room " - "on this server, or there is no pending knock", - target, - room_id, + # or perhaps this is a remote room that a local user has knocked on + elif current_membership_type == Membership.KNOCK: + knock = await self.store.get_event(current_membership_event_id) + return await self.remote_rescind_knock( + knock.event_id, txn_id, requester, content ) - raise SynapseError(404, "Not a known room") - elif effective_membership_state == Membership.KNOCK: if not is_host_in_room: # The knock needs to be sent over federation instead From 50354ef3acee1ac47c088447b4eacf45278f0d2a Mon Sep 17 00:00:00 2001 From: Andrew Morgan Date: Thu, 26 Nov 2020 16:47:56 +0000 Subject: [PATCH 048/112] Prevent repeated knocks on a room --- synapse/event_auth.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/synapse/event_auth.py b/synapse/event_auth.py index 4629515a59b8..3cd9b6735fd7 100644 --- a/synapse/event_auth.py +++ b/synapse/event_auth.py @@ -355,6 +355,8 @@ def _is_membership_change_allowed( raise AuthError(403, "You cannot knock for other users") elif target_in_room: raise AuthError(403, "You cannot knock on a room you are already in") + elif caller_knocked: + raise AuthError(403, "You already have a pending knock for this room") elif target_banned: raise AuthError(403, "You are banned from this room") else: From 7c0979634ac21f67fe721d679b15a079eda26e63 Mon Sep 17 00:00:00 2001 From: Andrew Morgan Date: Thu, 26 Nov 2020 16:56:23 +0000 Subject: [PATCH 049/112] Revert "Remove unnecessary arguments from knocking _serialize_payload functions" This reverts commit 3fb055c92adbf12e5c5a0388c1d7e934b5b89e33. It was found that these arguments are necessary and nice to keep around following a discussion in https://github.com/matrix-org/synapse/pull/8809. --- synapse/replication/http/membership.py | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/synapse/replication/http/membership.py b/synapse/replication/http/membership.py index 7631a7f280b6..98ab24f24d7c 100644 --- a/synapse/replication/http/membership.py +++ b/synapse/replication/http/membership.py @@ -115,11 +115,17 @@ def __init__(self, hs): @staticmethod async def _serialize_payload( # type: ignore - requester: Requester, remote_room_hosts: List[str], content: JsonDict, + requester: Requester, + room_id: str, + user_id: str, + remote_room_hosts: List[str], + content: JsonDict, ): """ Args: requester: The user making the request, according to the access token. + room_id: The ID of the room to knock on. + user_id: The ID of the knocking user. remote_room_hosts: Servers to try and send the knock via. content: The event content to use for the knock event. """ @@ -239,10 +245,14 @@ def __init__(self, hs: "HomeServer"): @staticmethod async def _serialize_payload( # type: ignore - txn_id: Optional[str], requester: Requester, content: JsonDict, + knock_event_id: str, + txn_id: Optional[str], + requester: Requester, + content: JsonDict, ): """ Args: + knock_event_id: The ID of the knock to be rescinded. txn_id: An optional transaction ID supplied by the client. requester: The user making the rescind request, according to the access token. content: The content to include in the rescind event. From 510da340b40f8fe83a524e4c3e6df836139ab5e4 Mon Sep 17 00:00:00 2001 From: Andrew Morgan Date: Fri, 27 Nov 2020 11:05:48 +0000 Subject: [PATCH 050/112] Prevent invite->knock membership changes --- synapse/event_auth.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/synapse/event_auth.py b/synapse/event_auth.py index 3cd9b6735fd7..6109490b99c8 100644 --- a/synapse/event_auth.py +++ b/synapse/event_auth.py @@ -357,6 +357,8 @@ def _is_membership_change_allowed( raise AuthError(403, "You cannot knock on a room you are already in") elif caller_knocked: raise AuthError(403, "You already have a pending knock for this room") + elif caller_invited: + raise AuthError(403, "You are already invited to this room") elif target_banned: raise AuthError(403, "You are banned from this room") else: From c1f4b62fc87129a99b60630169a037754118851e Mon Sep 17 00:00:00 2001 From: Andrew Morgan Date: Fri, 27 Nov 2020 18:23:30 +0000 Subject: [PATCH 051/112] Make sure we actually include stripped room state for knocking users --- synapse/rest/client/v2_alpha/sync.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/synapse/rest/client/v2_alpha/sync.py b/synapse/rest/client/v2_alpha/sync.py index 10313c31561a..31498432178a 100644 --- a/synapse/rest/client/v2_alpha/sync.py +++ b/synapse/rest/client/v2_alpha/sync.py @@ -356,7 +356,11 @@ async def encode_knocked( knocked = {} for room in rooms: knock = await self._event_serializer.serialize_event( - room.knock, time_now, token_id=token_id, event_format=event_formatter, + room.knock, + time_now, + token_id=token_id, + event_format=event_formatter, + include_stripped_room_state=True, ) # Extract the `unsigned` key from the knock event. From 338861f295742b960611abd1b97765a7ba8f6f95 Mon Sep 17 00:00:00 2001 From: Andrew Morgan Date: Mon, 30 Nov 2020 17:14:08 +0000 Subject: [PATCH 052/112] /unstable/*_xyz.amorgan.knock/ -> /unstable/xyz.amorgan.knock/* --- synapse/federation/federation_client.py | 2 +- synapse/federation/federation_server.py | 2 +- synapse/federation/transport/client.py | 18 +++++++++++------- synapse/federation/transport/server.py | 8 ++++---- synapse/handlers/federation.py | 13 +++++++------ 5 files changed, 24 insertions(+), 19 deletions(-) diff --git a/synapse/federation/federation_client.py b/synapse/federation/federation_client.py index fbd1bd2153f7..895ac59d3691 100644 --- a/synapse/federation/federation_client.py +++ b/synapse/federation/federation_client.py @@ -910,7 +910,7 @@ async def send_request(destination: str) -> JsonDict: return await self._do_send_knock(destination, pdu) return await self._try_destination_list( - "send_xyz.amorgan.knock", destinations, send_request + "xyz.amorgan.knock/send_knock", destinations, send_request ) async def _do_send_knock(self, destination: str, pdu: EventBase) -> JsonDict: diff --git a/synapse/federation/federation_server.py b/synapse/federation/federation_server.py index 6006b3860762..6a2631ee18ce 100644 --- a/synapse/federation/federation_server.py +++ b/synapse/federation/federation_server.py @@ -570,7 +570,7 @@ async def on_send_leave_request(self, origin: str, content: JsonDict) -> dict: async def on_make_knock_request( self, origin: str, room_id: str, user_id: str, ) -> Dict[str, Union[EventBase, str]]: - """We've received a /make_xyz.amorgan.knock/ request, so we create a partial knock + """We've received a /make_knock/ request, so we create a partial knock event for the room and hand that back, along with the room version, to the knocking homeserver. We do *not* persist or process this event until the other server has signed it and sent it back. diff --git a/synapse/federation/transport/client.py b/synapse/federation/transport/client.py index 8aab18c96f64..d64ecfdcfa6f 100644 --- a/synapse/federation/transport/client.py +++ b/synapse/federation/transport/client.py @@ -221,11 +221,15 @@ async def make_membership_event( # Knock currently uses an unstable prefix if membership == Membership.KNOCK: - prefix = FEDERATION_UNSTABLE_PREFIX + # Create a path in the form of /unstable/xyz.amorgan.knock/make_knock/... + path = _create_path( + FEDERATION_UNSTABLE_PREFIX + "/xyz.amorgan.knock", + "/make_knock/%s/%s", + room_id, + user_id, + ) else: - prefix = FEDERATION_V1_PREFIX - - path = _create_path(prefix, "/make_%s/%s/%s", membership, room_id, user_id) + path = _create_v1_path("/make_%s/%s/%s", membership, room_id, user_id) ignore_backoff = False retry_on_dns_fail = False @@ -309,7 +313,7 @@ async def send_knock_v2( ) -> JsonDict: """ Sends a signed knock membership event to a remote server. This is the second - step for knocking after /make_xyz.amorgan.knock. + step for knocking after make_knock. Args: destination: The remote homeserver. @@ -328,8 +332,8 @@ async def send_knock_v2( The list of state events may be empty. """ path = _create_path( - FEDERATION_UNSTABLE_PREFIX, - "/send_xyz.amorgan.knock/%s/%s", + FEDERATION_UNSTABLE_PREFIX + "/xyz.amorgan.knock", + "/send_knock/%s/%s", room_id, event_id, ) diff --git a/synapse/federation/transport/server.py b/synapse/federation/transport/server.py index a55225eaa83b..aaf125daba25 100644 --- a/synapse/federation/transport/server.py +++ b/synapse/federation/transport/server.py @@ -544,9 +544,9 @@ async def on_PUT(self, origin, content, query, room_id, event_id): class FederationMakeKnockServlet(BaseFederationServlet): - PATH = "/make_xyz.amorgan.knock/(?P[^/]*)/(?P[^/]*)" + PATH = "/make_knock/(?P[^/]*)/(?P[^/]*)" - PREFIX = FEDERATION_UNSTABLE_PREFIX + PREFIX = FEDERATION_UNSTABLE_PREFIX + "/xyz.amorgan.knock" async def on_GET(self, origin, content, query, room_id, user_id): content = await self.handler.on_make_knock_request(origin, room_id, user_id) @@ -554,9 +554,9 @@ async def on_GET(self, origin, content, query, room_id, user_id): class FederationV2SendKnockServlet(BaseFederationServlet): - PATH = "/send_xyz.amorgan.knock/(?P[^/]*)/(?P[^/]*)" + PATH = "/send_knock/(?P[^/]*)/(?P[^/]*)" - PREFIX = FEDERATION_UNSTABLE_PREFIX + PREFIX = FEDERATION_UNSTABLE_PREFIX + "/xyz.amorgan.knock" async def on_PUT(self, origin, content, query, room_id, event_id): content = await self.handler.on_send_knock_request(origin, content, room_id) diff --git a/synapse/handlers/federation.py b/synapse/handlers/federation.py index 2982c9fbe8e3..67b6ec62534b 100644 --- a/synapse/handlers/federation.py +++ b/synapse/handlers/federation.py @@ -1446,9 +1446,9 @@ async def do_knock( ) -> Tuple[str, int]: """Sends the knock to the remote server. - This first triggers a /make_xyz.amorgan.knock request that returns a partial + This first triggers a make_knock request that returns a partial event that we can fill out and sign. This is then sent to the - remote server via /send_xyz.amorgan.knock. + remote server via send_knock. Knock events must be signed by the knockee's server before distributing. @@ -1478,7 +1478,7 @@ async def do_knock( room_id=event.room_id, room_version=event_format_version ) - # Initially try the host that we successfully called /make_xyz.amorgan.knock on + # Initially try the host that we successfully called /make_knock on try: target_hosts.remove(origin) target_hosts.insert(0, origin) @@ -1843,7 +1843,7 @@ async def on_send_leave_request(self, origin, pdu): async def on_make_knock_request( self, origin: str, room_id: str, user_id: str ) -> EventBase: - """We've received a /make_xyz.amorgan.knock/ request, so we create a partial + """We've received a make_knock request, so we create a partial knock event for the room and return that. We do *not* persist or process it until the other server has signed it and sent it back. @@ -1857,7 +1857,7 @@ async def on_make_knock_request( """ if get_domain_from_id(user_id) != origin: logger.info( - "Get /make_xyz.amorgan.knock request for user %r" + "Get /xyz.amorgan.knock/make_knock request for user %r" "from different origin %s, ignoring", user_id, origin, @@ -1924,7 +1924,8 @@ async def on_send_knock_request( if get_domain_from_id(event.sender) != origin: logger.info( - "Got /send_xyz.amorgan.knock request for user %r from different origin %s", + "Got /xyz.amorgan.knock/send_knock request for user %r " + "from different origin %s", event.sender, origin, ) From 0c4bdb0e9be023e8e697f9eeab3144a66862ba29 Mon Sep 17 00:00:00 2001 From: Andrew Morgan Date: Mon, 30 Nov 2020 17:33:51 +0000 Subject: [PATCH 053/112] Prevent modification of the unsigned dict where we store knock room state --- synapse/rest/client/v2_alpha/sync.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/synapse/rest/client/v2_alpha/sync.py b/synapse/rest/client/v2_alpha/sync.py index 31498432178a..116b87f65bad 100644 --- a/synapse/rest/client/v2_alpha/sync.py +++ b/synapse/rest/client/v2_alpha/sync.py @@ -367,6 +367,9 @@ async def encode_knocked( # This is where we (cheekily) store the knock state events unsigned = knock.setdefault("unsigned", {}) + # Duplicate the dictionary in order to avoid modifying the original + unsigned = dict(unsigned) + # Extract the stripped room state from the unsigned dict # This is for clients to get a little bit of information about # the room they've knocked on, without revealing any sensitive information From d394eb93502c977cc25e3f9e626b70f9675c3467 Mon Sep 17 00:00:00 2001 From: Andrew Morgan Date: Mon, 30 Nov 2020 17:38:05 +0000 Subject: [PATCH 054/112] Provide reasoning for appending the knock to knock_state --- synapse/rest/client/v2_alpha/sync.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/synapse/rest/client/v2_alpha/sync.py b/synapse/rest/client/v2_alpha/sync.py index 116b87f65bad..582c999abd76 100644 --- a/synapse/rest/client/v2_alpha/sync.py +++ b/synapse/rest/client/v2_alpha/sync.py @@ -375,10 +375,11 @@ async def encode_knocked( # the room they've knocked on, without revealing any sensitive information knocked_state = list(unsigned.pop("knock_room_state", [])) - # Append the actual knock membership event itself as well - # TODO: I *believe* this is just for the client's sake of track its membership - # state in each room, but I could be wrong. This certainly doesn't seem like it - # could have any negative effects besides resource usage + # Append the actual knock membership event itself as well. This provides + # the client with: + # + # * A knock state event that they can use for easier internal tracking + # * The rough timestamp of when the knock occurred contained within the event knocked_state.append(knock) # Build the `knock_state` dictionary, which will contain the state of the From 0f62e2de3cc9818a9e7abb41088707cb02c48ed3 Mon Sep 17 00:00:00 2001 From: Andrew Morgan Date: Tue, 1 Dec 2020 16:57:10 +0000 Subject: [PATCH 055/112] Allow specifying room version in RestHelper.create_room_as. Add typing --- tests/rest/client/v1/utils.py | 27 +++++++++++++++++++++++++-- 1 file changed, 25 insertions(+), 2 deletions(-) diff --git a/tests/rest/client/v1/utils.py b/tests/rest/client/v1/utils.py index b58768675b86..737c38c396dd 100644 --- a/tests/rest/client/v1/utils.py +++ b/tests/rest/client/v1/utils.py @@ -41,14 +41,37 @@ class RestHelper: auth_user_id = attr.ib() def create_room_as( - self, room_creator=None, is_public=True, tok=None, expect_code=200, - ): + self, + room_creator: str = None, + is_public: bool = True, + room_version: str = None, + tok: str = None, + expect_code: int = 200, + ) -> str: + """ + Create a room. + + Args: + room_creator: The user ID to create the room with. + is_public: If True, the `visibility` parameter will be set to the + default (public). Otherwise, the `visibility` parameter will be set + to "private". + room_version: The room version to create the room as. Defaults to Synapse's + default room version. + tok: The access token to use in the request. + expect_code: The expected HTTP response code. + + Returns: + The ID of the newly created room. + """ temp_id = self.auth_user_id self.auth_user_id = room_creator path = "/_matrix/client/r0/createRoom" content = {} if not is_public: content["visibility"] = "private" + if room_version: + content["room_version"] = room_version if tok: path = path + "?access_token=%s" % tok From 60466b82618310421c84b56de0fbe0491ddb2c8a Mon Sep 17 00:00:00 2001 From: Andrew Morgan Date: Tue, 1 Dec 2020 19:18:53 +0000 Subject: [PATCH 056/112] make sure to include stripped state events for local knocks as well! The type change was necessary as DEFAULT_ROOM_STATE_TYPES is actually a list of str. --- synapse/handlers/message.py | 8 ++++++++ synapse/storage/databases/main/events_worker.py | 2 +- 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/synapse/handlers/message.py b/synapse/handlers/message.py index c1df9a8ca2a4..2fcba6abfc36 100644 --- a/synapse/handlers/message.py +++ b/synapse/handlers/message.py @@ -41,6 +41,7 @@ ) from synapse.api.room_versions import KNOWN_ROOM_VERSIONS, RoomVersions from synapse.api.urls import ConsentURIBuilder +from synapse.config.api import DEFAULT_ROOM_STATE_TYPES from synapse.events import EventBase from synapse.events.builder import EventBuilder from synapse.events.snapshot import EventContext @@ -1126,6 +1127,13 @@ async def persist_and_notify_client_event( # TODO: Make sure the signatures actually are correct. event.signatures.update(returned_invite.signatures) + if event.content["membership"] == Membership.KNOCK: + event.unsigned[ + "knock_room_state" + ] = await self.store.get_stripped_room_state_from_event_context( + context, DEFAULT_ROOM_STATE_TYPES, + ) + if event.type == EventTypes.Redaction: original_event = await self.store.get_event( event.redacts, diff --git a/synapse/storage/databases/main/events_worker.py b/synapse/storage/databases/main/events_worker.py index 4732685f6e60..f21baffd10db 100644 --- a/synapse/storage/databases/main/events_worker.py +++ b/synapse/storage/databases/main/events_worker.py @@ -529,7 +529,7 @@ def _get_events_from_cache(self, events, allow_rejected, update_metrics=True): async def get_stripped_room_state_from_event_context( self, context: EventContext, - state_types_to_include: List[EventTypes], + state_types_to_include: List[str], membership_user_id: Optional[str] = None, ) -> List[JsonDict]: """ From aaa6c88ed89bd8bd10cea9e39dcdc3778f7f3c13 Mon Sep 17 00:00:00 2001 From: Andrew Morgan Date: Tue, 1 Dec 2020 16:57:54 +0000 Subject: [PATCH 057/112] Add unit tests for knocking This specifically tests the stripped state events that come down /sync, as at the current stage in the MSC the exact types to send are not fixed. There's also a tiny type fix included in a nearby test. --- tests/rest/client/v2_alpha/test_sync.py | 143 +++++++++++++++++++++++- 1 file changed, 141 insertions(+), 2 deletions(-) diff --git a/tests/rest/client/v2_alpha/test_sync.py b/tests/rest/client/v2_alpha/test_sync.py index 31ac0fccb8f3..9b77f79058d0 100644 --- a/tests/rest/client/v2_alpha/test_sync.py +++ b/tests/rest/client/v2_alpha/test_sync.py @@ -18,7 +18,8 @@ import synapse.rest.admin from synapse.api.constants import EventContentFields, EventTypes, RelationTypes from synapse.rest.client.v1 import login, room -from synapse.rest.client.v2_alpha import read_marker, sync +from synapse.rest.client.v2_alpha import knock, read_marker, sync +from synapse.types import RoomAlias from tests import unittest from tests.server import TimedOutException @@ -314,6 +315,144 @@ def test_sync_backwards_typing(self): self.make_request("GET", sync_url % (access_token, next_batch)) +class SyncKnockTestCase(unittest.HomeserverTestCase): + servlets = [ + synapse.rest.admin.register_servlets, + login.register_servlets, + room.register_servlets, + sync.register_servlets, + knock.register_servlets, + ] + + def prepare(self, reactor, clock, hs): + self.store = hs.get_datastore() + self.url = "/sync?since=%s" + self.next_batch = "s0" + + # Register the first user (used to create the room to knock on). + self.user_id = self.register_user("kermit", "monkey") + self.tok = self.login("kermit", "monkey") + + # Create the room we'll knock on. + self.room_id = self.helper.create_room_as( + self.user_id, + is_public=False, + room_version="xyz.amorgan.knock", + tok=self.tok, + ) + + # Register the second user (used to knock on the room). + self.knocker = self.register_user("knocker", "monkey") + self.knocker_tok = self.login("knocker", "monkey") + + # Perform an initial sync for the knocking user + request, channel = self.make_request( + "GET", self.url % self.next_batch, access_token=self.tok, + ) + self.assertEqual(channel.code, 200, channel.json_body) + + # Store the next batch for the next request. + self.next_batch = channel.json_body["next_batch"] + + # Set up some room state to test with. + + # To set a canonical alias, we'll need to point an alias at the room first. + canonical_alias = "#fancy_alias:test" + self.get_success( + self.store.create_room_alias_association( + RoomAlias.from_string(canonical_alias), self.room_id, ["test"] + ) + ) + + self.room_state = { + # We need to set the room's join rules to allow knocking + EventTypes.JoinRules: { + "content": {"join_rule": "xyz.amorgan.knock"}, + "state_key": "", + }, + # The rest are state events that are recommended to strip and send to clients + EventTypes.Name: {"content": {"name": "A cool room"}, "state_key": ""}, + EventTypes.RoomAvatar: { + "content": { + "info": { + "h": 398, + "mimetype": "image/jpeg", + "size": 31037, + "w": 394, + }, + "url": "mxc://example.org/JWEIFJgwEIhweiWJE", + }, + "state_key": "", + }, + EventTypes.RoomEncryption: { + "content": {"algorithm": "m.megolm.v1.aes-sha2"}, + "state_key": "", + }, + EventTypes.CanonicalAlias: { + "content": {"alias": canonical_alias, "alt_aliases": []}, + "state_key": "", + }, + } + + for event_type, event in self.room_state.items(): + self.helper.send_state( + self.room_id, event_type, event["content"], tok=self.tok, + ) + + # We expect the knock event to be included as well + self.room_state[EventTypes.Member] = { + "content": {"membership": "xyz.amorgan.knock", "displayname": "knocker"}, + "state_key": "@knocker:test", + } + + def test_knock_room_state(self): + """Tests that /sync returns state from a room after knocking on it.""" + # Knock on a room + request, channel = self.make_request( + "POST", + "/_matrix/client/unstable/xyz.amorgan.knock/%s" % (self.room_id,), + b"{}", + self.knocker_tok, + shorthand=False, + ) + self.assertEquals(200, channel.code, channel.result) + + # Check that /sync includes stripped state from the room + request, channel = self.make_request( + "GET", self.url % self.next_batch, access_token=self.knocker_tok, + ) + self.assertEqual(channel.code, 200, channel.json_body) + + knock_entry = channel.json_body["rooms"]["xyz.amorgan.knock"] + room_state_events = knock_entry[self.room_id]["knock_state"]["events"] + for event in room_state_events: + event_type = event["type"] + if event_type not in self.room_state: + raise Exception( + "Unexpected room state type '%s' in knock sync result" + % (event_type,) + ) + + # Check the state content matches + self.assertDictEqual( + self.room_state[event_type]["content"], event["content"] + ) + + # Check the state key is correct + self.assertEqual( + self.room_state[event_type]["state_key"], event["state_key"] + ) + + # Ensure the event has been stripped + self.assertNotIn("signatures", event) + + # Pop once we've found and processed a state event + self.room_state.pop(event_type) + + # Check that all expected state events were accounted for + self.assertEqual(len(self.room_state), 0) + + class UnreadMessagesTestCase(unittest.HomeserverTestCase): servlets = [ synapse.rest.admin.register_servlets, @@ -447,7 +586,7 @@ def test_unread_counts(self): ) self._check_unread_count(5) - def _check_unread_count(self, expected_count: True): + def _check_unread_count(self, expected_count: int): """Syncs and compares the unread count with the expected value.""" request, channel = self.make_request( From fa6618990be7d6ef2396da4a6a2801173290989c Mon Sep 17 00:00:00 2001 From: Andrew Morgan Date: Wed, 2 Dec 2020 17:28:31 +0000 Subject: [PATCH 058/112] Use an OrderedDict for knock room state --- tests/rest/client/v2_alpha/test_sync.py | 75 ++++++++++++++++--------- 1 file changed, 47 insertions(+), 28 deletions(-) diff --git a/tests/rest/client/v2_alpha/test_sync.py b/tests/rest/client/v2_alpha/test_sync.py index 9b77f79058d0..543ecae39c75 100644 --- a/tests/rest/client/v2_alpha/test_sync.py +++ b/tests/rest/client/v2_alpha/test_sync.py @@ -14,6 +14,7 @@ # See the License for the specific language governing permissions and # limitations under the License. import json +from collections import OrderedDict import synapse.rest.admin from synapse.api.constants import EventContentFields, EventTypes, RelationTypes @@ -364,35 +365,50 @@ def prepare(self, reactor, clock, hs): ) ) - self.room_state = { - # We need to set the room's join rules to allow knocking - EventTypes.JoinRules: { - "content": {"join_rule": "xyz.amorgan.knock"}, - "state_key": "", - }, - # The rest are state events that are recommended to strip and send to clients - EventTypes.Name: {"content": {"name": "A cool room"}, "state_key": ""}, - EventTypes.RoomAvatar: { - "content": { - "info": { - "h": 398, - "mimetype": "image/jpeg", - "size": 31037, - "w": 394, + # We use an OrderedDict here to ensure that the knock membership appears last + self.room_state = OrderedDict( + [ + # We need to set the room's join rules to allow knocking + ( + EventTypes.JoinRules, + {"content": {"join_rule": "xyz.amorgan.knock"}, "state_key": ""}, + ), + # The rest are state events that are recommended to strip and send to clients + ( + EventTypes.Name, + {"content": {"name": "A cool room"}, "state_key": ""}, + ), + ( + EventTypes.RoomAvatar, + { + "content": { + "info": { + "h": 398, + "mimetype": "image/jpeg", + "size": 31037, + "w": 394, + }, + "url": "mxc://example.org/JWEIFJgwEIhweiWJE", + }, + "state_key": "", }, - "url": "mxc://example.org/JWEIFJgwEIhweiWJE", - }, - "state_key": "", - }, - EventTypes.RoomEncryption: { - "content": {"algorithm": "m.megolm.v1.aes-sha2"}, - "state_key": "", - }, - EventTypes.CanonicalAlias: { - "content": {"alias": canonical_alias, "alt_aliases": []}, - "state_key": "", - }, - } + ), + ( + EventTypes.RoomEncryption, + { + "content": {"algorithm": "m.megolm.v1.aes-sha2"}, + "state_key": "", + }, + ), + ( + EventTypes.CanonicalAlias, + { + "content": {"alias": canonical_alias, "alt_aliases": []}, + "state_key": "", + }, + ), + ] + ) for event_type, event in self.room_state.items(): self.helper.send_state( @@ -452,6 +468,9 @@ def test_knock_room_state(self): # Check that all expected state events were accounted for self.assertEqual(len(self.room_state), 0) + # Ensure that the knock membership event came last + self.assertEqual(room_state_events[-1]["type"], EventTypes.Member) + class UnreadMessagesTestCase(unittest.HomeserverTestCase): servlets = [ From 220ec91965f1397d5abd2ee0c11bee02bfe38d91 Mon Sep 17 00:00:00 2001 From: Andrew Morgan Date: Wed, 2 Dec 2020 17:31:16 +0000 Subject: [PATCH 059/112] AssertEquals calls AssertDictEqual when acting on dicts --- tests/rest/client/v2_alpha/test_sync.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/tests/rest/client/v2_alpha/test_sync.py b/tests/rest/client/v2_alpha/test_sync.py index 543ecae39c75..ab32753bf3d4 100644 --- a/tests/rest/client/v2_alpha/test_sync.py +++ b/tests/rest/client/v2_alpha/test_sync.py @@ -450,9 +450,7 @@ def test_knock_room_state(self): ) # Check the state content matches - self.assertDictEqual( - self.room_state[event_type]["content"], event["content"] - ) + self.assertEquals(self.room_state[event_type]["content"], event["content"]) # Check the state key is correct self.assertEqual( From 328a2744f03b032a6d55271ab470d90cb54feabc Mon Sep 17 00:00:00 2001 From: Andrew Morgan Date: Wed, 2 Dec 2020 17:32:19 +0000 Subject: [PATCH 060/112] Use an assertIn instead of raising an Exception --- tests/rest/client/v2_alpha/test_sync.py | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/tests/rest/client/v2_alpha/test_sync.py b/tests/rest/client/v2_alpha/test_sync.py index ab32753bf3d4..7b11cc72c622 100644 --- a/tests/rest/client/v2_alpha/test_sync.py +++ b/tests/rest/client/v2_alpha/test_sync.py @@ -443,11 +443,7 @@ def test_knock_room_state(self): room_state_events = knock_entry[self.room_id]["knock_state"]["events"] for event in room_state_events: event_type = event["type"] - if event_type not in self.room_state: - raise Exception( - "Unexpected room state type '%s' in knock sync result" - % (event_type,) - ) + self.assertIn(event_type, self.room_state) # Check the state content matches self.assertEquals(self.room_state[event_type]["content"], event["content"]) From 06aa1749f8e8771b61e7b1e7b721ebe63d7b00d9 Mon Sep 17 00:00:00 2001 From: Andrew Morgan Date: Thu, 3 Dec 2020 16:03:01 +0000 Subject: [PATCH 061/112] Remove unnecessary parameter shorthand being on has no effect if the path starts with /_matrix --- tests/rest/client/v2_alpha/test_sync.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/rest/client/v2_alpha/test_sync.py b/tests/rest/client/v2_alpha/test_sync.py index 7b11cc72c622..d5c049d8f2a9 100644 --- a/tests/rest/client/v2_alpha/test_sync.py +++ b/tests/rest/client/v2_alpha/test_sync.py @@ -429,7 +429,6 @@ def test_knock_room_state(self): "/_matrix/client/unstable/xyz.amorgan.knock/%s" % (self.room_id,), b"{}", self.knocker_tok, - shorthand=False, ) self.assertEquals(200, channel.code, channel.result) From 8dcc9f8fc7a90c2d9a8848d54960cc24b25369d7 Mon Sep 17 00:00:00 2001 From: Andrew Morgan Date: Thu, 3 Dec 2020 17:06:23 +0000 Subject: [PATCH 062/112] Add tests for federated stripped room state. test only some room state included The changes included here cover a few things: * A new testcase was added that performs federated make_knock and send_knock calls, checking that all expected stripped room state is included in the response to send_knock. * Abstracting the stripped room state checks from SyncKnockTestCase into generic functions that have been moved into FederationKnockingTestCase. We then import these into SyncKnockTestCase instead. * Two functions were created. One that sends the test state into a room, and another that takes that test state + the stripped state events received while knocking and compares them. * The abstracted function for checking state now also checks that not all state in the room is given to the knocker. For instance, we probably don't want to give widget URLs to someone knocking on a room before we've accepted them! Thus we include a "secret state event" in the room and check that the knocker does not receive it. When writing the federation tests it took quite a while for me to figure out how to get a homeserver to accept calls from another homeserver that didn't exist. To do so, I ended up mocking out the event signature and auth checking. These checks aren't relavent to this test, and are instead checked by those in Complement. --- tests/federation/transport/test_knocking.py | 289 ++++++++++++++++++++ tests/rest/client/v2_alpha/test_sync.py | 109 ++------ 2 files changed, 310 insertions(+), 88 deletions(-) create mode 100644 tests/federation/transport/test_knocking.py diff --git a/tests/federation/transport/test_knocking.py b/tests/federation/transport/test_knocking.py new file mode 100644 index 000000000000..0606fa8b81f2 --- /dev/null +++ b/tests/federation/transport/test_knocking.py @@ -0,0 +1,289 @@ +from collections import OrderedDict +from typing import Dict, List + +from mock import Mock + +from twisted.internet.defer import succeed + +from synapse import event_auth +from synapse.api.constants import EventTypes +from synapse.api.room_versions import RoomVersions +from synapse.config.ratelimiting import FederationRateLimitConfig +from synapse.events import builder +from synapse.federation.transport import server as federation_server +from synapse.rest import admin +from synapse.rest.client.v1 import login, room +from synapse.server import HomeServer +from synapse.types import RoomAlias +from synapse.util.ratelimitutils import FederationRateLimiter + +from tests.test_utils import event_injection, make_awaitable +from tests.unittest import FederatingHomeserverTestCase, HomeserverTestCase + +# An identifier to use while MSC2304 is not in a stable release of the spec +KNOCK_UNSTABLE_IDENTIFIER = "xyz.amorgan.knock" + +# An event type that we do not expect to be given to users knocking on a room +SECRET_STATE_EVENT_TYPE = "com.example.secret" + + +class FederationKnockingTestCase(FederatingHomeserverTestCase): + servlets = [ + admin.register_servlets, + room.register_servlets, + login.register_servlets, + ] + + def prepare(self, reactor, clock, homeserver): + self.store = homeserver.get_datastore() + + # Have this homeserver auto-approve all event signature checking + def approve_all_signature_checking(_, ev): + return [succeed(ev[0])] + + homeserver.get_federation_server()._check_sigs_and_hashes = ( + approve_all_signature_checking + ) + + # Have this homeserver skip event auth checks. + # + # While this prevent membership transistion checks, that is already + # tested elsewhere + event_auth.check = Mock(return_value=make_awaitable(None)) + + class Authenticator: + def authenticate_request(self, request, content): + return make_awaitable("other.example.com") + + ratelimiter = FederationRateLimiter( + clock, + FederationRateLimitConfig( + window_size=1, + sleep_limit=1, + sleep_msec=1, + reject_limit=1000, + concurrent_requests=1000, + ), + ) + federation_server.register_servlets( + homeserver, self.resource, Authenticator(), ratelimiter + ) + + return super().prepare(reactor, clock, homeserver) + + def test_room_state_returned_when_knocking(self): + """ + Tests that specific, stripped state events from a room are returned after + a remote homeserver successfully knocks on a local room. + """ + user_id = self.register_user("u1", "you the one") + user_token = self.login("u1", "you the one") + + fake_knocking_user_id = "@user:other.example.com" + + # Create a room with a room version that includes knocking + room_id = self.helper.create_room_as( + "u1", + is_public=False, + room_version=KNOCK_UNSTABLE_IDENTIFIER, + tok=user_token, + ) + + # Update the join rules and add additional state to the room to check for later + expected_room_state = send_example_state_events_to_room( + self, self.hs, room_id, user_id + ) + + request, channel = self.make_request( + "GET", + "/_matrix/federation/unstable/%s/make_knock/%s/%s" + % (KNOCK_UNSTABLE_IDENTIFIER, room_id, fake_knocking_user_id), + ) + self.assertEquals(200, channel.code, channel.result) + + # Note: We don't expect the knock membership event to be sent over federation as + # part of the stripped room state, as the knocking homeserver already has that + # event. It is only done for clients during /sync + + # Extract the generated knock event json + knock_event = channel.json_body["event"] + + # Check that the event has things we expect in it + self.assertEquals(knock_event["room_id"], room_id) + self.assertEquals(knock_event["sender"], fake_knocking_user_id) + self.assertEquals(knock_event["state_key"], fake_knocking_user_id) + self.assertEquals(knock_event["type"], EventTypes.Member) + self.assertEquals( + knock_event["content"]["membership"], KNOCK_UNSTABLE_IDENTIFIER + ) + + # Turn the event json dict into a proper event. + # We won't sign it properly, but that's OK as we stub out event auth in `prepare` + signed_knock_event = builder.create_local_event_from_event_dict( + self.clock, + self.hs.hostname, + self.hs.signing_key, + room_version=RoomVersions.MSC2403_DEV, + event_dict=knock_event, + ) + + # Convert our proper event back to json dict format + signed_knock_event_json = signed_knock_event.get_pdu_json( + self.clock.time_msec() + ) + + # Send the signed knock event into the room + request, channel = self.make_request( + "PUT", + "/_matrix/federation/unstable/%s/send_knock/%s/%s" + % (KNOCK_UNSTABLE_IDENTIFIER, room_id, signed_knock_event.event_id), + signed_knock_event_json, + ) + self.assertEquals(200, channel.code, channel.result) + + # Check that we got the stripped room state in return + room_state_events = channel.json_body["knock_state_events"] + + # Validate the stripped room state events + check_knock_room_state_against_room_state( + self, room_state_events, expected_room_state + ) + + +def send_example_state_events_to_room( + testcase: HomeserverTestCase, hs: "HomeServer", room_id: str, sender: str, +) -> OrderedDict: + """Adds some state a room. State events are those that should be sent to a knocking + user after they knock on the room, as well as some state that *shouldn't* be sent + to the knocking user. + + Args: + testcase: The testcase that is currently active. + hs: The homeserver of the sender. + room_id: The ID of the room to send state into. + sender: The ID of the user to send state as. Must be in the room. + + Returns: + The OrderedDict of event types and content that a user is expected to see + after knocking on a room. + """ + # To set a canonical alias, we'll need to point an alias at the room first. + canonical_alias = "#fancy_alias:test" + testcase.get_success( + testcase.store.create_room_alias_association( + RoomAlias.from_string(canonical_alias), room_id, ["test"] + ) + ) + + # Send some state that we *don't* expect to be given to knocking users + secret_state_event_type = "com.example.secret" + testcase.get_success( + event_injection.inject_event( + hs, + room_version=KNOCK_UNSTABLE_IDENTIFIER, + room_id=room_id, + sender=sender, + type=secret_state_event_type, + state_key="", + content={"secret": "password"}, + ) + ) + + # We use an OrderedDict here to ensure that the knock membership appears last. + # Note that order only matters when sending stripped state to clients, not federated + # homeservers. + room_state = OrderedDict( + [ + # We need to set the room's join rules to allow knocking + ( + EventTypes.JoinRules, + {"content": {"join_rule": "xyz.amorgan.knock"}, "state_key": ""}, + ), + # Below are state events that are to be stripped and sent to clients + (EventTypes.Name, {"content": {"name": "A cool room"}, "state_key": ""},), + ( + EventTypes.RoomAvatar, + { + "content": { + "info": { + "h": 398, + "mimetype": "image/jpeg", + "size": 31037, + "w": 394, + }, + "url": "mxc://example.org/JWEIFJgwEIhweiWJE", + }, + "state_key": "", + }, + ), + ( + EventTypes.RoomEncryption, + {"content": {"algorithm": "m.megolm.v1.aes-sha2"}, "state_key": ""}, + ), + ( + EventTypes.CanonicalAlias, + { + "content": {"alias": canonical_alias, "alt_aliases": []}, + "state_key": "", + }, + ), + ] + ) + + for event_type, event_dict in room_state.items(): + event_content, state_key = event_dict.values() + + testcase.get_success( + event_injection.inject_event( + hs, + room_version=KNOCK_UNSTABLE_IDENTIFIER, + room_id=room_id, + sender=sender, + type=event_type, + state_key=state_key, + content=event_content, + ) + ) + + return room_state + + +def check_knock_room_state_against_room_state( + testcase: HomeserverTestCase, + knock_room_state: List[Dict], + expected_room_state: Dict, +) -> None: + """Test a list of stripped room state events received over federation against an + dict of expected state events. + + Args: + testcase: The testcase that is currently active. + knock_room_state: The list of room state that was received over federation. + expected_room_state: A dict containing the room state we expect to see in + `knock_room_state`. + """ + for event in knock_room_state: + event_type = event["type"] + testcase.assertIn(event_type, expected_room_state) + + # Check the state content matches + testcase.assertEquals( + expected_room_state[event_type]["content"], event["content"] + ) + + # Check the state key is correct + testcase.assertEqual( + expected_room_state[event_type]["state_key"], event["state_key"] + ) + + # Ensure the event has been stripped + testcase.assertNotIn("signatures", event) + + # Pop once we've found and processed a state event + expected_room_state.pop(event_type) + + # Check that all expected state events were accounted for + testcase.assertEqual(len(expected_room_state), 0) + + # Ensure that no excess state was included + testcase.assertNotIn(SECRET_STATE_EVENT_TYPE, knock_room_state) diff --git a/tests/rest/client/v2_alpha/test_sync.py b/tests/rest/client/v2_alpha/test_sync.py index d5c049d8f2a9..6c5b504f03f3 100644 --- a/tests/rest/client/v2_alpha/test_sync.py +++ b/tests/rest/client/v2_alpha/test_sync.py @@ -14,15 +14,17 @@ # See the License for the specific language governing permissions and # limitations under the License. import json -from collections import OrderedDict import synapse.rest.admin from synapse.api.constants import EventContentFields, EventTypes, RelationTypes from synapse.rest.client.v1 import login, room from synapse.rest.client.v2_alpha import knock, read_marker, sync -from synapse.types import RoomAlias from tests import unittest +from tests.federation.transport.test_knocking import ( + check_knock_room_state_against_room_state, + send_example_state_events_to_room, +) from tests.server import TimedOutException @@ -346,7 +348,7 @@ def prepare(self, reactor, clock, hs): self.knocker = self.register_user("knocker", "monkey") self.knocker_tok = self.login("knocker", "monkey") - # Perform an initial sync for the knocking user + # Perform an initial sync for the knocking user. request, channel = self.make_request( "GET", self.url % self.next_batch, access_token=self.tok, ) @@ -356,70 +358,9 @@ def prepare(self, reactor, clock, hs): self.next_batch = channel.json_body["next_batch"] # Set up some room state to test with. - - # To set a canonical alias, we'll need to point an alias at the room first. - canonical_alias = "#fancy_alias:test" - self.get_success( - self.store.create_room_alias_association( - RoomAlias.from_string(canonical_alias), self.room_id, ["test"] - ) - ) - - # We use an OrderedDict here to ensure that the knock membership appears last - self.room_state = OrderedDict( - [ - # We need to set the room's join rules to allow knocking - ( - EventTypes.JoinRules, - {"content": {"join_rule": "xyz.amorgan.knock"}, "state_key": ""}, - ), - # The rest are state events that are recommended to strip and send to clients - ( - EventTypes.Name, - {"content": {"name": "A cool room"}, "state_key": ""}, - ), - ( - EventTypes.RoomAvatar, - { - "content": { - "info": { - "h": 398, - "mimetype": "image/jpeg", - "size": 31037, - "w": 394, - }, - "url": "mxc://example.org/JWEIFJgwEIhweiWJE", - }, - "state_key": "", - }, - ), - ( - EventTypes.RoomEncryption, - { - "content": {"algorithm": "m.megolm.v1.aes-sha2"}, - "state_key": "", - }, - ), - ( - EventTypes.CanonicalAlias, - { - "content": {"alias": canonical_alias, "alt_aliases": []}, - "state_key": "", - }, - ), - ] - ) - - for event_type, event in self.room_state.items(): - self.helper.send_state( - self.room_id, event_type, event["content"], tok=self.tok, - ) - - # We expect the knock event to be included as well - self.room_state[EventTypes.Member] = { - "content": {"membership": "xyz.amorgan.knock", "displayname": "knocker"}, - "state_key": "@knocker:test", - } + self.expected_room_state = send_example_state_events_to_room( + self, hs, self.room_id, self.user_id + ) def test_knock_room_state(self): """Tests that /sync returns state from a room after knocking on it.""" @@ -432,38 +373,30 @@ def test_knock_room_state(self): ) self.assertEquals(200, channel.code, channel.result) + # We expect to see the knock event in the stripped room state later + self.expected_room_state[EventTypes.Member] = { + "content": {"membership": "xyz.amorgan.knock", "displayname": "knocker"}, + "state_key": "@knocker:test", + } + # Check that /sync includes stripped state from the room request, channel = self.make_request( "GET", self.url % self.next_batch, access_token=self.knocker_tok, ) self.assertEqual(channel.code, 200, channel.json_body) + # Extract the stripped room state events from /sync knock_entry = channel.json_body["rooms"]["xyz.amorgan.knock"] room_state_events = knock_entry[self.room_id]["knock_state"]["events"] - for event in room_state_events: - event_type = event["type"] - self.assertIn(event_type, self.room_state) - - # Check the state content matches - self.assertEquals(self.room_state[event_type]["content"], event["content"]) - - # Check the state key is correct - self.assertEqual( - self.room_state[event_type]["state_key"], event["state_key"] - ) - - # Ensure the event has been stripped - self.assertNotIn("signatures", event) - # Pop once we've found and processed a state event - self.room_state.pop(event_type) - - # Check that all expected state events were accounted for - self.assertEqual(len(self.room_state), 0) - - # Ensure that the knock membership event came last + # Validate that the knock membership event came last self.assertEqual(room_state_events[-1]["type"], EventTypes.Member) + # Validate the stripped room state events + check_knock_room_state_against_room_state( + self, room_state_events, self.expected_room_state + ) + class UnreadMessagesTestCase(unittest.HomeserverTestCase): servlets = [ From 51db6d3d1b4fdc862038d63c3a52dc3fd6370ad9 Mon Sep 17 00:00:00 2001 From: Andrew Morgan Date: Thu, 3 Dec 2020 18:59:36 +0000 Subject: [PATCH 063/112] Add license to test_knocking.py --- tests/federation/transport/test_knocking.py | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/tests/federation/transport/test_knocking.py b/tests/federation/transport/test_knocking.py index 0606fa8b81f2..013daaaf4b98 100644 --- a/tests/federation/transport/test_knocking.py +++ b/tests/federation/transport/test_knocking.py @@ -1,3 +1,17 @@ +# -*- coding: utf-8 -*- +# Copyright 2020 Matrix.org Federation C.I.C +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. from collections import OrderedDict from typing import Dict, List From 48bd084818a3107b8a84cc1829030d54c41941cd Mon Sep 17 00:00:00 2001 From: Andrew Morgan <1342360+anoadragon453@users.noreply.github.com> Date: Thu, 3 Dec 2020 19:25:22 +0000 Subject: [PATCH 064/112] Apply suggestions from code review Co-authored-by: Patrick Cloke --- tests/federation/transport/test_knocking.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/federation/transport/test_knocking.py b/tests/federation/transport/test_knocking.py index 013daaaf4b98..a6ad91ac3c34 100644 --- a/tests/federation/transport/test_knocking.py +++ b/tests/federation/transport/test_knocking.py @@ -167,7 +167,7 @@ def test_room_state_returned_when_knocking(self): def send_example_state_events_to_room( testcase: HomeserverTestCase, hs: "HomeServer", room_id: str, sender: str, ) -> OrderedDict: - """Adds some state a room. State events are those that should be sent to a knocking + """Adds some state to a room. State events are those that should be sent to a knocking user after they knock on the room, as well as some state that *shouldn't* be sent to the knocking user. @@ -267,7 +267,7 @@ def check_knock_room_state_against_room_state( knock_room_state: List[Dict], expected_room_state: Dict, ) -> None: - """Test a list of stripped room state events received over federation against an + """Test a list of stripped room state events received over federation against a dict of expected state events. Args: From 65c047867b3cd7ab65165e869c566393e5e0ade4 Mon Sep 17 00:00:00 2001 From: Andrew Morgan Date: Thu, 3 Dec 2020 19:37:45 +0000 Subject: [PATCH 065/112] Explain the purpose behind each step in 'prepare' --- tests/federation/transport/test_knocking.py | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/tests/federation/transport/test_knocking.py b/tests/federation/transport/test_knocking.py index a6ad91ac3c34..148bea60c8f9 100644 --- a/tests/federation/transport/test_knocking.py +++ b/tests/federation/transport/test_knocking.py @@ -51,7 +51,11 @@ class FederationKnockingTestCase(FederatingHomeserverTestCase): def prepare(self, reactor, clock, homeserver): self.store = homeserver.get_datastore() - # Have this homeserver auto-approve all event signature checking + # We're not going to be properly signing events as our remote homeserver is fake, + # therefore disable event signature checks. + # Note that these checks are not relevant to this test case. + + # Have this homeserver auto-approve all event signature checking. def approve_all_signature_checking(_, ev): return [succeed(ev[0])] @@ -59,16 +63,17 @@ def approve_all_signature_checking(_, ev): approve_all_signature_checking ) - # Have this homeserver skip event auth checks. - # - # While this prevent membership transistion checks, that is already - # tested elsewhere + # Have this homeserver skip event auth checks. This is necessary due to + # event auth checks ensuring that events were signed the sender's homeserver. event_auth.check = Mock(return_value=make_awaitable(None)) + # Bypass authentication checks for federation requests originating from our + # test homeserver. class Authenticator: def authenticate_request(self, request, content): return make_awaitable("other.example.com") + # Override federation ratelimits ratelimiter = FederationRateLimiter( clock, FederationRateLimitConfig( From 0353288925b934f59b0739da9f7e60fe31eff341 Mon Sep 17 00:00:00 2001 From: Andrew Morgan Date: Thu, 3 Dec 2020 19:44:42 +0000 Subject: [PATCH 066/112] Replace hardcoded join rule with constant --- tests/federation/transport/test_knocking.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/federation/transport/test_knocking.py b/tests/federation/transport/test_knocking.py index 148bea60c8f9..8d9501a9acbe 100644 --- a/tests/federation/transport/test_knocking.py +++ b/tests/federation/transport/test_knocking.py @@ -20,7 +20,7 @@ from twisted.internet.defer import succeed from synapse import event_auth -from synapse.api.constants import EventTypes +from synapse.api.constants import EventTypes, JoinRules from synapse.api.room_versions import RoomVersions from synapse.config.ratelimiting import FederationRateLimitConfig from synapse.events import builder @@ -216,10 +216,10 @@ def send_example_state_events_to_room( # We need to set the room's join rules to allow knocking ( EventTypes.JoinRules, - {"content": {"join_rule": "xyz.amorgan.knock"}, "state_key": ""}, + {"content": {"join_rule": JoinRules.KNOCK}, "state_key": ""}, ), # Below are state events that are to be stripped and sent to clients - (EventTypes.Name, {"content": {"name": "A cool room"}, "state_key": ""},), + (EventTypes.Name, {"content": {"name": "A cool room"}, "state_key": ""}), ( EventTypes.RoomAvatar, { From c9a5eb7292493230863a3050f04eaff7ccbe2945 Mon Sep 17 00:00:00 2001 From: Andrew Morgan Date: Thu, 3 Dec 2020 19:47:30 +0000 Subject: [PATCH 067/112] Replace instances of KNOCK_UNSTABLE_IDENTIFIER being used for membership and room versions --- tests/federation/transport/test_knocking.py | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/tests/federation/transport/test_knocking.py b/tests/federation/transport/test_knocking.py index 8d9501a9acbe..6299ee09b389 100644 --- a/tests/federation/transport/test_knocking.py +++ b/tests/federation/transport/test_knocking.py @@ -20,7 +20,7 @@ from twisted.internet.defer import succeed from synapse import event_auth -from synapse.api.constants import EventTypes, JoinRules +from synapse.api.constants import EventTypes, JoinRules, Membership from synapse.api.room_versions import RoomVersions from synapse.config.ratelimiting import FederationRateLimitConfig from synapse.events import builder @@ -104,7 +104,7 @@ def test_room_state_returned_when_knocking(self): room_id = self.helper.create_room_as( "u1", is_public=False, - room_version=KNOCK_UNSTABLE_IDENTIFIER, + room_version=RoomVersions.MSC2403_DEV.identifier, tok=user_token, ) @@ -132,9 +132,7 @@ def test_room_state_returned_when_knocking(self): self.assertEquals(knock_event["sender"], fake_knocking_user_id) self.assertEquals(knock_event["state_key"], fake_knocking_user_id) self.assertEquals(knock_event["type"], EventTypes.Member) - self.assertEquals( - knock_event["content"]["membership"], KNOCK_UNSTABLE_IDENTIFIER - ) + self.assertEquals(knock_event["content"]["membership"], Membership.KNOCK) # Turn the event json dict into a proper event. # We won't sign it properly, but that's OK as we stub out event auth in `prepare` @@ -199,7 +197,7 @@ def send_example_state_events_to_room( testcase.get_success( event_injection.inject_event( hs, - room_version=KNOCK_UNSTABLE_IDENTIFIER, + room_version=RoomVersions.MSC2403_DEV.identifier, room_id=room_id, sender=sender, type=secret_state_event_type, @@ -255,7 +253,7 @@ def send_example_state_events_to_room( testcase.get_success( event_injection.inject_event( hs, - room_version=KNOCK_UNSTABLE_IDENTIFIER, + room_version=RoomVersions.MSC2403_DEV.identifier, room_id=room_id, sender=sender, type=event_type, From a718381fb3c618aa3c9126f57fcd4ee2ec44c889 Mon Sep 17 00:00:00 2001 From: Andrew Morgan Date: Thu, 3 Dec 2020 20:02:08 +0000 Subject: [PATCH 068/112] Use a mixin instead of passing 'self' in to static methods Unfortunately the diff here came out a lot more awful than I would've liked, but really we're just moving these two methods into a mixin class and then using that instead of the two methods separately. I also ensured that the top-level SECRET_STATE_EVENT_TYPE was used instead of creating an identical variable. --- tests/federation/transport/test_knocking.py | 252 ++++++++++---------- tests/rest/client/v2_alpha/test_sync.py | 15 +- 2 files changed, 134 insertions(+), 133 deletions(-) diff --git a/tests/federation/transport/test_knocking.py b/tests/federation/transport/test_knocking.py index 6299ee09b389..52b6a4cee552 100644 --- a/tests/federation/transport/test_knocking.py +++ b/tests/federation/transport/test_knocking.py @@ -32,7 +32,7 @@ from synapse.util.ratelimitutils import FederationRateLimiter from tests.test_utils import event_injection, make_awaitable -from tests.unittest import FederatingHomeserverTestCase, HomeserverTestCase +from tests.unittest import FederatingHomeserverTestCase, TestCase # An identifier to use while MSC2304 is not in a stable release of the spec KNOCK_UNSTABLE_IDENTIFIER = "xyz.amorgan.knock" @@ -41,7 +41,9 @@ SECRET_STATE_EVENT_TYPE = "com.example.secret" -class FederationKnockingTestCase(FederatingHomeserverTestCase): +class FederationKnockingTestCase( + FederatingHomeserverTestCase, KnockingStrippedStateEventHelperMixin +): servlets = [ admin.register_servlets, room.register_servlets, @@ -109,8 +111,8 @@ def test_room_state_returned_when_knocking(self): ) # Update the join rules and add additional state to the room to check for later - expected_room_state = send_example_state_events_to_room( - self, self.hs, room_id, user_id + expected_room_state = self.send_example_state_events_to_room( + self.hs, room_id, user_id ) request, channel = self.make_request( @@ -162,145 +164,143 @@ def test_room_state_returned_when_knocking(self): room_state_events = channel.json_body["knock_state_events"] # Validate the stripped room state events - check_knock_room_state_against_room_state( - self, room_state_events, expected_room_state + self.check_knock_room_state_against_room_state( + room_state_events, expected_room_state ) -def send_example_state_events_to_room( - testcase: HomeserverTestCase, hs: "HomeServer", room_id: str, sender: str, -) -> OrderedDict: - """Adds some state to a room. State events are those that should be sent to a knocking - user after they knock on the room, as well as some state that *shouldn't* be sent - to the knocking user. - - Args: - testcase: The testcase that is currently active. - hs: The homeserver of the sender. - room_id: The ID of the room to send state into. - sender: The ID of the user to send state as. Must be in the room. - - Returns: - The OrderedDict of event types and content that a user is expected to see - after knocking on a room. - """ - # To set a canonical alias, we'll need to point an alias at the room first. - canonical_alias = "#fancy_alias:test" - testcase.get_success( - testcase.store.create_room_alias_association( - RoomAlias.from_string(canonical_alias), room_id, ["test"] - ) - ) +class KnockingStrippedStateEventHelperMixin(TestCase): + def send_example_state_events_to_room( + self, hs: "HomeServer", room_id: str, sender: str, + ) -> OrderedDict: + """Adds some state to a room. State events are those that should be sent to a knocking + user after they knock on the room, as well as some state that *shouldn't* be sent + to the knocking user. - # Send some state that we *don't* expect to be given to knocking users - secret_state_event_type = "com.example.secret" - testcase.get_success( - event_injection.inject_event( - hs, - room_version=RoomVersions.MSC2403_DEV.identifier, - room_id=room_id, - sender=sender, - type=secret_state_event_type, - state_key="", - content={"secret": "password"}, - ) - ) - - # We use an OrderedDict here to ensure that the knock membership appears last. - # Note that order only matters when sending stripped state to clients, not federated - # homeservers. - room_state = OrderedDict( - [ - # We need to set the room's join rules to allow knocking - ( - EventTypes.JoinRules, - {"content": {"join_rule": JoinRules.KNOCK}, "state_key": ""}, - ), - # Below are state events that are to be stripped and sent to clients - (EventTypes.Name, {"content": {"name": "A cool room"}, "state_key": ""}), - ( - EventTypes.RoomAvatar, - { - "content": { - "info": { - "h": 398, - "mimetype": "image/jpeg", - "size": 31037, - "w": 394, - }, - "url": "mxc://example.org/JWEIFJgwEIhweiWJE", - }, - "state_key": "", - }, - ), - ( - EventTypes.RoomEncryption, - {"content": {"algorithm": "m.megolm.v1.aes-sha2"}, "state_key": ""}, - ), - ( - EventTypes.CanonicalAlias, - { - "content": {"alias": canonical_alias, "alt_aliases": []}, - "state_key": "", - }, - ), - ] - ) + Args: + hs: The homeserver of the sender. + room_id: The ID of the room to send state into. + sender: The ID of the user to send state as. Must be in the room. - for event_type, event_dict in room_state.items(): - event_content, state_key = event_dict.values() + Returns: + The OrderedDict of event types and content that a user is expected to see + after knocking on a room. + """ + # To set a canonical alias, we'll need to point an alias at the room first. + canonical_alias = "#fancy_alias:test" + self.get_success( + self.store.create_room_alias_association( + RoomAlias.from_string(canonical_alias), room_id, ["test"] + ) + ) - testcase.get_success( + # Send some state that we *don't* expect to be given to knocking users + self.get_success( event_injection.inject_event( hs, room_version=RoomVersions.MSC2403_DEV.identifier, room_id=room_id, sender=sender, - type=event_type, - state_key=state_key, - content=event_content, + type=SECRET_STATE_EVENT_TYPE, + state_key="", + content={"secret": "password"}, ) ) - return room_state - - -def check_knock_room_state_against_room_state( - testcase: HomeserverTestCase, - knock_room_state: List[Dict], - expected_room_state: Dict, -) -> None: - """Test a list of stripped room state events received over federation against a - dict of expected state events. - - Args: - testcase: The testcase that is currently active. - knock_room_state: The list of room state that was received over federation. - expected_room_state: A dict containing the room state we expect to see in - `knock_room_state`. - """ - for event in knock_room_state: - event_type = event["type"] - testcase.assertIn(event_type, expected_room_state) - - # Check the state content matches - testcase.assertEquals( - expected_room_state[event_type]["content"], event["content"] + # We use an OrderedDict here to ensure that the knock membership appears last. + # Note that order only matters when sending stripped state to clients, not federated + # homeservers. + room_state = OrderedDict( + [ + # We need to set the room's join rules to allow knocking + ( + EventTypes.JoinRules, + {"content": {"join_rule": JoinRules.KNOCK}, "state_key": ""}, + ), + # Below are state events that are to be stripped and sent to clients + ( + EventTypes.Name, + {"content": {"name": "A cool room"}, "state_key": ""}, + ), + ( + EventTypes.RoomAvatar, + { + "content": { + "info": { + "h": 398, + "mimetype": "image/jpeg", + "size": 31037, + "w": 394, + }, + "url": "mxc://example.org/JWEIFJgwEIhweiWJE", + }, + "state_key": "", + }, + ), + ( + EventTypes.RoomEncryption, + {"content": {"algorithm": "m.megolm.v1.aes-sha2"}, "state_key": ""}, + ), + ( + EventTypes.CanonicalAlias, + { + "content": {"alias": canonical_alias, "alt_aliases": []}, + "state_key": "", + }, + ), + ] ) - # Check the state key is correct - testcase.assertEqual( - expected_room_state[event_type]["state_key"], event["state_key"] - ) + for event_type, event_dict in room_state.items(): + event_content, state_key = event_dict.values() + + self.get_success( + event_injection.inject_event( + hs, + room_version=RoomVersions.MSC2403_DEV.identifier, + room_id=room_id, + sender=sender, + type=event_type, + state_key=state_key, + content=event_content, + ) + ) + + return room_state + + def check_knock_room_state_against_room_state( + self, knock_room_state: List[Dict], expected_room_state: Dict, + ) -> None: + """Test a list of stripped room state events received over federation against a + dict of expected state events. + + Args: + knock_room_state: The list of room state that was received over federation. + expected_room_state: A dict containing the room state we expect to see in + `knock_room_state`. + """ + for event in knock_room_state: + event_type = event["type"] + self.assertIn(event_type, expected_room_state) + + # Check the state content matches + self.assertEquals( + expected_room_state[event_type]["content"], event["content"] + ) + + # Check the state key is correct + self.assertEqual( + expected_room_state[event_type]["state_key"], event["state_key"] + ) - # Ensure the event has been stripped - testcase.assertNotIn("signatures", event) + # Ensure the event has been stripped + self.assertNotIn("signatures", event) - # Pop once we've found and processed a state event - expected_room_state.pop(event_type) + # Pop once we've found and processed a state event + expected_room_state.pop(event_type) - # Check that all expected state events were accounted for - testcase.assertEqual(len(expected_room_state), 0) + # Check that all expected state events were accounted for + self.assertEqual(len(expected_room_state), 0) - # Ensure that no excess state was included - testcase.assertNotIn(SECRET_STATE_EVENT_TYPE, knock_room_state) + # Ensure that no excess state was included + self.assertNotIn(SECRET_STATE_EVENT_TYPE, knock_room_state) diff --git a/tests/rest/client/v2_alpha/test_sync.py b/tests/rest/client/v2_alpha/test_sync.py index 6c5b504f03f3..85dbcccd2c31 100644 --- a/tests/rest/client/v2_alpha/test_sync.py +++ b/tests/rest/client/v2_alpha/test_sync.py @@ -22,8 +22,7 @@ from tests import unittest from tests.federation.transport.test_knocking import ( - check_knock_room_state_against_room_state, - send_example_state_events_to_room, + KnockingStrippedStateEventHelperMixin, ) from tests.server import TimedOutException @@ -318,7 +317,9 @@ def test_sync_backwards_typing(self): self.make_request("GET", sync_url % (access_token, next_batch)) -class SyncKnockTestCase(unittest.HomeserverTestCase): +class SyncKnockTestCase( + unittest.HomeserverTestCase, KnockingStrippedStateEventHelperMixin +): servlets = [ synapse.rest.admin.register_servlets, login.register_servlets, @@ -358,8 +359,8 @@ def prepare(self, reactor, clock, hs): self.next_batch = channel.json_body["next_batch"] # Set up some room state to test with. - self.expected_room_state = send_example_state_events_to_room( - self, hs, self.room_id, self.user_id + self.expected_room_state = self.send_example_state_events_to_room( + hs, self.room_id, self.user_id ) def test_knock_room_state(self): @@ -393,8 +394,8 @@ def test_knock_room_state(self): self.assertEqual(room_state_events[-1]["type"], EventTypes.Member) # Validate the stripped room state events - check_knock_room_state_against_room_state( - self, room_state_events, self.expected_room_state + self.check_knock_room_state_against_room_state( + room_state_events, self.expected_room_state ) From fb9061189e8234cccfe51bec393ada6d06f9451c Mon Sep 17 00:00:00 2001 From: Andrew Morgan Date: Thu, 3 Dec 2020 20:05:16 +0000 Subject: [PATCH 069/112] Move KnockingStrippedStateEventHelperMixin to the top of test_knocking Kept this change in a separate commit in order to not make the last commit's diff even more confusing. --- tests/federation/transport/test_knocking.py | 256 ++++++++++---------- 1 file changed, 128 insertions(+), 128 deletions(-) diff --git a/tests/federation/transport/test_knocking.py b/tests/federation/transport/test_knocking.py index 52b6a4cee552..9d44a03d5022 100644 --- a/tests/federation/transport/test_knocking.py +++ b/tests/federation/transport/test_knocking.py @@ -41,134 +41,6 @@ SECRET_STATE_EVENT_TYPE = "com.example.secret" -class FederationKnockingTestCase( - FederatingHomeserverTestCase, KnockingStrippedStateEventHelperMixin -): - servlets = [ - admin.register_servlets, - room.register_servlets, - login.register_servlets, - ] - - def prepare(self, reactor, clock, homeserver): - self.store = homeserver.get_datastore() - - # We're not going to be properly signing events as our remote homeserver is fake, - # therefore disable event signature checks. - # Note that these checks are not relevant to this test case. - - # Have this homeserver auto-approve all event signature checking. - def approve_all_signature_checking(_, ev): - return [succeed(ev[0])] - - homeserver.get_federation_server()._check_sigs_and_hashes = ( - approve_all_signature_checking - ) - - # Have this homeserver skip event auth checks. This is necessary due to - # event auth checks ensuring that events were signed the sender's homeserver. - event_auth.check = Mock(return_value=make_awaitable(None)) - - # Bypass authentication checks for federation requests originating from our - # test homeserver. - class Authenticator: - def authenticate_request(self, request, content): - return make_awaitable("other.example.com") - - # Override federation ratelimits - ratelimiter = FederationRateLimiter( - clock, - FederationRateLimitConfig( - window_size=1, - sleep_limit=1, - sleep_msec=1, - reject_limit=1000, - concurrent_requests=1000, - ), - ) - federation_server.register_servlets( - homeserver, self.resource, Authenticator(), ratelimiter - ) - - return super().prepare(reactor, clock, homeserver) - - def test_room_state_returned_when_knocking(self): - """ - Tests that specific, stripped state events from a room are returned after - a remote homeserver successfully knocks on a local room. - """ - user_id = self.register_user("u1", "you the one") - user_token = self.login("u1", "you the one") - - fake_knocking_user_id = "@user:other.example.com" - - # Create a room with a room version that includes knocking - room_id = self.helper.create_room_as( - "u1", - is_public=False, - room_version=RoomVersions.MSC2403_DEV.identifier, - tok=user_token, - ) - - # Update the join rules and add additional state to the room to check for later - expected_room_state = self.send_example_state_events_to_room( - self.hs, room_id, user_id - ) - - request, channel = self.make_request( - "GET", - "/_matrix/federation/unstable/%s/make_knock/%s/%s" - % (KNOCK_UNSTABLE_IDENTIFIER, room_id, fake_knocking_user_id), - ) - self.assertEquals(200, channel.code, channel.result) - - # Note: We don't expect the knock membership event to be sent over federation as - # part of the stripped room state, as the knocking homeserver already has that - # event. It is only done for clients during /sync - - # Extract the generated knock event json - knock_event = channel.json_body["event"] - - # Check that the event has things we expect in it - self.assertEquals(knock_event["room_id"], room_id) - self.assertEquals(knock_event["sender"], fake_knocking_user_id) - self.assertEquals(knock_event["state_key"], fake_knocking_user_id) - self.assertEquals(knock_event["type"], EventTypes.Member) - self.assertEquals(knock_event["content"]["membership"], Membership.KNOCK) - - # Turn the event json dict into a proper event. - # We won't sign it properly, but that's OK as we stub out event auth in `prepare` - signed_knock_event = builder.create_local_event_from_event_dict( - self.clock, - self.hs.hostname, - self.hs.signing_key, - room_version=RoomVersions.MSC2403_DEV, - event_dict=knock_event, - ) - - # Convert our proper event back to json dict format - signed_knock_event_json = signed_knock_event.get_pdu_json( - self.clock.time_msec() - ) - - # Send the signed knock event into the room - request, channel = self.make_request( - "PUT", - "/_matrix/federation/unstable/%s/send_knock/%s/%s" - % (KNOCK_UNSTABLE_IDENTIFIER, room_id, signed_knock_event.event_id), - signed_knock_event_json, - ) - self.assertEquals(200, channel.code, channel.result) - - # Check that we got the stripped room state in return - room_state_events = channel.json_body["knock_state_events"] - - # Validate the stripped room state events - self.check_knock_room_state_against_room_state( - room_state_events, expected_room_state - ) - - class KnockingStrippedStateEventHelperMixin(TestCase): def send_example_state_events_to_room( self, hs: "HomeServer", room_id: str, sender: str, @@ -304,3 +176,131 @@ def check_knock_room_state_against_room_state( # Ensure that no excess state was included self.assertNotIn(SECRET_STATE_EVENT_TYPE, knock_room_state) + + +class FederationKnockingTestCase( + FederatingHomeserverTestCase, KnockingStrippedStateEventHelperMixin +): + servlets = [ + admin.register_servlets, + room.register_servlets, + login.register_servlets, + ] + + def prepare(self, reactor, clock, homeserver): + self.store = homeserver.get_datastore() + + # We're not going to be properly signing events as our remote homeserver is fake, + # therefore disable event signature checks. + # Note that these checks are not relevant to this test case. + + # Have this homeserver auto-approve all event signature checking. + def approve_all_signature_checking(_, ev): + return [succeed(ev[0])] + + homeserver.get_federation_server()._check_sigs_and_hashes = ( + approve_all_signature_checking + ) + + # Have this homeserver skip event auth checks. This is necessary due to + # event auth checks ensuring that events were signed the sender's homeserver. + event_auth.check = Mock(return_value=make_awaitable(None)) + + # Bypass authentication checks for federation requests originating from our + # test homeserver. + class Authenticator: + def authenticate_request(self, request, content): + return make_awaitable("other.example.com") + + # Override federation ratelimits + ratelimiter = FederationRateLimiter( + clock, + FederationRateLimitConfig( + window_size=1, + sleep_limit=1, + sleep_msec=1, + reject_limit=1000, + concurrent_requests=1000, + ), + ) + federation_server.register_servlets( + homeserver, self.resource, Authenticator(), ratelimiter + ) + + return super().prepare(reactor, clock, homeserver) + + def test_room_state_returned_when_knocking(self): + """ + Tests that specific, stripped state events from a room are returned after + a remote homeserver successfully knocks on a local room. + """ + user_id = self.register_user("u1", "you the one") + user_token = self.login("u1", "you the one") + + fake_knocking_user_id = "@user:other.example.com" + + # Create a room with a room version that includes knocking + room_id = self.helper.create_room_as( + "u1", + is_public=False, + room_version=RoomVersions.MSC2403_DEV.identifier, + tok=user_token, + ) + + # Update the join rules and add additional state to the room to check for later + expected_room_state = self.send_example_state_events_to_room( + self.hs, room_id, user_id + ) + + request, channel = self.make_request( + "GET", + "/_matrix/federation/unstable/%s/make_knock/%s/%s" + % (KNOCK_UNSTABLE_IDENTIFIER, room_id, fake_knocking_user_id), + ) + self.assertEquals(200, channel.code, channel.result) + + # Note: We don't expect the knock membership event to be sent over federation as + # part of the stripped room state, as the knocking homeserver already has that + # event. It is only done for clients during /sync + + # Extract the generated knock event json + knock_event = channel.json_body["event"] + + # Check that the event has things we expect in it + self.assertEquals(knock_event["room_id"], room_id) + self.assertEquals(knock_event["sender"], fake_knocking_user_id) + self.assertEquals(knock_event["state_key"], fake_knocking_user_id) + self.assertEquals(knock_event["type"], EventTypes.Member) + self.assertEquals(knock_event["content"]["membership"], Membership.KNOCK) + + # Turn the event json dict into a proper event. + # We won't sign it properly, but that's OK as we stub out event auth in `prepare` + signed_knock_event = builder.create_local_event_from_event_dict( + self.clock, + self.hs.hostname, + self.hs.signing_key, + room_version=RoomVersions.MSC2403_DEV, + event_dict=knock_event, + ) + + # Convert our proper event back to json dict format + signed_knock_event_json = signed_knock_event.get_pdu_json( + self.clock.time_msec() + ) + + # Send the signed knock event into the room + request, channel = self.make_request( + "PUT", + "/_matrix/federation/unstable/%s/send_knock/%s/%s" + % (KNOCK_UNSTABLE_IDENTIFIER, room_id, signed_knock_event.event_id), + signed_knock_event_json, + ) + self.assertEquals(200, channel.code, channel.result) + + # Check that we got the stripped room state in return + room_state_events = channel.json_body["knock_state_events"] + + # Validate the stripped room state events + self.check_knock_room_state_against_room_state( + room_state_events, expected_room_state + ) From c07413296d1953ef71b0c6903bb14db34641090c Mon Sep 17 00:00:00 2001 From: Andrew Morgan Date: Thu, 3 Dec 2020 20:42:24 +0000 Subject: [PATCH 070/112] Remove unnecessary check for excess knock state, and remove top-level var --- tests/federation/transport/test_knocking.py | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/tests/federation/transport/test_knocking.py b/tests/federation/transport/test_knocking.py index 9d44a03d5022..597b358de026 100644 --- a/tests/federation/transport/test_knocking.py +++ b/tests/federation/transport/test_knocking.py @@ -37,9 +37,6 @@ # An identifier to use while MSC2304 is not in a stable release of the spec KNOCK_UNSTABLE_IDENTIFIER = "xyz.amorgan.knock" -# An event type that we do not expect to be given to users knocking on a room -SECRET_STATE_EVENT_TYPE = "com.example.secret" - class KnockingStrippedStateEventHelperMixin(TestCase): def send_example_state_events_to_room( @@ -73,7 +70,7 @@ def send_example_state_events_to_room( room_version=RoomVersions.MSC2403_DEV.identifier, room_id=room_id, sender=sender, - type=SECRET_STATE_EVENT_TYPE, + type="com.example.secret", state_key="", content={"secret": "password"}, ) @@ -153,6 +150,9 @@ def check_knock_room_state_against_room_state( """ for event in knock_room_state: event_type = event["type"] + + # Check that this event type is one of those that we expected. + # Note: This will also check that no excess state was included self.assertIn(event_type, expected_room_state) # Check the state content matches @@ -174,9 +174,6 @@ def check_knock_room_state_against_room_state( # Check that all expected state events were accounted for self.assertEqual(len(expected_room_state), 0) - # Ensure that no excess state was included - self.assertNotIn(SECRET_STATE_EVENT_TYPE, knock_room_state) - class FederationKnockingTestCase( FederatingHomeserverTestCase, KnockingStrippedStateEventHelperMixin From e1ddbe279edf82e158c1263cbd7d85a542417357 Mon Sep 17 00:00:00 2001 From: Andrew Morgan Date: Fri, 4 Dec 2020 11:47:30 +0000 Subject: [PATCH 071/112] Extract values from dictionary in py3.5-friendly manner --- tests/federation/transport/test_knocking.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/federation/transport/test_knocking.py b/tests/federation/transport/test_knocking.py index 597b358de026..130ff9ff9bcf 100644 --- a/tests/federation/transport/test_knocking.py +++ b/tests/federation/transport/test_knocking.py @@ -121,7 +121,8 @@ def send_example_state_events_to_room( ) for event_type, event_dict in room_state.items(): - event_content, state_key = event_dict.values() + event_content = event_dict["content"] + state_key = event_dict["state_key"] self.get_success( event_injection.inject_event( From 4047067675d8270124c9827af72da85fde6fdab0 Mon Sep 17 00:00:00 2001 From: Andrew Morgan Date: Thu, 17 Dec 2020 15:38:49 +0000 Subject: [PATCH 072/112] Add config option to enable experimental knocking support --- docs/sample_config.yaml | 8 ++++++++ synapse/config/api.py | 10 ++++++++++ 2 files changed, 18 insertions(+) diff --git a/docs/sample_config.yaml b/docs/sample_config.yaml index 3b07211bf2e1..6d8666581c0f 100644 --- a/docs/sample_config.yaml +++ b/docs/sample_config.yaml @@ -1398,6 +1398,14 @@ metrics_flags: # - "m.room.encryption" # - "m.room.name" +# Uncomment to enable experimental room knocking support as defined by +# MSC2403. +# +# Note that the APIs used by this feature are unstable and will break in the +# future. +# +#msc2403_enabled: true + # A list of application service config files to use # diff --git a/synapse/config/api.py b/synapse/config/api.py index 0638ed8d2ecc..d55d71d6a6cd 100644 --- a/synapse/config/api.py +++ b/synapse/config/api.py @@ -14,6 +14,7 @@ # limitations under the License. from synapse.api.constants import EventTypes +from synapse.api.room_versions import KNOWN_ROOM_VERSIONS, RoomVersions from ._base import Config @@ -34,6 +35,7 @@ def read_config(self, config, **kwargs): self.room_invite_state_types = config.get( "room_invite_state_types", DEFAULT_ROOM_STATE_TYPES ) + self.msc2403_enabled = config.get("msc2403_enabled", False) def generate_config_section(cls, **kwargs): return """\ @@ -49,6 +51,14 @@ def generate_config_section(cls, **kwargs): # - "{RoomAvatar}" # - "{RoomEncryption}" # - "{Name}" + + # Uncomment to enable experimental room knocking support as defined by + # MSC2403. + # + # Note that the APIs used by this feature are unstable and will break in the + # future. + # + #msc2403_enabled: true """.format( **vars(EventTypes) ) From fbcf963bc0daa5e183fe69ff764833aaf7e92f86 Mon Sep 17 00:00:00 2001 From: Andrew Morgan Date: Thu, 17 Dec 2020 15:41:22 +0000 Subject: [PATCH 073/112] Only support knocking room version if knocking is enabled --- synapse/api/room_versions.py | 1 - synapse/config/api.py | 7 ++++++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/synapse/api/room_versions.py b/synapse/api/room_versions.py index ad26c7a6945f..328bf2890555 100644 --- a/synapse/api/room_versions.py +++ b/synapse/api/room_versions.py @@ -161,6 +161,5 @@ class RoomVersions: RoomVersions.V4, RoomVersions.V5, RoomVersions.V6, - RoomVersions.MSC2403_DEV, ) } # type: Dict[str, RoomVersion] diff --git a/synapse/config/api.py b/synapse/config/api.py index d55d71d6a6cd..e30892dfad61 100644 --- a/synapse/config/api.py +++ b/synapse/config/api.py @@ -35,7 +35,12 @@ def read_config(self, config, **kwargs): self.room_invite_state_types = config.get( "room_invite_state_types", DEFAULT_ROOM_STATE_TYPES ) - self.msc2403_enabled = config.get("msc2403_enabled", False) + msc2403_enabled = config.get("msc2403_enabled", False) + if msc2403_enabled: + # Enable the MSC2403 unstable room version + KNOWN_ROOM_VERSIONS.update( + {RoomVersions.MSC2403_DEV.identifier: RoomVersions.MSC2403_DEV} + ) def generate_config_section(cls, **kwargs): return """\ From abff21355e3e8ec5ffef17c1aabf86c536d96478 Mon Sep 17 00:00:00 2001 From: Andrew Morgan Date: Thu, 17 Dec 2020 16:57:14 +0000 Subject: [PATCH 074/112] Add config flag to tests --- tests/federation/transport/test_knocking.py | 3 ++- tests/rest/client/v2_alpha/test_sync.py | 2 ++ 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/tests/federation/transport/test_knocking.py b/tests/federation/transport/test_knocking.py index 130ff9ff9bcf..794aca66dc90 100644 --- a/tests/federation/transport/test_knocking.py +++ b/tests/federation/transport/test_knocking.py @@ -32,7 +32,7 @@ from synapse.util.ratelimitutils import FederationRateLimiter from tests.test_utils import event_injection, make_awaitable -from tests.unittest import FederatingHomeserverTestCase, TestCase +from tests.unittest import FederatingHomeserverTestCase, TestCase, override_config # An identifier to use while MSC2304 is not in a stable release of the spec KNOCK_UNSTABLE_IDENTIFIER = "xyz.amorgan.knock" @@ -227,6 +227,7 @@ def authenticate_request(self, request, content): return super().prepare(reactor, clock, homeserver) + @override_config({"msc2403_enabled": True}) def test_room_state_returned_when_knocking(self): """ Tests that specific, stripped state events from a room are returned after diff --git a/tests/rest/client/v2_alpha/test_sync.py b/tests/rest/client/v2_alpha/test_sync.py index 85dbcccd2c31..58311351efac 100644 --- a/tests/rest/client/v2_alpha/test_sync.py +++ b/tests/rest/client/v2_alpha/test_sync.py @@ -25,6 +25,7 @@ KnockingStrippedStateEventHelperMixin, ) from tests.server import TimedOutException +from tests.unittest import override_config class FilterTestCase(unittest.HomeserverTestCase): @@ -363,6 +364,7 @@ def prepare(self, reactor, clock, hs): hs, self.room_id, self.user_id ) + @override_config({"msc2403_enabled": True}) def test_knock_room_state(self): """Tests that /sync returns state from a room after knocking on it.""" # Knock on a room From 5362b4925390a8689d8279eaad39a1e3f991cfa2 Mon Sep 17 00:00:00 2001 From: Andrew Morgan Date: Thu, 17 Dec 2020 16:20:14 +0000 Subject: [PATCH 075/112] Remove unnecessary homeserver setup steps I thought these were necessary but apparently not so. They were originally cribbed from the federation catchip unit tests, and apparently not necessary for ours. I suppose because we do not hit the federation ratelimit, and the auth bypassing done via mocks in the test itself is already sufficient. --- tests/federation/transport/test_knocking.py | 24 --------------------- 1 file changed, 24 deletions(-) diff --git a/tests/federation/transport/test_knocking.py b/tests/federation/transport/test_knocking.py index 794aca66dc90..496a7865bfff 100644 --- a/tests/federation/transport/test_knocking.py +++ b/tests/federation/transport/test_knocking.py @@ -22,14 +22,11 @@ from synapse import event_auth from synapse.api.constants import EventTypes, JoinRules, Membership from synapse.api.room_versions import RoomVersions -from synapse.config.ratelimiting import FederationRateLimitConfig from synapse.events import builder -from synapse.federation.transport import server as federation_server from synapse.rest import admin from synapse.rest.client.v1 import login, room from synapse.server import HomeServer from synapse.types import RoomAlias -from synapse.util.ratelimitutils import FederationRateLimiter from tests.test_utils import event_injection, make_awaitable from tests.unittest import FederatingHomeserverTestCase, TestCase, override_config @@ -204,27 +201,6 @@ def approve_all_signature_checking(_, ev): # event auth checks ensuring that events were signed the sender's homeserver. event_auth.check = Mock(return_value=make_awaitable(None)) - # Bypass authentication checks for federation requests originating from our - # test homeserver. - class Authenticator: - def authenticate_request(self, request, content): - return make_awaitable("other.example.com") - - # Override federation ratelimits - ratelimiter = FederationRateLimiter( - clock, - FederationRateLimitConfig( - window_size=1, - sleep_limit=1, - sleep_msec=1, - reject_limit=1000, - concurrent_requests=1000, - ), - ) - federation_server.register_servlets( - homeserver, self.resource, Authenticator(), ratelimiter - ) - return super().prepare(reactor, clock, homeserver) @override_config({"msc2403_enabled": True}) From 69a820e46b726e84e77eabb6b0434422c1788a17 Mon Sep 17 00:00:00 2001 From: Andrew Morgan Date: Thu, 17 Dec 2020 17:04:00 +0000 Subject: [PATCH 076/112] Fix calls to make_request in knocking tests --- tests/federation/transport/test_knocking.py | 4 ++-- tests/rest/client/v2_alpha/test_sync.py | 6 +++--- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/tests/federation/transport/test_knocking.py b/tests/federation/transport/test_knocking.py index 496a7865bfff..f8fafaab6fd0 100644 --- a/tests/federation/transport/test_knocking.py +++ b/tests/federation/transport/test_knocking.py @@ -227,7 +227,7 @@ def test_room_state_returned_when_knocking(self): self.hs, room_id, user_id ) - request, channel = self.make_request( + channel = self.make_request( "GET", "/_matrix/federation/unstable/%s/make_knock/%s/%s" % (KNOCK_UNSTABLE_IDENTIFIER, room_id, fake_knocking_user_id), @@ -264,7 +264,7 @@ def test_room_state_returned_when_knocking(self): ) # Send the signed knock event into the room - request, channel = self.make_request( + channel = self.make_request( "PUT", "/_matrix/federation/unstable/%s/send_knock/%s/%s" % (KNOCK_UNSTABLE_IDENTIFIER, room_id, signed_knock_event.event_id), diff --git a/tests/rest/client/v2_alpha/test_sync.py b/tests/rest/client/v2_alpha/test_sync.py index 765f9afbf74f..cdcc6fceb39b 100644 --- a/tests/rest/client/v2_alpha/test_sync.py +++ b/tests/rest/client/v2_alpha/test_sync.py @@ -343,7 +343,7 @@ def prepare(self, reactor, clock, hs): self.knocker_tok = self.login("knocker", "monkey") # Perform an initial sync for the knocking user. - request, channel = self.make_request( + channel = self.make_request( "GET", self.url % self.next_batch, access_token=self.tok, ) self.assertEqual(channel.code, 200, channel.json_body) @@ -360,7 +360,7 @@ def prepare(self, reactor, clock, hs): def test_knock_room_state(self): """Tests that /sync returns state from a room after knocking on it.""" # Knock on a room - request, channel = self.make_request( + channel = self.make_request( "POST", "/_matrix/client/unstable/xyz.amorgan.knock/%s" % (self.room_id,), b"{}", @@ -375,7 +375,7 @@ def test_knock_room_state(self): } # Check that /sync includes stripped state from the room - request, channel = self.make_request( + channel = self.make_request( "GET", self.url % self.next_batch, access_token=self.knocker_tok, ) self.assertEqual(channel.code, 200, channel.json_body) From 6bdf465cd9a2184aa2b490f3a71012fa76711fcd Mon Sep 17 00:00:00 2001 From: Andrew Morgan Date: Wed, 30 Dec 2020 19:26:57 +0000 Subject: [PATCH 077/112] Fix FederationKnockingTestCase so that it no longer overrides event_auth.check for all tests We needed to disable event_auth checking for this test. However, the method in which we were doing so was disabling event_auth.check across all tests. Thus after this test ran, other unit tests failed in weird and wonderful ways. Take a different approach which acts on the homeserver created for this test, rather than mocking a file directly. --- tests/federation/transport/test_knocking.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/tests/federation/transport/test_knocking.py b/tests/federation/transport/test_knocking.py index f8fafaab6fd0..2cd6d70889bf 100644 --- a/tests/federation/transport/test_knocking.py +++ b/tests/federation/transport/test_knocking.py @@ -15,11 +15,8 @@ from collections import OrderedDict from typing import Dict, List -from mock import Mock - from twisted.internet.defer import succeed -from synapse import event_auth from synapse.api.constants import EventTypes, JoinRules, Membership from synapse.api.room_versions import RoomVersions from synapse.events import builder @@ -28,7 +25,7 @@ from synapse.server import HomeServer from synapse.types import RoomAlias -from tests.test_utils import event_injection, make_awaitable +from tests.test_utils import event_injection from tests.unittest import FederatingHomeserverTestCase, TestCase, override_config # An identifier to use while MSC2304 is not in a stable release of the spec @@ -199,7 +196,10 @@ def approve_all_signature_checking(_, ev): # Have this homeserver skip event auth checks. This is necessary due to # event auth checks ensuring that events were signed the sender's homeserver. - event_auth.check = Mock(return_value=make_awaitable(None)) + async def do_auth(origin, event, context, auth_events): + return context + + homeserver.get_federation_handler().do_auth = do_auth return super().prepare(reactor, clock, homeserver) From 91802c218e964925572cdc6594a73c6e701b9eef Mon Sep 17 00:00:00 2001 From: Andrew Morgan Date: Tue, 9 Feb 2021 13:06:48 +0000 Subject: [PATCH 078/112] Update config for enabling msc2403 to use experimental config section instead --- docs/sample_config.yaml | 8 -------- synapse/config/api.py | 15 --------------- synapse/config/experimental.py | 9 +++++++++ tests/federation/transport/test_knocking.py | 2 +- tests/rest/client/v2_alpha/test_sync.py | 2 +- 5 files changed, 11 insertions(+), 25 deletions(-) diff --git a/docs/sample_config.yaml b/docs/sample_config.yaml index 656eae3df5d9..4a4c77392c4e 100644 --- a/docs/sample_config.yaml +++ b/docs/sample_config.yaml @@ -1451,14 +1451,6 @@ metrics_flags: # - "m.room.encryption" # - "m.room.name" -# Uncomment to enable experimental room knocking support as defined by -# MSC2403. -# -# Note that the APIs used by this feature are unstable and will break in the -# future. -# -#msc2403_enabled: true - # A list of application service config files to use # diff --git a/synapse/config/api.py b/synapse/config/api.py index e30892dfad61..0638ed8d2ecc 100644 --- a/synapse/config/api.py +++ b/synapse/config/api.py @@ -14,7 +14,6 @@ # limitations under the License. from synapse.api.constants import EventTypes -from synapse.api.room_versions import KNOWN_ROOM_VERSIONS, RoomVersions from ._base import Config @@ -35,12 +34,6 @@ def read_config(self, config, **kwargs): self.room_invite_state_types = config.get( "room_invite_state_types", DEFAULT_ROOM_STATE_TYPES ) - msc2403_enabled = config.get("msc2403_enabled", False) - if msc2403_enabled: - # Enable the MSC2403 unstable room version - KNOWN_ROOM_VERSIONS.update( - {RoomVersions.MSC2403_DEV.identifier: RoomVersions.MSC2403_DEV} - ) def generate_config_section(cls, **kwargs): return """\ @@ -56,14 +49,6 @@ def generate_config_section(cls, **kwargs): # - "{RoomAvatar}" # - "{RoomEncryption}" # - "{Name}" - - # Uncomment to enable experimental room knocking support as defined by - # MSC2403. - # - # Note that the APIs used by this feature are unstable and will break in the - # future. - # - #msc2403_enabled: true """.format( **vars(EventTypes) ) diff --git a/synapse/config/experimental.py b/synapse/config/experimental.py index b1c1c51e4dcc..4b95310c58e5 100644 --- a/synapse/config/experimental.py +++ b/synapse/config/experimental.py @@ -13,6 +13,7 @@ # See the License for the specific language governing permissions and # limitations under the License. +from synapse.api.room_versions import KNOWN_ROOM_VERSIONS, RoomVersions from synapse.config._base import Config from synapse.types import JsonDict @@ -27,3 +28,11 @@ def read_config(self, config: JsonDict, **kwargs): # MSC2858 (multiple SSO identity providers) self.msc2858_enabled = experimental.get("msc2858_enabled", False) # type: bool + + # MSC2403 (room knocking) + self.msc2403_enabled = experimental.get("msc2403_enabled", False) # type: bool + if self.msc2403_enabled: + # Enable the MSC2403 unstable room version + KNOWN_ROOM_VERSIONS.update( + {RoomVersions.MSC2403_DEV.identifier: RoomVersions.MSC2403_DEV} + ) diff --git a/tests/federation/transport/test_knocking.py b/tests/federation/transport/test_knocking.py index 2cd6d70889bf..47316a5efc9b 100644 --- a/tests/federation/transport/test_knocking.py +++ b/tests/federation/transport/test_knocking.py @@ -203,7 +203,7 @@ async def do_auth(origin, event, context, auth_events): return super().prepare(reactor, clock, homeserver) - @override_config({"msc2403_enabled": True}) + @override_config({"experimental_features": {"msc2403_enabled": True}}) def test_room_state_returned_when_knocking(self): """ Tests that specific, stripped state events from a room are returned after diff --git a/tests/rest/client/v2_alpha/test_sync.py b/tests/rest/client/v2_alpha/test_sync.py index cdcc6fceb39b..170171d765bb 100644 --- a/tests/rest/client/v2_alpha/test_sync.py +++ b/tests/rest/client/v2_alpha/test_sync.py @@ -356,7 +356,7 @@ def prepare(self, reactor, clock, hs): hs, self.room_id, self.user_id ) - @override_config({"msc2403_enabled": True}) + @override_config({"experimental_features": {"msc2403_enabled": True}}) def test_knock_room_state(self): """Tests that /sync returns state from a room after knocking on it.""" # Knock on a room From 474a38bf3d374e426da4987d25d9a0dfe1ea827b Mon Sep 17 00:00:00 2001 From: Andrew Morgan Date: Wed, 17 Feb 2021 11:01:43 +0000 Subject: [PATCH 079/112] Send a ver query parameter for make_knock This informs the remote server of the room versions we support. If the room we're trying to knock on has a version that is not one of our supported room versions, the remote server will return an unsupported room version error. Noticed in https://github.com/matrix-org/matrix-doc/pull/2403#discussion_r577042144 --- synapse/federation/federation_server.py | 10 ++++++++-- synapse/federation/transport/server.py | 5 ++++- synapse/handlers/federation.py | 7 ++++++- 3 files changed, 18 insertions(+), 4 deletions(-) diff --git a/synapse/federation/federation_server.py b/synapse/federation/federation_server.py index fba1a59e23e2..c5e57b9d1177 100644 --- a/synapse/federation/federation_server.py +++ b/synapse/federation/federation_server.py @@ -569,7 +569,7 @@ async def on_send_leave_request(self, origin: str, content: JsonDict) -> dict: return {} async def on_make_knock_request( - self, origin: str, room_id: str, user_id: str, + self, origin: str, room_id: str, user_id: str, supported_versions: List[str] ) -> Dict[str, Union[EventBase, str]]: """We've received a /make_knock/ request, so we create a partial knock event for the room and hand that back, along with the room version, to the knocking @@ -580,16 +580,22 @@ async def on_make_knock_request( origin: The (verified) server name of the requesting server. room_id: The room to create the knock event in. user_id: The user to create the knock for. + supported_versions: The room versions supported by the requesting server. Returns: The partial knock event. """ origin_host, _ = parse_server_name(origin) await self.check_server_matches_acl(origin_host, room_id) - pdu = await self.handler.on_make_knock_request(origin, room_id, user_id) room_version = await self.store.get_room_version_id(room_id) + if room_version not in supported_versions: + logger.warning( + "Room version %s not in %s", room_version, supported_versions + ) + raise IncompatibleRoomVersionError(room_version=room_version) + pdu = await self.handler.on_make_knock_request(origin, room_id, user_id) time_now = self._clock.time_msec() return {"event": pdu.get_pdu_json(time_now), "room_version": room_version} diff --git a/synapse/federation/transport/server.py b/synapse/federation/transport/server.py index 46ee0dd6d404..cf10d4d63aef 100644 --- a/synapse/federation/transport/server.py +++ b/synapse/federation/transport/server.py @@ -549,7 +549,10 @@ class FederationMakeKnockServlet(BaseFederationServlet): PREFIX = FEDERATION_UNSTABLE_PREFIX + "/xyz.amorgan.knock" async def on_GET(self, origin, content, query, room_id, user_id): - content = await self.handler.on_make_knock_request(origin, room_id, user_id) + supported_versions = query.get(b"ver") + content = await self.handler.on_make_knock_request( + origin, room_id, user_id, supported_versions=supported_versions + ) return 200, content diff --git a/synapse/handlers/federation.py b/synapse/handlers/federation.py index f761d6217c03..359a7da563c6 100644 --- a/synapse/handlers/federation.py +++ b/synapse/handlers/federation.py @@ -1467,10 +1467,14 @@ async def do_knock( """ logger.debug("Knocking on room %s on behalf of user %s", room_id, knockee) + # Inform the remote server of the room versions we support + supported_room_versions = list(KNOWN_ROOM_VERSIONS.keys()) + # Ask the remote server to create a valid knock event for us. Once received, # we sign the event + params = {"ver": supported_room_versions} # type: Dict[str, Iterable[str]] origin, event, event_format_version = await self._make_and_verify_event( - target_hosts, room_id, knockee, Membership.KNOCK, content, + target_hosts, room_id, knockee, Membership.KNOCK, content, params=params ) # Record the room ID and its version so that we have a record of the room @@ -1871,6 +1875,7 @@ async def on_make_knock_request( raise SynapseError(403, "User not from origin", Codes.FORBIDDEN) room_version = await self.store.get_room_version_id(room_id) + builder = self.event_builder_factory.new( room_version, { From 09e444e0e2dc165f26a9c6b1c33315b66f5a21f8 Mon Sep 17 00:00:00 2001 From: Andrew Morgan Date: Wed, 17 Feb 2021 11:31:36 +0000 Subject: [PATCH 080/112] Lint with black==20.8b1 --- synapse/federation/federation_server.py | 11 ++++++++--- synapse/federation/transport/client.py | 6 +++++- synapse/handlers/federation.py | 6 +++++- synapse/handlers/message.py | 3 ++- synapse/handlers/room_member.py | 12 ++++++++++-- synapse/handlers/room_member_worker.py | 6 +++++- synapse/handlers/sync.py | 6 +++++- synapse/replication/http/membership.py | 14 +++++++++++--- synapse/rest/client/v2_alpha/knock.py | 5 ++++- tests/federation/transport/test_knocking.py | 9 +++++++-- tests/rest/client/v2_alpha/test_sync.py | 8 ++++++-- 11 files changed, 68 insertions(+), 18 deletions(-) diff --git a/synapse/federation/federation_server.py b/synapse/federation/federation_server.py index 6337accc7037..e84fad9d77ac 100644 --- a/synapse/federation/federation_server.py +++ b/synapse/federation/federation_server.py @@ -603,7 +603,10 @@ async def on_make_knock_request( return {"event": pdu.get_pdu_json(time_now), "room_version": room_version} async def on_send_knock_request( - self, origin: str, content: JsonDict, room_id: str, + self, + origin: str, + content: JsonDict, + room_id: str, ) -> Dict[str, List[JsonDict]]: """ We have received a knock event for a room. Verify and send the event into the room @@ -636,8 +639,10 @@ async def on_send_knock_request( # Retrieve stripped state events from the room and send them back to the remote # server. This will allow the remote server's clients to display information # related to the room while the knock request is pending. - stripped_room_state = await self.store.get_stripped_room_state_from_event_context( - event_context, DEFAULT_ROOM_STATE_TYPES + stripped_room_state = ( + await self.store.get_stripped_room_state_from_event_context( + event_context, DEFAULT_ROOM_STATE_TYPES + ) ) return {"knock_state_events": stripped_room_state} diff --git a/synapse/federation/transport/client.py b/synapse/federation/transport/client.py index 2cdda43568fa..a19d817d9134 100644 --- a/synapse/federation/transport/client.py +++ b/synapse/federation/transport/client.py @@ -309,7 +309,11 @@ async def send_leave_v2(self, destination, room_id, event_id, content): @log_function async def send_knock_v2( - self, destination: str, room_id: str, event_id: str, content: JsonDict, + self, + destination: str, + room_id: str, + event_id: str, + content: JsonDict, ) -> JsonDict: """ Sends a signed knock membership event to a remote server. This is the second diff --git a/synapse/handlers/federation.py b/synapse/handlers/federation.py index 5c8485c51bb1..4925f3032f23 100644 --- a/synapse/handlers/federation.py +++ b/synapse/handlers/federation.py @@ -1451,7 +1451,11 @@ async def do_invite_join( @log_function async def do_knock( - self, target_hosts: List[str], room_id: str, knockee: str, content: JsonDict, + self, + target_hosts: List[str], + room_id: str, + knockee: str, + content: JsonDict, ) -> Tuple[str, int]: """Sends the knock to the remote server. diff --git a/synapse/handlers/message.py b/synapse/handlers/message.py index 1aded280c779..f1c447f22c38 100644 --- a/synapse/handlers/message.py +++ b/synapse/handlers/message.py @@ -1181,7 +1181,8 @@ async def persist_and_notify_client_event( event.unsigned[ "knock_room_state" ] = await self.store.get_stripped_room_state_from_event_context( - context, DEFAULT_ROOM_STATE_TYPES, + context, + DEFAULT_ROOM_STATE_TYPES, ) if event.type == EventTypes.Redaction: diff --git a/synapse/handlers/room_member.py b/synapse/handlers/room_member.py index 548e72f282c3..ba39dd67c443 100644 --- a/synapse/handlers/room_member.py +++ b/synapse/handlers/room_member.py @@ -131,7 +131,11 @@ async def _remote_join( @abc.abstractmethod async def remote_knock( - self, remote_room_hosts: List[str], room_id: str, user: UserID, content: dict, + self, + remote_room_hosts: List[str], + room_id: str, + user: UserID, + content: dict, ) -> Tuple[str, int]: """Try and knock on a room that this server is not in @@ -1329,7 +1333,11 @@ async def _generate_local_out_of_band_leave( return result_event.event_id, result_event.internal_metadata.stream_ordering async def remote_knock( - self, remote_room_hosts: List[str], room_id: str, user: UserID, content: dict, + self, + remote_room_hosts: List[str], + room_id: str, + user: UserID, + content: dict, ) -> Tuple[str, int]: """Sends a knock to a room. Attempts to do so via one remote out of a given list. diff --git a/synapse/handlers/room_member_worker.py b/synapse/handlers/room_member_worker.py index 926d09f40c0a..428dae191447 100644 --- a/synapse/handlers/room_member_worker.py +++ b/synapse/handlers/room_member_worker.py @@ -112,7 +112,11 @@ async def remote_rescind_knock( return ret["event_id"], ret["stream_id"] async def remote_knock( - self, remote_room_hosts: List[str], room_id: str, user: UserID, content: dict, + self, + remote_room_hosts: List[str], + room_id: str, + user: UserID, + content: dict, ) -> Tuple[str, int]: """Sends a knock to a room. diff --git a/synapse/handlers/sync.py b/synapse/handlers/sync.py index fa6794734b16..905938224658 100644 --- a/synapse/handlers/sync.py +++ b/synapse/handlers/sync.py @@ -1744,7 +1744,11 @@ async def _get_rooms_changed( room_entries.append(entry) return _RoomChanges( - room_entries, invited, knocked, newly_joined_rooms, newly_left_rooms, + room_entries, + invited, + knocked, + newly_joined_rooms, + newly_left_rooms, ) async def _get_all_rooms( diff --git a/synapse/replication/http/membership.py b/synapse/replication/http/membership.py index ce526139567f..d1394478b0b9 100644 --- a/synapse/replication/http/membership.py +++ b/synapse/replication/http/membership.py @@ -145,7 +145,10 @@ async def _serialize_payload( # type: ignore } async def _handle_request( # type: ignore - self, request: Request, room_id: str, user_id: str, + self, + request: Request, + room_id: str, + user_id: str, ): content = parse_json_object_from_request(request) @@ -281,7 +284,9 @@ async def _serialize_payload( # type: ignore } async def _handle_request( # type: ignore - self, request: Request, knock_event_id: str, + self, + request: Request, + knock_event_id: str, ): content = parse_json_object_from_request(request) @@ -294,7 +299,10 @@ async def _handle_request( # type: ignore # hopefully we're now on the master, so this won't recurse! event_id, stream_id = await self.member_handler.remote_rescind_knock( - knock_event_id, txn_id, requester, event_content, + knock_event_id, + txn_id, + requester, + event_content, ) return 200, {"event_id": event_id, "stream_id": stream_id} diff --git a/synapse/rest/client/v2_alpha/knock.py b/synapse/rest/client/v2_alpha/knock.py index 8439da447e2c..75b7f665c9a5 100644 --- a/synapse/rest/client/v2_alpha/knock.py +++ b/synapse/rest/client/v2_alpha/knock.py @@ -53,7 +53,10 @@ def __init__(self, hs: "HomeServer"): self.auth = hs.get_auth() async def on_POST( - self, request: Request, room_identifier: str, txn_id: Optional[str] = None, + self, + request: Request, + room_identifier: str, + txn_id: Optional[str] = None, ) -> Tuple[int, JsonDict]: requester = await self.auth.get_user_by_req(request) diff --git a/tests/federation/transport/test_knocking.py b/tests/federation/transport/test_knocking.py index 47316a5efc9b..e3f105fa3803 100644 --- a/tests/federation/transport/test_knocking.py +++ b/tests/federation/transport/test_knocking.py @@ -34,7 +34,10 @@ class KnockingStrippedStateEventHelperMixin(TestCase): def send_example_state_events_to_room( - self, hs: "HomeServer", room_id: str, sender: str, + self, + hs: "HomeServer", + room_id: str, + sender: str, ) -> OrderedDict: """Adds some state to a room. State events are those that should be sent to a knocking user after they knock on the room, as well as some state that *shouldn't* be sent @@ -133,7 +136,9 @@ def send_example_state_events_to_room( return room_state def check_knock_room_state_against_room_state( - self, knock_room_state: List[Dict], expected_room_state: Dict, + self, + knock_room_state: List[Dict], + expected_room_state: Dict, ) -> None: """Test a list of stripped room state events received over federation against a dict of expected state events. diff --git a/tests/rest/client/v2_alpha/test_sync.py b/tests/rest/client/v2_alpha/test_sync.py index 9a50149c9108..7d41926bdaa9 100644 --- a/tests/rest/client/v2_alpha/test_sync.py +++ b/tests/rest/client/v2_alpha/test_sync.py @@ -344,7 +344,9 @@ def prepare(self, reactor, clock, hs): # Perform an initial sync for the knocking user. channel = self.make_request( - "GET", self.url % self.next_batch, access_token=self.tok, + "GET", + self.url % self.next_batch, + access_token=self.tok, ) self.assertEqual(channel.code, 200, channel.json_body) @@ -376,7 +378,9 @@ def test_knock_room_state(self): # Check that /sync includes stripped state from the room channel = self.make_request( - "GET", self.url % self.next_batch, access_token=self.knocker_tok, + "GET", + self.url % self.next_batch, + access_token=self.knocker_tok, ) self.assertEqual(channel.code, 200, channel.json_body) From 1475f2b7aa6dd88ed53d4735f0ba619454ea46eb Mon Sep 17 00:00:00 2001 From: Andrew Morgan Date: Wed, 17 Feb 2021 11:46:47 +0000 Subject: [PATCH 081/112] Decode room version query parameters --- synapse/federation/transport/server.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/synapse/federation/transport/server.py b/synapse/federation/transport/server.py index 65959d7b977f..9e06c48f48de 100644 --- a/synapse/federation/transport/server.py +++ b/synapse/federation/transport/server.py @@ -552,7 +552,8 @@ class FederationMakeKnockServlet(BaseFederationServlet): PREFIX = FEDERATION_UNSTABLE_PREFIX + "/xyz.amorgan.knock" async def on_GET(self, origin, content, query, room_id, user_id): - supported_versions = query.get(b"ver") + versions = query.get(b"ver") + supported_versions = [v.decode("utf-8") for v in versions] content = await self.handler.on_make_knock_request( origin, room_id, user_id, supported_versions=supported_versions ) From 4250f5048fcebfa94984bf3ea643bc8c460e7eee Mon Sep 17 00:00:00 2001 From: Andrew Morgan Date: Wed, 17 Feb 2021 12:26:58 +0000 Subject: [PATCH 082/112] Use parse_list_from_args instead of manually parsing --- synapse/federation/transport/server.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/synapse/federation/transport/server.py b/synapse/federation/transport/server.py index 9e06c48f48de..c5e1243286c7 100644 --- a/synapse/federation/transport/server.py +++ b/synapse/federation/transport/server.py @@ -15,7 +15,6 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. - import functools import logging import re @@ -34,6 +33,7 @@ parse_boolean_from_args, parse_integer_from_args, parse_json_object_from_request, + parse_list_from_args, parse_string_from_args, ) from synapse.logging.context import run_in_background @@ -552,8 +552,12 @@ class FederationMakeKnockServlet(BaseFederationServlet): PREFIX = FEDERATION_UNSTABLE_PREFIX + "/xyz.amorgan.knock" async def on_GET(self, origin, content, query, room_id, user_id): - versions = query.get(b"ver") - supported_versions = [v.decode("utf-8") for v in versions] + try: + # Retrieve the room versions the remote homeserver claims to support + supported_versions = parse_list_from_args(query, "ver", encoding="utf-8") + except KeyError: + raise SynapseError(400, "Missing required query parameter 'ver'") + content = await self.handler.on_make_knock_request( origin, room_id, user_id, supported_versions=supported_versions ) From c05b1501f11f6a168ce393a707fcb359356c8d63 Mon Sep 17 00:00:00 2001 From: Andrew Morgan Date: Wed, 17 Feb 2021 12:32:26 +0000 Subject: [PATCH 083/112] Update knocking test to send ver query parameter --- tests/federation/transport/test_knocking.py | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/tests/federation/transport/test_knocking.py b/tests/federation/transport/test_knocking.py index e3f105fa3803..ebf03c84d65a 100644 --- a/tests/federation/transport/test_knocking.py +++ b/tests/federation/transport/test_knocking.py @@ -234,8 +234,15 @@ def test_room_state_returned_when_knocking(self): channel = self.make_request( "GET", - "/_matrix/federation/unstable/%s/make_knock/%s/%s" - % (KNOCK_UNSTABLE_IDENTIFIER, room_id, fake_knocking_user_id), + "/_matrix/federation/unstable/%s/make_knock/%s/%s?ver=%s" + % ( + KNOCK_UNSTABLE_IDENTIFIER, + room_id, + fake_knocking_user_id, + # Inform the remote that we support the room version of the room we're + # knocking on + RoomVersions.MSC2403_DEV.identifier, + ), ) self.assertEquals(200, channel.code, channel.result) From 44b502644bc4e0473e3734349459bebd2cff0e17 Mon Sep 17 00:00:00 2001 From: Andrew Morgan Date: Fri, 26 Feb 2021 12:36:53 +0000 Subject: [PATCH 084/112] Allow knock->knock transitions --- synapse/event_auth.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/synapse/event_auth.py b/synapse/event_auth.py index eb1eda17a625..4e20851d7f52 100644 --- a/synapse/event_auth.py +++ b/synapse/event_auth.py @@ -355,8 +355,6 @@ def _is_membership_change_allowed( raise AuthError(403, "You cannot knock for other users") elif target_in_room: raise AuthError(403, "You cannot knock on a room you are already in") - elif caller_knocked: - raise AuthError(403, "You already have a pending knock for this room") elif caller_invited: raise AuthError(403, "You are already invited to this room") elif target_banned: From 5c6aee74123d661e69af2ed24b371ffb47658332 Mon Sep 17 00:00:00 2001 From: Andrew Morgan Date: Mon, 29 Mar 2021 17:12:17 +0100 Subject: [PATCH 085/112] Copyrig**n**t -> Copyright --- synapse/federation/federation_client.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/synapse/federation/federation_client.py b/synapse/federation/federation_client.py index 6b1f7dcb8910..874129c29283 100644 --- a/synapse/federation/federation_client.py +++ b/synapse/federation/federation_client.py @@ -1,7 +1,7 @@ # -*- coding: utf-8 -*- # Copyright 2015, 2016 OpenMarket Ltd -# Copyrignt 2020 Sorunome -# Copyrignt 2020 The Matrix.org Foundation C.I.C. +# Copyright 2020 Sorunome +# Copyright 2020 The Matrix.org Foundation C.I.C. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. From 37a674d635ddca87e528113aa28953055058ff13 Mon Sep 17 00:00:00 2001 From: Andrew Morgan Date: Thu, 1 Apr 2021 18:30:06 +0100 Subject: [PATCH 086/112] Some fixes off the back of the merge --- synapse/replication/http/membership.py | 4 ++-- synapse/rest/admin/rooms.py | 7 ++++--- synapse/rest/client/v2_alpha/knock.py | 7 +++++-- synapse/rest/client/v2_alpha/sync.py | 2 +- synapse/storage/databases/main/events_worker.py | 2 +- 5 files changed, 13 insertions(+), 9 deletions(-) diff --git a/synapse/replication/http/membership.py b/synapse/replication/http/membership.py index b8491ca80e66..2812ac12fcd9 100644 --- a/synapse/replication/http/membership.py +++ b/synapse/replication/http/membership.py @@ -146,7 +146,7 @@ async def _serialize_payload( # type: ignore async def _handle_request( # type: ignore self, - request: Request, + request: SynapseRequest, room_id: str, user_id: str, ): @@ -284,7 +284,7 @@ async def _serialize_payload( # type: ignore async def _handle_request( # type: ignore self, - request: Request, + request: SynapseRequest, knock_event_id: str, ): content = parse_json_object_from_request(request) diff --git a/synapse/rest/admin/rooms.py b/synapse/rest/admin/rooms.py index c6c2df2062c3..cfe1bebb91b2 100644 --- a/synapse/rest/admin/rooms.py +++ b/synapse/rest/admin/rooms.py @@ -14,7 +14,7 @@ # limitations under the License. import logging from http import HTTPStatus -from typing import TYPE_CHECKING, Optional, Tuple +from typing import TYPE_CHECKING, List, Optional, Tuple from urllib import parse as urlparse from synapse.api.constants import EventTypes, JoinRules, Membership @@ -25,7 +25,6 @@ assert_params_in_dict, parse_integer, parse_json_object_from_request, - parse_list_from_args, parse_string, ) from synapse.http.site import SynapseRequest @@ -410,7 +409,9 @@ async def on_POST( # Get the room ID from the identifier. try: - remote_room_hosts = parse_list_from_args(request.args, "server_name") + remote_room_hosts = [ + x.decode("ascii") for x in request.args[b"server_name"] + ] # type: Optional[List[str]] except Exception: remote_room_hosts = None room_id, remote_room_hosts = await self.resolve_room_id( diff --git a/synapse/rest/client/v2_alpha/knock.py b/synapse/rest/client/v2_alpha/knock.py index 75b7f665c9a5..f344666a59f0 100644 --- a/synapse/rest/client/v2_alpha/knock.py +++ b/synapse/rest/client/v2_alpha/knock.py @@ -25,6 +25,7 @@ parse_json_object_from_request, parse_list_from_args, ) +from synapse.http.site import SynapseRequest from synapse.logging.opentracing import set_tag from synapse.rest.client.transactions import HttpTransactionCache from synapse.types import JsonDict, RoomAlias, RoomID @@ -54,7 +55,7 @@ def __init__(self, hs: "HomeServer"): async def on_POST( self, - request: Request, + request: SynapseRequest, room_identifier: str, txn_id: Optional[str] = None, ) -> Tuple[int, JsonDict]: @@ -68,7 +69,9 @@ async def on_POST( if RoomID.is_valid(room_identifier): room_id = room_identifier try: - remote_room_hosts = parse_list_from_args(request.args, "server_name") + remote_room_hosts = parse_list_from_args( + request.args, "server_name" # type: ignore + ) except KeyError: remote_room_hosts = None elif RoomAlias.is_valid(room_identifier): diff --git a/synapse/rest/client/v2_alpha/sync.py b/synapse/rest/client/v2_alpha/sync.py index fb9e82abe857..c01ba14cd29c 100644 --- a/synapse/rest/client/v2_alpha/sync.py +++ b/synapse/rest/client/v2_alpha/sync.py @@ -14,7 +14,7 @@ # limitations under the License. import itertools import logging -from typing import TYPE_CHECKING, Tuple, Any, Callable, Dict, List +from typing import TYPE_CHECKING, Any, Callable, Dict, List, Tuple from synapse.api.constants import Membership, PresenceState from synapse.api.errors import Codes, StoreError, SynapseError diff --git a/synapse/storage/databases/main/events_worker.py b/synapse/storage/databases/main/events_worker.py index 572711d3d11a..c00780969f6e 100644 --- a/synapse/storage/databases/main/events_worker.py +++ b/synapse/storage/databases/main/events_worker.py @@ -544,7 +544,7 @@ def _get_events_from_cache(self, events, allow_rejected, update_metrics=True): async def get_stripped_room_state_from_event_context( self, context: EventContext, - state_types_to_include: Container[EventTypes], + state_types_to_include: Container[str], membership_user_id: Optional[str] = None, ) -> List[JsonDict]: """ From 6c72538b358d64088cd185eeea0a933727513b84 Mon Sep 17 00:00:00 2001 From: Andrew Morgan Date: Thu, 1 Apr 2021 18:31:42 +0100 Subject: [PATCH 087/112] Standardise room version variable names --- synapse/api/room_versions.py | 23 +++++++++++---------- synapse/config/experimental.py | 2 +- tests/federation/transport/test_knocking.py | 10 ++++----- 3 files changed, 18 insertions(+), 17 deletions(-) diff --git a/synapse/api/room_versions.py b/synapse/api/room_versions.py index 04218f83af74..28b453eeb4db 100644 --- a/synapse/api/room_versions.py +++ b/synapse/api/room_versions.py @@ -73,7 +73,7 @@ class RoomVersion: msc3083_join_rules = attr.ib(type=bool) # MSC2403: Allows join_rules to be set to 'knock', changes auth rules to allow sending # m.room.membership event with membership 'knock'. - allow_knocking = attr.ib(type=bool) + msc2403_knocking = attr.ib(type=bool) class RoomVersions: @@ -88,7 +88,7 @@ class RoomVersions: limit_notifications_power_levels=False, msc2176_redaction_rules=False, msc3083_join_rules=False, - allow_knocking=False, + msc2403_knocking=False, ) V2 = RoomVersion( "2", @@ -101,7 +101,7 @@ class RoomVersions: limit_notifications_power_levels=False, msc2176_redaction_rules=False, msc3083_join_rules=False, - allow_knocking=False, + msc2403_knocking=False, ) V3 = RoomVersion( "3", @@ -114,7 +114,7 @@ class RoomVersions: limit_notifications_power_levels=False, msc2176_redaction_rules=False, msc3083_join_rules=False, - allow_knocking=False, + msc2403_knocking=False, ) V4 = RoomVersion( "4", @@ -127,7 +127,7 @@ class RoomVersions: limit_notifications_power_levels=False, msc2176_redaction_rules=False, msc3083_join_rules=False, - allow_knocking=False, + msc2403_knocking=False, ) V5 = RoomVersion( "5", @@ -140,7 +140,7 @@ class RoomVersions: limit_notifications_power_levels=False, msc2176_redaction_rules=False, msc3083_join_rules=False, - allow_knocking=False, + msc2403_knocking=False, ) V6 = RoomVersion( "6", @@ -153,7 +153,7 @@ class RoomVersions: limit_notifications_power_levels=True, msc2176_redaction_rules=False, msc3083_join_rules=False, - allow_knocking=False, + msc2403_knocking=False, ) MSC2176 = RoomVersion( "org.matrix.msc2176", @@ -166,7 +166,7 @@ class RoomVersions: limit_notifications_power_levels=True, msc2176_redaction_rules=True, msc3083_join_rules=False, - allow_knocking=False, + msc2403_knocking=False, ) MSC3083 = RoomVersion( "org.matrix.msc3083", @@ -179,9 +179,9 @@ class RoomVersions: limit_notifications_power_levels=True, msc2176_redaction_rules=False, msc3083_join_rules=True, - allow_knocking=False, + msc2403_knocking=False, ) - MSC2403_DEV = RoomVersion( + MSC2403 = RoomVersion( "xyz.amorgan.knock", RoomDisposition.UNSTABLE, EventFormatVersions.V3, @@ -192,7 +192,7 @@ class RoomVersions: limit_notifications_power_levels=True, msc2176_redaction_rules=False, msc3083_join_rules=False, - allow_knocking=True, + msc2403_knocking=True, ) @@ -208,4 +208,5 @@ class RoomVersions: RoomVersions.MSC2176, ) # Note that we do not include MSC3083 here unless it is enabled in the config. + # Note that we do not include MSC2043 here unless it is enabled in the config. } # type: Dict[str, RoomVersion] diff --git a/synapse/config/experimental.py b/synapse/config/experimental.py index 310ee59c8c2c..12770551d6c5 100644 --- a/synapse/config/experimental.py +++ b/synapse/config/experimental.py @@ -42,5 +42,5 @@ def read_config(self, config: JsonDict, **kwargs): if self.msc2403_enabled: # Enable the MSC2403 unstable room version KNOWN_ROOM_VERSIONS.update( - {RoomVersions.MSC2403_DEV.identifier: RoomVersions.MSC2403_DEV} + {RoomVersions.MSC2403.identifier: RoomVersions.MSC2403} ) diff --git a/tests/federation/transport/test_knocking.py b/tests/federation/transport/test_knocking.py index ebf03c84d65a..e76a344bc6ee 100644 --- a/tests/federation/transport/test_knocking.py +++ b/tests/federation/transport/test_knocking.py @@ -64,7 +64,7 @@ def send_example_state_events_to_room( self.get_success( event_injection.inject_event( hs, - room_version=RoomVersions.MSC2403_DEV.identifier, + room_version=RoomVersions.MSC2403.identifier, room_id=room_id, sender=sender, type="com.example.secret", @@ -124,7 +124,7 @@ def send_example_state_events_to_room( self.get_success( event_injection.inject_event( hs, - room_version=RoomVersions.MSC2403_DEV.identifier, + room_version=RoomVersions.MSC2403.identifier, room_id=room_id, sender=sender, type=event_type, @@ -223,7 +223,7 @@ def test_room_state_returned_when_knocking(self): room_id = self.helper.create_room_as( "u1", is_public=False, - room_version=RoomVersions.MSC2403_DEV.identifier, + room_version=RoomVersions.MSC2403.identifier, tok=user_token, ) @@ -241,7 +241,7 @@ def test_room_state_returned_when_knocking(self): fake_knocking_user_id, # Inform the remote that we support the room version of the room we're # knocking on - RoomVersions.MSC2403_DEV.identifier, + RoomVersions.MSC2403.identifier, ), ) self.assertEquals(200, channel.code, channel.result) @@ -266,7 +266,7 @@ def test_room_state_returned_when_knocking(self): self.clock, self.hs.hostname, self.hs.signing_key, - room_version=RoomVersions.MSC2403_DEV, + room_version=RoomVersions.MSC2403, event_dict=knock_event, ) From 024230cd159e7a94078e9290c060a54812472dbe Mon Sep 17 00:00:00 2001 From: Andrew Morgan Date: Thu, 1 Apr 2021 18:48:52 +0100 Subject: [PATCH 088/112] Use room_prejoin_state types --- synapse/federation/federation_server.py | 7 ++++--- synapse/handlers/message.py | 7 +++---- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/synapse/federation/federation_server.py b/synapse/federation/federation_server.py index 87c09d0f69f0..e44dabcbc3f7 100644 --- a/synapse/federation/federation_server.py +++ b/synapse/federation/federation_server.py @@ -47,7 +47,6 @@ ) from synapse.api.ratelimiting import Ratelimiter from synapse.api.room_versions import KNOWN_ROOM_VERSIONS -from synapse.config.api import DEFAULT_ROOM_STATE_TYPES from synapse.events import EventBase from synapse.federation.federation_base import FederationBase, event_from_pdu_json from synapse.federation.persistence import TransactionActions @@ -138,9 +137,11 @@ def __init__(self, hs: "HomeServer"): ) # type: ResponseCache[Tuple[str, str]] self._federation_metrics_domains = ( - hs.get_config().federation.federation_metrics_domains + hs.config.federation.federation_metrics_domains ) + self._room_prejoin_state_types = hs.config.api.room_prejoin_state + async def on_backfill_request( self, origin: str, room_id: str, versions: List[str], limit: int ) -> Tuple[int, Dict[str, Any]]: @@ -659,7 +660,7 @@ async def on_send_knock_request( # related to the room while the knock request is pending. stripped_room_state = ( await self.store.get_stripped_room_state_from_event_context( - event_context, DEFAULT_ROOM_STATE_TYPES + event_context, self._room_prejoin_state_types ) ) return {"knock_state_events": stripped_room_state} diff --git a/synapse/handlers/message.py b/synapse/handlers/message.py index 83512e55524d..a8d3fc2cf4f0 100644 --- a/synapse/handlers/message.py +++ b/synapse/handlers/message.py @@ -41,7 +41,6 @@ ) from synapse.api.room_versions import KNOWN_ROOM_VERSIONS, RoomVersions from synapse.api.urls import ConsentURIBuilder -from synapse.config.api import DEFAULT_ROOM_STATE_TYPES from synapse.events import EventBase from synapse.events.builder import EventBuilder from synapse.events.snapshot import EventContext @@ -387,7 +386,7 @@ def __init__(self, hs: "HomeServer"): self._events_shard_config = self.config.worker.events_shard_config self._instance_name = hs.get_instance_name() - self.room_invite_state_types = self.hs.config.api.room_prejoin_state + self.room_prejoin_state_types = self.hs.config.api.room_prejoin_state self.membership_types_to_include_profile_data_in = { Membership.JOIN, @@ -1166,7 +1165,7 @@ async def persist_and_notify_client_event( "invite_room_state" ] = await self.store.get_stripped_room_state_from_event_context( context, - self.room_invite_state_types, + self.room_prejoin_state_types, membership_user_id=event.sender, ) @@ -1189,7 +1188,7 @@ async def persist_and_notify_client_event( "knock_room_state" ] = await self.store.get_stripped_room_state_from_event_context( context, - DEFAULT_ROOM_STATE_TYPES, + self.room_prejoin_state_types, ) if event.type == EventTypes.Redaction: From e7f250a3435522f1d506e8684a06b875985209d5 Mon Sep 17 00:00:00 2001 From: Andrew Morgan Date: Thu, 1 Apr 2021 19:05:11 +0100 Subject: [PATCH 089/112] update schema version of table migration --- .../11add_knock_members_to_stats.sql} | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) rename synapse/storage/databases/main/schema/delta/{58/24add_knock_members_to_stats.sql => 59/11add_knock_members_to_stats.sql} (77%) diff --git a/synapse/storage/databases/main/schema/delta/58/24add_knock_members_to_stats.sql b/synapse/storage/databases/main/schema/delta/59/11add_knock_members_to_stats.sql similarity index 77% rename from synapse/storage/databases/main/schema/delta/58/24add_knock_members_to_stats.sql rename to synapse/storage/databases/main/schema/delta/59/11add_knock_members_to_stats.sql index 658f55a38452..c1e740f848eb 100644 --- a/synapse/storage/databases/main/schema/delta/58/24add_knock_members_to_stats.sql +++ b/synapse/storage/databases/main/schema/delta/59/11add_knock_members_to_stats.sql @@ -13,5 +13,5 @@ * limitations under the License. */ -ALTER TABLE room_stats_current ADD knocked_members INT NOT NULL DEFAULT '0'; -ALTER TABLE room_stats_historical ADD knocked_members BIGINT NOT NULL DEFAULT '0'; +ALTER TABLE room_stats_current ADD COLUMN knocked_members INT NOT NULL DEFAULT '0'; +ALTER TABLE room_stats_historical ADD COLUMN knocked_members BIGINT NOT NULL DEFAULT '0'; From c838fd4b1f1c1bd0a2e6f368183053045681005c Mon Sep 17 00:00:00 2001 From: Andrew Morgan Date: Fri, 4 Jun 2021 16:40:40 +0100 Subject: [PATCH 090/112] Add m.room.create as an expected event type to stripped state We added m.room.create as one of the state event types to return by default in https://github.com/matrix-org/synapse/issues/9448 in order to allow inspecting the 'type' of a room (which is stored in the create event) without first needing to join a room. --- tests/federation/transport/test_knocking.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/tests/federation/transport/test_knocking.py b/tests/federation/transport/test_knocking.py index e76a344bc6ee..718316134f16 100644 --- a/tests/federation/transport/test_knocking.py +++ b/tests/federation/transport/test_knocking.py @@ -133,6 +133,16 @@ def send_example_state_events_to_room( ) ) + # Finally, we expect to see the m.room.create event of the room as part of the + # stripped state. We don't need to inject this event though. + room_state[EventTypes.Create] = { + "content": { + "creator": sender, + "room_version": RoomVersions.MSC2403.identifier, + }, + "state_key": "", + } + return room_state def check_knock_room_state_against_room_state( From cfccaf43f48a18a1e73c641a622671ed3ec4e2f3 Mon Sep 17 00:00:00 2001 From: Andrew Morgan Date: Mon, 7 Jun 2021 18:27:40 +0100 Subject: [PATCH 091/112] Persist incoming knock event in addition to computing its context --- synapse/handlers/federation.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/synapse/handlers/federation.py b/synapse/handlers/federation.py index f693cac859b7..2d313140e6d5 100644 --- a/synapse/handlers/federation.py +++ b/synapse/handlers/federation.py @@ -2074,6 +2074,8 @@ async def on_send_knock_request( context = await self.state_handler.compute_event_context(event) + await self._auth_and_persist_event(origin, event, context) + event_allowed = await self.third_party_event_rules.check_event_allowed( event, context ) From 98367bab46c840f72710d021690e70a02ce10d70 Mon Sep 17 00:00:00 2001 From: Andrew Morgan Date: Mon, 7 Jun 2021 19:39:37 +0100 Subject: [PATCH 092/112] Fix test after do_auth function rename do_auth was renamed to _check_event_auth in #9800. --- tests/federation/transport/test_knocking.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/tests/federation/transport/test_knocking.py b/tests/federation/transport/test_knocking.py index 718316134f16..3ef41ddd4f72 100644 --- a/tests/federation/transport/test_knocking.py +++ b/tests/federation/transport/test_knocking.py @@ -210,11 +210,13 @@ def approve_all_signature_checking(_, ev): ) # Have this homeserver skip event auth checks. This is necessary due to - # event auth checks ensuring that events were signed the sender's homeserver. - async def do_auth(origin, event, context, auth_events): + # event auth checks ensuring that events were signed by the sender's homeserver. + async def _check_event_auth( + origin, event, context, state, auth_events, backfilled + ): return context - homeserver.get_federation_handler().do_auth = do_auth + homeserver.get_federation_handler()._check_event_auth = _check_event_auth return super().prepare(reactor, clock, homeserver) From ebcdd0d2d9fdbfa6858c4a18d03b6fdde7ee9f4a Mon Sep 17 00:00:00 2001 From: Andrew Morgan <1342360+anoadragon453@users.noreply.github.com> Date: Tue, 8 Jun 2021 10:47:01 +0100 Subject: [PATCH 093/112] Apply suggestions from code review Co-authored-by: Patrick Cloke --- synapse/federation/transport/client.py | 4 +--- synapse/handlers/federation.py | 4 +--- synapse/handlers/room_member_worker.py | 3 +-- synapse/handlers/stats.py | 3 +-- synapse/replication/http/membership.py | 2 +- 5 files changed, 5 insertions(+), 11 deletions(-) diff --git a/synapse/federation/transport/client.py b/synapse/federation/transport/client.py index 600bc75e4dae..0af48ff6edd6 100644 --- a/synapse/federation/transport/client.py +++ b/synapse/federation/transport/client.py @@ -1,7 +1,5 @@ -# Copyright 2014-2016 OpenMarket Ltd -# Copyright 2018 New Vector Ltd +# Copyright 2014-2021 The Matrix.org Foundation C.I.C. # Copyright 2020 Sorunome -# Copyright 2020 The Matrix.org Foundation C.I.C. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/synapse/handlers/federation.py b/synapse/handlers/federation.py index 2d313140e6d5..f1323c44c150 100644 --- a/synapse/handlers/federation.py +++ b/synapse/handlers/federation.py @@ -1,6 +1,4 @@ -# Copyright 2014-2016 OpenMarket Ltd -# Copyright 2017-2018 New Vector Ltd -# Copyright 2019-2020 The Matrix.org Foundation C.I.C. +# Copyright 2014-2021 The Matrix.org Foundation C.I.C. # Copyright 2020 Sorunome # # Licensed under the Apache License, Version 2.0 (the "License"); diff --git a/synapse/handlers/room_member_worker.py b/synapse/handlers/room_member_worker.py index 3fdad8cdd8e7..221552a2a64b 100644 --- a/synapse/handlers/room_member_worker.py +++ b/synapse/handlers/room_member_worker.py @@ -1,5 +1,4 @@ -# Copyright 2018 New Vector Ltd -# Copyright 2020 The Matrix.org Foundation C.I.C. +# Copyright 2018-2021 The Matrix.org Foundation C.I.C. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/synapse/handlers/stats.py b/synapse/handlers/stats.py index 761e956b6e6a..4e45d1da573c 100644 --- a/synapse/handlers/stats.py +++ b/synapse/handlers/stats.py @@ -1,6 +1,5 @@ -# Copyright 2018 New Vector Ltd +# Copyright 2018-2021 The Matrix.org Foundation C.I.C. # Copyright 2020 Sorunome -# Copyright 2020 The Matrix.org Foundation C.I.C. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/synapse/replication/http/membership.py b/synapse/replication/http/membership.py index bd03030b4bf4..ea97ed5df9e1 100644 --- a/synapse/replication/http/membership.py +++ b/synapse/replication/http/membership.py @@ -115,7 +115,7 @@ class ReplicationRemoteKnockRestServlet(ReplicationEndpoint): PATH_ARGS = ("room_id", "user_id") def __init__(self, hs): - super().__init__(hs) + super().__init__(hs: "HomeServer") self.federation_handler = hs.get_federation_handler() self.store = hs.get_datastore() From 662fe987bcd1d02d93e64d166a36d4a4f0d00ee1 Mon Sep 17 00:00:00 2001 From: Andrew Morgan <1342360+anoadragon453@users.noreply.github.com> Date: Tue, 8 Jun 2021 13:39:45 +0100 Subject: [PATCH 094/112] Apply suggestions from code review I blame the above mess on the github outage o:) Co-authored-by: Patrick Cloke --- synapse/config/experimental.py | 4 +--- synapse/federation/federation_client.py | 3 +-- 2 files changed, 2 insertions(+), 5 deletions(-) diff --git a/synapse/config/experimental.py b/synapse/config/experimental.py index 937d2786a6a0..37668079e7b2 100644 --- a/synapse/config/experimental.py +++ b/synapse/config/experimental.py @@ -35,6 +35,4 @@ def read_config(self, config: JsonDict, **kwargs): self.msc2403_enabled = experimental.get("msc2403_enabled", False) # type: bool if self.msc2403_enabled: # Enable the MSC2403 unstable room version - KNOWN_ROOM_VERSIONS.update( - {RoomVersions.MSC2403.identifier: RoomVersions.MSC2403} - ) + KNOWN_ROOM_VERSIONS[RoomVersions.MSC2403.identifier] = RoomVersions.MSC2403 diff --git a/synapse/federation/federation_client.py b/synapse/federation/federation_client.py index 07c9541e65f4..fc93990e25f5 100644 --- a/synapse/federation/federation_client.py +++ b/synapse/federation/federation_client.py @@ -1,6 +1,5 @@ -# Copyright 2015, 2016 OpenMarket Ltd +# Copyright 2015-2021 The Matrix.org Foundation C.I.C. # Copyright 2020 Sorunome -# Copyright 2020 The Matrix.org Foundation C.I.C. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. From 7932671e6122b66b3207bc9aeb17a2c7a0944dd2 Mon Sep 17 00:00:00 2001 From: Andrew Morgan Date: Tue, 8 Jun 2021 13:50:25 +0100 Subject: [PATCH 095/112] send_knock_v2 -> v1 --- synapse/federation/federation_client.py | 2 +- synapse/federation/transport/client.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/synapse/federation/federation_client.py b/synapse/federation/federation_client.py index fc93990e25f5..a927eecab0c3 100644 --- a/synapse/federation/federation_client.py +++ b/synapse/federation/federation_client.py @@ -957,7 +957,7 @@ async def _do_send_knock(self, destination: str, pdu: EventBase) -> JsonDict: """ time_now = self._clock.time_msec() - return await self.transport_layer.send_knock_v2( + return await self.transport_layer.send_knock_v1( destination=destination, room_id=pdu.room_id, event_id=pdu.event_id, diff --git a/synapse/federation/transport/client.py b/synapse/federation/transport/client.py index 0af48ff6edd6..902dc92a82a3 100644 --- a/synapse/federation/transport/client.py +++ b/synapse/federation/transport/client.py @@ -333,7 +333,7 @@ async def send_leave_v2(self, destination, room_id, event_id, content): return response @log_function - async def send_knock_v2( + async def send_knock_v1( self, destination: str, room_id: str, From 5d27f534e066ffe011e0341419f5f239f23d7cd0 Mon Sep 17 00:00:00 2001 From: Andrew Morgan Date: Tue, 8 Jun 2021 13:52:24 +0100 Subject: [PATCH 096/112] fix type hint --- synapse/replication/http/membership.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/synapse/replication/http/membership.py b/synapse/replication/http/membership.py index ea97ed5df9e1..043c25f63d9e 100644 --- a/synapse/replication/http/membership.py +++ b/synapse/replication/http/membership.py @@ -114,8 +114,8 @@ class ReplicationRemoteKnockRestServlet(ReplicationEndpoint): NAME = "remote_knock" PATH_ARGS = ("room_id", "user_id") - def __init__(self, hs): - super().__init__(hs: "HomeServer") + def __init__(self, hs: "HomeServer"): + super().__init__(hs) self.federation_handler = hs.get_federation_handler() self.store = hs.get_datastore() From 56d84e5ceffb739c081e09b26293a224b8a1bfb6 Mon Sep 17 00:00:00 2001 From: Andrew Morgan Date: Tue, 8 Jun 2021 17:18:38 +0100 Subject: [PATCH 097/112] Fix type hints on request.args --- synapse/rest/client/v1/room.py | 18 +++++++++++++----- synapse/rest/client/v2_alpha/knock.py | 7 +++++-- 2 files changed, 18 insertions(+), 7 deletions(-) diff --git a/synapse/rest/client/v1/room.py b/synapse/rest/client/v1/room.py index 86c0e3b011c7..ddfeee74e3e3 100644 --- a/synapse/rest/client/v1/room.py +++ b/synapse/rest/client/v1/room.py @@ -16,7 +16,7 @@ """ This module contains REST servlets to do with rooms: /rooms/ """ import logging import re -from typing import TYPE_CHECKING, Optional, Tuple +from typing import TYPE_CHECKING, Dict, List, Optional, Tuple from urllib import parse as urlparse from synapse.api.constants import EventTypes, Membership @@ -278,7 +278,12 @@ def register(self, http_server): PATTERNS = "/join/(?P[^/]*)" register_txn_path(self, PATTERNS, http_server) - async def on_POST(self, request, room_identifier, txn_id=None): + async def on_POST( + self, + request: SynapseRequest, + room_identifier: str, + txn_id: Optional[str] = None, + ): requester = await self.auth.get_user_by_req(request, allow_guest=True) try: @@ -291,16 +296,19 @@ async def on_POST(self, request, room_identifier, txn_id=None): if RoomID.is_valid(room_identifier): room_id = room_identifier try: + # twisted.web.server.Request.args is incorrectly defined as Optional[Any] + args: Dict[bytes, List[bytes]] = request.args # type: ignore + remote_room_hosts = parse_strings_from_args( - request.args, "server_name", required=False + args, "server_name", required=False ) except KeyError: remote_room_hosts = None elif RoomAlias.is_valid(room_identifier): handler = self.room_member_handler room_alias = RoomAlias.from_string(room_identifier) - room_id, remote_room_hosts = await handler.lookup_room_alias(room_alias) - room_id = room_id.to_string() + room_id_obj, remote_room_hosts = await handler.lookup_room_alias(room_alias) + room_id = room_id_obj.to_string() else: raise SynapseError( 400, "%s was not legal room ID or room alias" % (room_identifier,) diff --git a/synapse/rest/client/v2_alpha/knock.py b/synapse/rest/client/v2_alpha/knock.py index 91de314310a5..bdae7b85eb94 100644 --- a/synapse/rest/client/v2_alpha/knock.py +++ b/synapse/rest/client/v2_alpha/knock.py @@ -14,7 +14,7 @@ # See the License for the specific language governing permissions and # limitations under the License. import logging -from typing import TYPE_CHECKING, Optional, Tuple +from typing import TYPE_CHECKING, Dict, List, Optional, Tuple from twisted.web.server import Request @@ -69,8 +69,11 @@ async def on_POST( if RoomID.is_valid(room_identifier): room_id = room_identifier try: + # twisted.web.server.Request.args is incorrectly defined as Optional[Any] + args: Dict[bytes, List[bytes]] = request.args # type: ignore + remote_room_hosts = parse_strings_from_args( - request.args, "server_name", required=False + args, "server_name", required=False ) except KeyError: remote_room_hosts = None From 2fa4274a8a63f9fdb8e79a9d2d758426cd00e000 Mon Sep 17 00:00:00 2001 From: Andrew Morgan Date: Tue, 8 Jun 2021 17:23:01 +0100 Subject: [PATCH 098/112] Remove room_stats_historical database alteration room_stats_historical doesn't appear to be ever read from. See #9602. --- .../schema/main/delta/59/11add_knock_members_to_stats.sql | 1 - 1 file changed, 1 deletion(-) diff --git a/synapse/storage/schema/main/delta/59/11add_knock_members_to_stats.sql b/synapse/storage/schema/main/delta/59/11add_knock_members_to_stats.sql index c1e740f848eb..9ef0d5110d38 100644 --- a/synapse/storage/schema/main/delta/59/11add_knock_members_to_stats.sql +++ b/synapse/storage/schema/main/delta/59/11add_knock_members_to_stats.sql @@ -14,4 +14,3 @@ */ ALTER TABLE room_stats_current ADD COLUMN knocked_members INT NOT NULL DEFAULT '0'; -ALTER TABLE room_stats_historical ADD COLUMN knocked_members BIGINT NOT NULL DEFAULT '0'; From 8f24db523c227c95ca4919082701d2eb91be2d9b Mon Sep 17 00:00:00 2001 From: Andrew Morgan Date: Tue, 8 Jun 2021 17:30:29 +0100 Subject: [PATCH 099/112] check_sigs_and_hashes -> check_sigs_and_hash another renamed method --- tests/federation/transport/test_knocking.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/federation/transport/test_knocking.py b/tests/federation/transport/test_knocking.py index 3ef41ddd4f72..d38deed1ff8b 100644 --- a/tests/federation/transport/test_knocking.py +++ b/tests/federation/transport/test_knocking.py @@ -202,10 +202,10 @@ def prepare(self, reactor, clock, homeserver): # Note that these checks are not relevant to this test case. # Have this homeserver auto-approve all event signature checking. - def approve_all_signature_checking(_, ev): - return [succeed(ev[0])] + async def approve_all_signature_checking(_, pdu): + return pdu - homeserver.get_federation_server()._check_sigs_and_hashes = ( + homeserver.get_federation_server()._check_sigs_and_hash = ( approve_all_signature_checking ) From 01457e6f78aa12b2508a87a43726918e1d5bc477 Mon Sep 17 00:00:00 2001 From: Andrew Morgan Date: Tue, 8 Jun 2021 17:38:18 +0100 Subject: [PATCH 100/112] lint --- tests/federation/transport/test_knocking.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/tests/federation/transport/test_knocking.py b/tests/federation/transport/test_knocking.py index d38deed1ff8b..efebe0db9f73 100644 --- a/tests/federation/transport/test_knocking.py +++ b/tests/federation/transport/test_knocking.py @@ -15,8 +15,6 @@ from collections import OrderedDict from typing import Dict, List -from twisted.internet.defer import succeed - from synapse.api.constants import EventTypes, JoinRules, Membership from synapse.api.room_versions import RoomVersions from synapse.events import builder From 1576f2305de42b66d35d21aae9d3a49f74237de1 Mon Sep 17 00:00:00 2001 From: Andrew Morgan Date: Tue, 8 Jun 2021 17:46:33 +0100 Subject: [PATCH 101/112] Restrict event_auth knock checks to room versions that support knocking --- synapse/event_auth.py | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/synapse/event_auth.py b/synapse/event_auth.py index 77831b62def3..187cefa14a16 100644 --- a/synapse/event_auth.py +++ b/synapse/event_auth.py @@ -258,7 +258,11 @@ def _is_membership_change_allowed( caller_in_room = caller and caller.membership == Membership.JOIN caller_invited = caller and caller.membership == Membership.INVITE - caller_knocked = caller and caller.membership == Membership.KNOCK + caller_knocked = ( + caller + and room_version.msc2403_knocking + and caller.membership == Membership.KNOCK + ) # get info about the target key = (EventTypes.Member, target_user_id) @@ -302,7 +306,9 @@ def _is_membership_change_allowed( return # Require the user to be in the room for membership changes other than join/knock. - if Membership.JOIN != membership and Membership.KNOCK != membership: + if Membership.JOIN != membership and ( + RoomVersion.msc2403_knocking and Membership.KNOCK != membership + ): # If the user has been invited or has knocked, they are allowed to change their # membership event to leave if ( @@ -344,7 +350,9 @@ def _is_membership_change_allowed( and join_rule == JoinRules.MSC3083_RESTRICTED ): pass - elif join_rule in (JoinRules.INVITE, JoinRules.KNOCK): + elif join_rule == JoinRules.INVITE or ( + room_version.msc2403_knocking and join_rule == JoinRules.KNOCK + ): if not caller_in_room and not caller_invited: raise AuthError(403, "You are not invited to this room.") else: @@ -363,7 +371,7 @@ def _is_membership_change_allowed( elif Membership.BAN == membership: if user_level < ban_level or user_level <= target_level: raise AuthError(403, "You don't have permission to ban") - elif Membership.KNOCK == membership: + elif room_version.msc2403_knocking and Membership.KNOCK == membership: if join_rule != JoinRules.KNOCK: raise AuthError(403, "You don't have permission to knock") elif target_user_id != event.user_id: From 813fc249de0f59842ef683ad7a0dc777008d3b82 Mon Sep 17 00:00:00 2001 From: Andrew Morgan Date: Tue, 8 Jun 2021 18:52:15 +0100 Subject: [PATCH 102/112] Generalise message for incompatible room version over federation --- synapse/api/errors.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/synapse/api/errors.py b/synapse/api/errors.py index 0231c79079e1..4cb8bbaf701e 100644 --- a/synapse/api/errors.py +++ b/synapse/api/errors.py @@ -449,7 +449,7 @@ def __init__(self, room_version: str): super().__init__( code=400, msg="Your homeserver does not support the features required to " - "join this room", + "interact with this room", errcode=Codes.INCOMPATIBLE_ROOM_VERSION, ) From bea438e3205008ca4f27f88ea4107a9bfa11546a Mon Sep 17 00:00:00 2001 From: Andrew Morgan Date: Tue, 8 Jun 2021 19:15:50 +0100 Subject: [PATCH 103/112] Room version checks for knocking-related actions This commit adds checks for the room version in: * Building client and federation servlets * Building membership events * Attempting to knock on a room over federation * In the event auth rules themselves --- synapse/event_auth.py | 1 + synapse/federation/federation_client.py | 15 ++++++++++++++- synapse/federation/federation_server.py | 14 ++++++++++++++ synapse/federation/transport/client.py | 8 +++++++- synapse/federation/transport/server.py | 21 ++++++++++++++++++--- synapse/rest/__init__.py | 5 ++++- 6 files changed, 58 insertions(+), 6 deletions(-) diff --git a/synapse/event_auth.py b/synapse/event_auth.py index 187cefa14a16..33d7c6024147 100644 --- a/synapse/event_auth.py +++ b/synapse/event_auth.py @@ -289,6 +289,7 @@ def _is_membership_change_allowed( { "caller_in_room": caller_in_room, "caller_invited": caller_invited, + "caller_knocked": caller_knocked, "target_banned": target_banned, "target_in_room": target_in_room, "membership": membership, diff --git a/synapse/federation/federation_client.py b/synapse/federation/federation_client.py index 40c27583b9a2..03ec14ce877a 100644 --- a/synapse/federation/federation_client.py +++ b/synapse/federation/federation_client.py @@ -90,6 +90,7 @@ def __init__(self, hs: "HomeServer"): self._clock.looping_call(self._clear_tried_cache, 60 * 1000) self.state = hs.get_state_handler() self.transport_layer = hs.get_federation_transport_client() + self._msc2403_enabled = hs.config.experimental.msc2403_enabled self.hostname = hs.hostname self.signing_key = hs.signing_key @@ -620,7 +621,12 @@ async def make_membership_event( SynapseError: if the chosen remote server returns a 300/400 code, or no servers successfully handle the request. """ - valid_memberships = {Membership.JOIN, Membership.LEAVE, Membership.KNOCK} + valid_memberships = {Membership.JOIN, Membership.LEAVE} + + # Allow knocking if the feature is enabled + if self._msc2403_enabled: + valid_memberships.add(Membership.KNOCK) + if membership not in valid_memberships: raise RuntimeError( "make_membership_event called with membership='%s', must be one of %s" @@ -639,6 +645,13 @@ async def send_request(destination: str) -> Tuple[str, EventBase, RoomVersion]: if not room_version: raise UnsupportedRoomVersionError() + if not room_version.msc2403_knocking and membership == Membership.KNOCK: + raise SynapseError( + 400, + "This room version does not support knocking", + errcode=Codes.FORBIDDEN, + ) + pdu_dict = ret.get("event", None) if not isinstance(pdu_dict, dict): raise InvalidResponseError("Bad 'event' field in response") diff --git a/synapse/federation/federation_server.py b/synapse/federation/federation_server.py index 0fd848b02963..551a59a9f46d 100644 --- a/synapse/federation/federation_server.py +++ b/synapse/federation/federation_server.py @@ -615,6 +615,13 @@ async def on_make_knock_request( ) raise IncompatibleRoomVersionError(room_version=room_version) + if not KNOWN_ROOM_VERSIONS[room_version].msc2403_knocking: + raise SynapseError( + 403, + "This room version does not support knocking", + errcode=Codes.FORBIDDEN, + ) + pdu = await self.handler.on_make_knock_request(origin, room_id, user_id) time_now = self._clock.time_msec() return {"event": pdu.get_pdu_json(time_now), "room_version": room_version} @@ -641,6 +648,13 @@ async def on_send_knock_request( logger.debug("on_send_knock_request: content: %s", content) room_version = await self.store.get_room_version(room_id) + if not room_version.msc2403_knocking: + raise SynapseError( + 403, + "This room version does not support knocking", + errcode=Codes.FORBIDDEN, + ) + pdu = event_from_pdu_json(content, room_version) origin_host, _ = parse_server_name(origin) diff --git a/synapse/federation/transport/client.py b/synapse/federation/transport/client.py index 902dc92a82a3..af0c679ed987 100644 --- a/synapse/federation/transport/client.py +++ b/synapse/federation/transport/client.py @@ -47,6 +47,7 @@ class TransportLayerClient: def __init__(self, hs): self.server_name = hs.hostname self.client = hs.get_federation_http_client() + self._msc2403_enabled = hs.config.experimental.msc2403_enabled @log_function def get_room_state_ids(self, destination, room_id, event_id): @@ -220,7 +221,12 @@ async def make_membership_event( Fails with ``FederationDeniedError`` if the remote destination is not in our federation whitelist """ - valid_memberships = {Membership.JOIN, Membership.LEAVE, Membership.KNOCK} + valid_memberships = {Membership.JOIN, Membership.LEAVE} + + # Allow knocking if the feature is enabled + if self._msc2403_enabled: + valid_memberships.add(Membership.KNOCK) + if membership not in valid_memberships: raise RuntimeError( "make_membership_event called with membership='%s', must be one of %s" diff --git a/synapse/federation/transport/server.py b/synapse/federation/transport/server.py index f983336d531a..b9aa29a90afd 100644 --- a/synapse/federation/transport/server.py +++ b/synapse/federation/transport/server.py @@ -584,7 +584,7 @@ async def on_GET(self, origin, content, query, room_id, user_id): return 200, content -class FederationV2SendKnockServlet(BaseFederationServerServlet): +class FederationV1SendKnockServlet(BaseFederationServerServlet): PATH = "/send_knock/(?P[^/]*)/(?P[^/]*)" PREFIX = FEDERATION_UNSTABLE_PREFIX + "/xyz.amorgan.knock" @@ -1595,13 +1595,11 @@ async def on_GET(self, origin, content, query, room_id): FederationQueryServlet, FederationMakeJoinServlet, FederationMakeLeaveServlet, - FederationMakeKnockServlet, FederationEventServlet, FederationV1SendJoinServlet, FederationV2SendJoinServlet, FederationV1SendLeaveServlet, FederationV2SendLeaveServlet, - FederationV2SendKnockServlet, FederationV1InviteServlet, FederationV2InviteServlet, FederationGetMissingEventsServlet, @@ -1655,6 +1653,13 @@ async def on_GET(self, origin, content, query, room_id): FederationGroupsRenewAttestaionServlet, ) # type: Tuple[Type[BaseFederationServlet], ...] + +MSC2403_SERVLET_CLASSES = ( + FederationV1SendKnockServlet, + FederationMakeKnockServlet, +) + + DEFAULT_SERVLET_GROUPS = ( "federation", "room_list", @@ -1697,6 +1702,16 @@ def register_servlets( server_name=hs.hostname, ).register(resource) + # Register msc2403 (knocking) servlets if the feature is enabled + if hs.config.experimental.msc2403_enabled: + for servletclass in MSC2403_SERVLET_CLASSES: + servletclass( + hs=hs, + authenticator=authenticator, + ratelimiter=ratelimiter, + server_name=hs.hostname, + ).register(resource) + if "openid" in servlet_groups: for servletclass in OPENID_SERVLET_CLASSES: servletclass( diff --git a/synapse/rest/__init__.py b/synapse/rest/__init__.py index d29f2fea5ed3..138411ad19ab 100644 --- a/synapse/rest/__init__.py +++ b/synapse/rest/__init__.py @@ -121,7 +121,10 @@ def register_servlets(client_resource, hs): account_validity.register_servlets(hs, client_resource) relations.register_servlets(hs, client_resource) password_policy.register_servlets(hs, client_resource) - knock.register_servlets(hs, client_resource) + + # Register msc2403 (knocking) servlets if the feature is enabled + if hs.config.experimental.msc2403_enabled: + knock.register_servlets(hs, client_resource) # moving to /_synapse/admin admin.register_servlets_for_client_rest_resource(hs, client_resource) From 277e952b5bfd66c980ff736aba5f5b9714d8a9eb Mon Sep 17 00:00:00 2001 From: Andrew Morgan Date: Wed, 9 Jun 2021 14:44:50 +0100 Subject: [PATCH 104/112] Consolidate copyright headers --- synapse/federation/transport/server.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/synapse/federation/transport/server.py b/synapse/federation/transport/server.py index b9aa29a90afd..fe5fb6bee728 100644 --- a/synapse/federation/transport/server.py +++ b/synapse/federation/transport/server.py @@ -1,6 +1,4 @@ -# Copyright 2014-2016 OpenMarket Ltd -# Copyright 2018 New Vector Ltd -# Copyright 2019-2020 The Matrix.org Foundation C.I.C. +# Copyright 2014-2021 The Matrix.org Foundation C.I.C. # Copyright 2020 Sorunome # # Licensed under the Apache License, Version 2.0 (the "License"); From 2f73e32d35d7ebfbe373d6830e7fda936e449dc8 Mon Sep 17 00:00:00 2001 From: Andrew Morgan Date: Wed, 9 Jun 2021 14:47:11 +0100 Subject: [PATCH 105/112] Guard another check behind experimental knocking config option --- synapse/handlers/room_member.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/synapse/handlers/room_member.py b/synapse/handlers/room_member.py index a49a61a34c87..b8f2c5e94868 100644 --- a/synapse/handlers/room_member.py +++ b/synapse/handlers/room_member.py @@ -707,7 +707,10 @@ async def update_membership_locked( knock.event_id, txn_id, requester, content ) - elif effective_membership_state == Membership.KNOCK: + elif ( + self.config.experimental.msc2403_enabled + and effective_membership_state == Membership.KNOCK + ): if not is_host_in_room: # The knock needs to be sent over federation instead remote_room_hosts.append(get_domain_from_id(room_id)) From 1db05adc60298956d93fefe404c193aba79d8afb Mon Sep 17 00:00:00 2001 From: Andrew Morgan Date: Wed, 9 Jun 2021 15:34:03 +0100 Subject: [PATCH 106/112] Remove try/except on KeyError for query parameter parsing This was no longer needed after switching to parse_strings_from_args. --- synapse/rest/client/v1/room.py | 14 ++++++-------- synapse/rest/client/v2_alpha/knock.py | 16 +++++++--------- 2 files changed, 13 insertions(+), 17 deletions(-) diff --git a/synapse/rest/client/v1/room.py b/synapse/rest/client/v1/room.py index ddfeee74e3e3..16d087ea60f0 100644 --- a/synapse/rest/client/v1/room.py +++ b/synapse/rest/client/v1/room.py @@ -295,15 +295,13 @@ async def on_POST( if RoomID.is_valid(room_identifier): room_id = room_identifier - try: - # twisted.web.server.Request.args is incorrectly defined as Optional[Any] - args: Dict[bytes, List[bytes]] = request.args # type: ignore - remote_room_hosts = parse_strings_from_args( - args, "server_name", required=False - ) - except KeyError: - remote_room_hosts = None + # twisted.web.server.Request.args is incorrectly defined as Optional[Any] + args: Dict[bytes, List[bytes]] = request.args # type: ignore + + remote_room_hosts = parse_strings_from_args( + args, "server_name", required=False + ) elif RoomAlias.is_valid(room_identifier): handler = self.room_member_handler room_alias = RoomAlias.from_string(room_identifier) diff --git a/synapse/rest/client/v2_alpha/knock.py b/synapse/rest/client/v2_alpha/knock.py index bdae7b85eb94..14068df5e853 100644 --- a/synapse/rest/client/v2_alpha/knock.py +++ b/synapse/rest/client/v2_alpha/knock.py @@ -68,15 +68,13 @@ async def on_POST( if RoomID.is_valid(room_identifier): room_id = room_identifier - try: - # twisted.web.server.Request.args is incorrectly defined as Optional[Any] - args: Dict[bytes, List[bytes]] = request.args # type: ignore - - remote_room_hosts = parse_strings_from_args( - args, "server_name", required=False - ) - except KeyError: - remote_room_hosts = None + + # twisted.web.server.Request.args is incorrectly defined as Optional[Any] + args: Dict[bytes, List[bytes]] = request.args # type: ignore + + remote_room_hosts = parse_strings_from_args( + args, "server_name", required=False + ) elif RoomAlias.is_valid(room_identifier): handler = self.room_member_handler room_alias = RoomAlias.from_string(room_identifier) From 9c034ef2fe808f31d21fc83dd914866171d13790 Mon Sep 17 00:00:00 2001 From: Andrew Morgan Date: Wed, 9 Jun 2021 15:41:31 +0100 Subject: [PATCH 107/112] Remove unnecessary coding: utf-8 lines --- synapse/rest/client/v2_alpha/knock.py | 1 - tests/federation/transport/test_knocking.py | 1 - 2 files changed, 2 deletions(-) diff --git a/synapse/rest/client/v2_alpha/knock.py b/synapse/rest/client/v2_alpha/knock.py index 14068df5e853..f046bf9cb324 100644 --- a/synapse/rest/client/v2_alpha/knock.py +++ b/synapse/rest/client/v2_alpha/knock.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # Copyright 2020 Sorunome # Copyright 2020 The Matrix.org Foundation C.I.C. # diff --git a/tests/federation/transport/test_knocking.py b/tests/federation/transport/test_knocking.py index efebe0db9f73..121aa88cfa34 100644 --- a/tests/federation/transport/test_knocking.py +++ b/tests/federation/transport/test_knocking.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # Copyright 2020 Matrix.org Federation C.I.C # # Licensed under the Apache License, Version 2.0 (the "License"); From a2bd3456c4dad3ec8dde833df60afe5a8533aeb9 Mon Sep 17 00:00:00 2001 From: Andrew Morgan Date: Wed, 9 Jun 2021 15:42:57 +0100 Subject: [PATCH 108/112] lint --- synapse/handlers/room_member.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/synapse/handlers/room_member.py b/synapse/handlers/room_member.py index b8f2c5e94868..c26963b1e1de 100644 --- a/synapse/handlers/room_member.py +++ b/synapse/handlers/room_member.py @@ -708,8 +708,8 @@ async def update_membership_locked( ) elif ( - self.config.experimental.msc2403_enabled - and effective_membership_state == Membership.KNOCK + self.config.experimental.msc2403_enabled + and effective_membership_state == Membership.KNOCK ): if not is_host_in_room: # The knock needs to be sent over federation instead From 02942483a4f4b6382e5d427f7ec87b09936381e5 Mon Sep 17 00:00:00 2001 From: Andrew Morgan Date: Wed, 9 Jun 2021 15:53:43 +0100 Subject: [PATCH 109/112] Add knocked_members column back to room_stats_historical This table's usefulness is debatable (see #9602), but is currently used by both the codebase and tests. Thus for now I'm leaving it in, but it may well be removed in a future PR. --- .../schema/main/delta/59/11add_knock_members_to_stats.sql | 1 + 1 file changed, 1 insertion(+) diff --git a/synapse/storage/schema/main/delta/59/11add_knock_members_to_stats.sql b/synapse/storage/schema/main/delta/59/11add_knock_members_to_stats.sql index 9ef0d5110d38..56c0ad000329 100644 --- a/synapse/storage/schema/main/delta/59/11add_knock_members_to_stats.sql +++ b/synapse/storage/schema/main/delta/59/11add_knock_members_to_stats.sql @@ -14,3 +14,4 @@ */ ALTER TABLE room_stats_current ADD COLUMN knocked_members INT NOT NULL DEFAULT '0'; +ALTER TABLE room_stats_historical ADD COLUMN knocked_members BIGINT NOT NULL DEFAULT '0'; \ No newline at end of file From c5788b87dad35ab64f1a5b1bbeb0dd25d668a11a Mon Sep 17 00:00:00 2001 From: Andrew Morgan Date: Wed, 9 Jun 2021 17:35:50 +0100 Subject: [PATCH 110/112] Nuke remaining coding: utf-8 line --- synapse/config/account_validity.py | 1 - 1 file changed, 1 deletion(-) diff --git a/synapse/config/account_validity.py b/synapse/config/account_validity.py index c58a7d95a785..957de7f3a613 100644 --- a/synapse/config/account_validity.py +++ b/synapse/config/account_validity.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # Copyright 2020 The Matrix.org Foundation C.I.C. # # Licensed under the Apache License, Version 2.0 (the "License"); From 15c23c4e4502c4f21b8c4c4f158335aa173cb2be Mon Sep 17 00:00:00 2001 From: Andrew Morgan Date: Wed, 9 Jun 2021 17:51:31 +0100 Subject: [PATCH 111/112] Ensure we check that we currently support the desired room version during make_knock We may not if we create a knock room when knocking is enabled, then disabling knocking. We don't want to allow knocks in that case. --- synapse/federation/federation_server.py | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/synapse/federation/federation_server.py b/synapse/federation/federation_server.py index 551a59a9f46d..e5b9fb42f47a 100644 --- a/synapse/federation/federation_server.py +++ b/synapse/federation/federation_server.py @@ -608,14 +608,17 @@ async def on_make_knock_request( origin_host, _ = parse_server_name(origin) await self.check_server_matches_acl(origin_host, room_id) - room_version = await self.store.get_room_version_id(room_id) - if room_version not in supported_versions: + room_version = await self.store.get_room_version(room_id) + + # Check that this room version is supported by the remote homeserver + if room_version.identifier not in supported_versions: logger.warning( "Room version %s not in %s", room_version, supported_versions ) raise IncompatibleRoomVersionError(room_version=room_version) - if not KNOWN_ROOM_VERSIONS[room_version].msc2403_knocking: + # Check that this room supports knocking as defined by its room version + if not room_version.msc2403_knocking: raise SynapseError( 403, "This room version does not support knocking", @@ -648,6 +651,8 @@ async def on_send_knock_request( logger.debug("on_send_knock_request: content: %s", content) room_version = await self.store.get_room_version(room_id) + + # Check that this room supports knocking as defined by its room version if not room_version.msc2403_knocking: raise SynapseError( 403, From da85886bec05818ea3ebb7f18e4f5372aa8c7fa7 Mon Sep 17 00:00:00 2001 From: Andrew Morgan Date: Wed, 9 Jun 2021 18:38:05 +0100 Subject: [PATCH 112/112] Don't accidentally use a RoomVersion object where we want a str --- synapse/federation/federation_server.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/synapse/federation/federation_server.py b/synapse/federation/federation_server.py index e5b9fb42f47a..2b07f1852953 100644 --- a/synapse/federation/federation_server.py +++ b/synapse/federation/federation_server.py @@ -613,9 +613,9 @@ async def on_make_knock_request( # Check that this room version is supported by the remote homeserver if room_version.identifier not in supported_versions: logger.warning( - "Room version %s not in %s", room_version, supported_versions + "Room version %s not in %s", room_version.identifier, supported_versions ) - raise IncompatibleRoomVersionError(room_version=room_version) + raise IncompatibleRoomVersionError(room_version=room_version.identifier) # Check that this room supports knocking as defined by its room version if not room_version.msc2403_knocking: @@ -627,7 +627,10 @@ async def on_make_knock_request( pdu = await self.handler.on_make_knock_request(origin, room_id, user_id) time_now = self._clock.time_msec() - return {"event": pdu.get_pdu_json(time_now), "room_version": room_version} + return { + "event": pdu.get_pdu_json(time_now), + "room_version": room_version.identifier, + } async def on_send_knock_request( self,